diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md
index 4bb592733..61044f207 100644
--- a/src/plugins/AIComponents/README.md
+++ b/src/plugins/AIComponents/README.md
@@ -1,6 +1,6 @@
# AI components (`stream-chat-react/ai-components`)
-AI-first building blocks for Stream Chat: a message composer with attachments and speech-to-text, a Markdown renderer with syntax highlighting and Chart.js support, a typewriter-style streaming message and an "AI is thinking" indicator.
+AI-first building blocks for Stream Chat: a message composer with attachments and speech-to-text, a Markdown renderer with syntax highlighting and Chart.js support, a typewriter-style streaming message, a hook that reveals text at the pace a voice agent speaks it, and an "AI is thinking" indicator.
These components used to ship as the separate package `@stream-io/chat-react-ai`. They now live in `stream-chat-react` behind the optional entry point `stream-chat-react/ai-components`, with a stylesheet at `stream-chat-react/css/ai-components.css`. The main entry point (`stream-chat-react`) never imports from this plugin, so apps that do not use it pay nothing for it.
@@ -32,8 +32,8 @@ Load `ai-components.css` in addition to `stream-chat-react/css/index.css`.
From `stream-chat-react/ai-components`:
- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator`, `VoiceWave`, `VoiceGlow`
-- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useAudioLevel`
-- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `VoiceAgentState`, `VoiceGlowRendering`, `VoiceVisualizationProps`, `VoiceWaveProps`, `VoiceGlowProps`
+- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useAudioLevel`, `useHeardText`
+- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `VoiceAgentState`, `VoiceGlowRendering`, `VoiceVisualizationProps`, `VoiceWaveProps`, `VoiceGlowProps`, `HeardTextProgress`, `HeardTextFunction`, `HeardTextSubscribe`, `UseHeardTextParams`, `UseHeardTextResult`
`useMessageTextStreaming` is not exported from here; import it from the main entry point (`stream-chat-react`).
@@ -618,6 +618,65 @@ function VoiceInput() {
}
```
+### `useHeardText`
+
+Reveals a text at the pace the user hears it spoken, for apps that talk to a voice AI agent: the agent's reply is written into the chat as the model generates it, far ahead of the agent's voice. The hook is standalone (React only, no chat SDK imports) and takes the speech progress from the app as a plain function, so it works with any voice backend.
+
+```ts
+type HeardTextProgress = { chars: number; done: boolean };
+type HeardTextFunction = (now: number) => HeardTextProgress | undefined;
+type HeardTextSubscribe = (listener: () => void) => () => void;
+
+const useHeardText: (params: UseHeardTextParams) => UseHeardTextResult;
+```
+
+`heard(now)` returns how many characters of the text the user has heard by `now` (a `performance.now()` timestamp) and whether the speech for it has finished. `chars` counts Unicode code points (`Array.from(text).length`), not UTF-16 code units; `Infinity` means the whole text. Return `undefined` only for a text that is not spoken at all. A text that will be spoken but whose audio has not started, or that the voice source does not know yet, must return `{ chars: 0, done: false }`: while `heard` returns `undefined` the full text is shown, and the reveal never moves back past what has been shown. `heard` is also called during render, so keep it free of side effects.
+
+#### Props
+
+| Name | Type | Required | Description |
+| ------------ | -------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `text` | `string` | yes | The full text to reveal. |
+| `heard` | `HeardTextFunction` | yes | The speech progress of `text`. |
+| `subscribe` | `HeardTextSubscribe` | no | Change notifications from the speech progress source; each one makes the hook check `heard` again. See below. |
+| `generating` | `boolean` | no | `true` while the text is still being written. The full text is revealed once `heard` reports `done` and this is falsy. |
+| `live` | `boolean` | no | Defaults to `true`. `false` reveals the full text at once, for example for a message loaded from history. See below. |
+
+#### Returns
+
+| Name | Type | Description |
+| ----------- | ------------ | ---------------------------------------------------------------------------------- |
+| `heardText` | `string` | The part of the text the user has heard (the full text when not paced). |
+| `isPaced` | `boolean` | `false` when `heard` returned `undefined`; `heardText` is then the full text. |
+| `skip` | `() => void` | Reveal the full text from now on, for example when the user stops the AI response. |
+
+Behaviour:
+
+- It reveals the first `floor(chars)` code points of `text` and never moves backwards while `text` grows or is truncated. Emoji and other astral characters are never split.
+- When `text` is replaced by a different text (the next reply rendered by the same component), the reveal starts over: the length shown and `skip()` are reset, and `live` is read again. `live` is otherwise read only at mount.
+- While the text is paced and its speech is not done, it samples `heard` on every animation frame, and stops when the speech is done or the component unmounts. `subscribe` notifications and changes of `text` or `generating` make it check again. Without `subscribe`, nothing else does: pass it when `heard` can change after the speech is done, or start returning a value after returning `undefined`.
+- If `heard` returns `undefined` at first and a value later, the reveal continues from the length already shown.
+- `heard` and `subscribe` are read through refs: passing new functions on every render (inline arrow functions) does not restart the pacing. `subscribe` is called once and unsubscribed on unmount; to switch to another speech source, remount the component (for example with a `key`).
+
+#### Example
+
+```tsx
+import { useHeardText } from 'stream-chat-react/ai-components';
+
+function SpokenReply({ generating, session, text, turnId }) {
+ const { heardText } = useHeardText({
+ generating,
+ heard: (now) => session.speech.heard(turnId, now),
+ subscribe: (listener) => session.speech.subscribe(listener),
+ text,
+ });
+
+ return
{heardText}
;
+}
+```
+
+A drop-in for the `StreamedMessageText` component slot of `stream-chat-react`, built on this hook, is shown in the docs.
+
### `useAudioLevel`
Reads the live level of a `MediaStream`'s audio (a microphone from `getUserMedia({ audio: true })`, or a remote track) for the `audioLevel` prop of `VoiceWave` and `VoiceGlow`.
diff --git a/src/plugins/AIComponents/__tests__/useHeardText.test.ts b/src/plugins/AIComponents/__tests__/useHeardText.test.ts
new file mode 100644
index 000000000..8d3301964
--- /dev/null
+++ b/src/plugins/AIComponents/__tests__/useHeardText.test.ts
@@ -0,0 +1,453 @@
+import { act, renderHook } from '@testing-library/react';
+
+import { useHeardText } from '..';
+import type {
+ HeardTextFunction,
+ HeardTextProgress,
+ HeardTextSubscribe,
+ UseHeardTextParams,
+} from '..';
+
+const TEXT = 'Hello there, how can I help you today?';
+const FRAME_MS = 16;
+
+const advance = (ms: number) =>
+ act(() => {
+ vi.advanceTimersByTime(ms);
+ });
+
+const renderHeardText = (initialProps: UseHeardTextParams) =>
+ renderHook((props: UseHeardTextParams) => useHeardText(props), { initialProps });
+
+describe('useHeardText', () => {
+ beforeEach(() => {
+ vi.useFakeTimers({
+ toFake: [
+ 'setTimeout',
+ 'clearTimeout',
+ 'setInterval',
+ 'clearInterval',
+ 'requestAnimationFrame',
+ 'cancelAnimationFrame',
+ 'performance',
+ 'Date',
+ ],
+ });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('reveals the text at the pace it is heard', () => {
+ const start = performance.now();
+ // 20 characters per second
+ const heard = vi.fn((now) => ({
+ chars: ((now - start) / 1000) * 20,
+ done: false,
+ }));
+ const { result } = renderHeardText({ heard, text: TEXT });
+
+ expect(result.current.heardText).toBe('');
+ expect(result.current.isPaced).toBe(true);
+
+ // requestAnimationFrame fires every 16 ms under fake timers
+ advance(512);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 10));
+
+ advance(512);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 20));
+
+ advance(5000);
+ // clamped to the text length, the full text while speech is still in progress
+ expect(result.current.heardText).toBe(TEXT);
+ expect(heard).toHaveBeenCalledWith(expect.any(Number));
+ });
+
+ it('floors fractional character counts', () => {
+ const { result } = renderHeardText({
+ heard: () => ({ chars: 4.9, done: false }),
+ text: TEXT,
+ });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 4));
+ });
+
+ it('treats NaN, negative and non-numeric counts as nothing heard', () => {
+ let progress: HeardTextProgress = { chars: Number.NaN, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: TEXT });
+ expect(result.current.heardText).toBe('');
+
+ progress = { chars: -3, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe('');
+
+ progress = { chars: Number.NEGATIVE_INFINITY, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe('');
+
+ // a JS caller returning no count at all
+ progress = { done: false } as unknown as HeardTextProgress;
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe('');
+ });
+
+ it('treats an infinite count as the whole text heard', () => {
+ const { result } = renderHeardText({
+ heard: () => ({ chars: Number.POSITIVE_INFINITY, done: false }),
+ text: TEXT,
+ });
+ expect(result.current).toMatchObject({ heardText: TEXT, isPaced: true });
+ });
+
+ it('never moves the reveal backwards', () => {
+ let progress: HeardTextProgress = { chars: 12, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 12));
+
+ progress = { chars: 3, done: false };
+ advance(FRAME_MS * 3);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 12));
+
+ progress = { chars: 15, done: false };
+ advance(FRAME_MS * 2);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 15));
+ });
+
+ it('reveals the full text once the speech is done and the text is not generating', () => {
+ let progress: HeardTextProgress = { chars: 5, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 5));
+
+ progress = { chars: 8, done: true };
+ advance(FRAME_MS * 2);
+ expect(result.current.heardText).toBe(TEXT);
+ });
+
+ it('keeps pacing a done text while it is still generating', () => {
+ const heard: HeardTextFunction = () => ({ chars: 8, done: true });
+ const { rerender, result } = renderHeardText({ generating: true, heard, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 8));
+
+ rerender({ generating: false, heard, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT);
+ });
+
+ it('stops ticking once the speech is done', () => {
+ const heard = vi.fn(() => ({ chars: 8, done: true }));
+ renderHeardText({ generating: true, heard, text: TEXT });
+ const callCount = heard.mock.calls.length;
+
+ advance(FRAME_MS * 20);
+ expect(heard).toHaveBeenCalledTimes(callCount);
+ });
+
+ it('stops ticking on unmount', () => {
+ const heard = vi.fn(() => ({ chars: 1, done: false }));
+ const { unmount } = renderHeardText({ heard, text: TEXT });
+ advance(FRAME_MS * 3);
+ unmount();
+ const callCount = heard.mock.calls.length;
+
+ advance(FRAME_MS * 20);
+ expect(heard).toHaveBeenCalledTimes(callCount);
+ });
+
+ it('falls back to the timeout when requestAnimationFrame is unavailable', () => {
+ vi.stubGlobal('requestAnimationFrame', undefined);
+ try {
+ let progress: HeardTextProgress = { chars: 2, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: TEXT });
+ progress = { chars: 6, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 6));
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it('wakes up when the subscribed source notifies and unsubscribes on unmount', () => {
+ let progress: HeardTextProgress = { chars: 2, done: true };
+ let notify: (() => void) | undefined;
+ const unsubscribe = vi.fn();
+ const subscribe = vi.fn((listener) => {
+ notify = listener;
+ return unsubscribe;
+ });
+ const { result, unmount } = renderHeardText({
+ generating: true,
+ heard: () => progress,
+ subscribe,
+ text: TEXT,
+ });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 2));
+
+ // settled (done), so no ticking until the source notifies
+ progress = { chars: 7, done: false };
+ advance(FRAME_MS * 3);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 2));
+
+ act(() => notify?.());
+ expect(result.current.heardText).toBe(TEXT.slice(0, 7));
+
+ // ticking again
+ progress = { chars: 9, done: false };
+ advance(FRAME_MS * 2);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 9));
+
+ expect(subscribe).toHaveBeenCalledTimes(1);
+ unmount();
+ expect(unsubscribe).toHaveBeenCalledTimes(1);
+ });
+
+ it('is not paced while heard returns undefined', () => {
+ const heard = vi.fn(() => undefined);
+ const { rerender, result } = renderHeardText({ heard, text: 'Hello' });
+ expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: false });
+
+ // no ticking while not paced
+ const callCount = heard.mock.calls.length;
+ advance(FRAME_MS * 10);
+ expect(heard).toHaveBeenCalledTimes(callCount);
+
+ // re-checked when the text changes
+ rerender({ heard, text: 'Hello world' });
+ expect(result.current).toMatchObject({ heardText: 'Hello world', isPaced: false });
+ expect(heard.mock.calls.length).toBeGreaterThan(callCount);
+ });
+
+ it('starts pacing from the length already shown when it becomes paced', () => {
+ let progress: HeardTextProgress | undefined;
+ let notify: (() => void) | undefined;
+ const subscribe: HeardTextSubscribe = (listener) => {
+ notify = listener;
+ return () => undefined;
+ };
+ const { rerender, result } = renderHeardText({
+ heard: () => progress,
+ subscribe,
+ text: 'Hello',
+ });
+ expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: false });
+
+ progress = { chars: 0, done: false };
+ act(() => notify?.());
+ // paced now, but does not move back past the 5 characters already shown
+ expect(result.current).toMatchObject({ heardText: 'Hello', isPaced: true });
+
+ rerender({ heard: () => progress, subscribe, text: 'Hello world' });
+ expect(result.current.heardText).toBe('Hello');
+
+ progress = { chars: 8, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe('Hello wo');
+ });
+
+ it('reveals the full text at once when not live', () => {
+ const { result } = renderHeardText({
+ generating: true,
+ heard: () => ({ chars: 0, done: false }),
+ live: false,
+ text: TEXT,
+ });
+ expect(result.current).toMatchObject({ heardText: TEXT, isPaced: true });
+ });
+
+ it('reads live only once, at mount', () => {
+ const heard: HeardTextFunction = () => ({ chars: 3, done: false });
+ const { rerender, result } = renderHeardText({ heard, live: true, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 3));
+
+ rerender({ heard, live: false, text: TEXT });
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 3));
+ });
+
+ it('does not restart pacing when heard and subscribe get a new identity each render', () => {
+ let progress: HeardTextProgress = { chars: 4, done: false };
+ const unsubscribe = vi.fn();
+ const subscribeImpl = vi.fn(() => unsubscribe);
+ const props = (): UseHeardTextParams => ({
+ // new function identities on every render, as with inline arrow functions
+ heard: () => progress,
+ subscribe: (listener) => subscribeImpl(listener),
+ text: TEXT,
+ });
+ const { rerender, result } = renderHeardText(props());
+ expect(result.current.heardText).toBe(TEXT.slice(0, 4));
+
+ for (let i = 0; i < 5; i++) rerender(props());
+ expect(subscribeImpl).toHaveBeenCalledTimes(1);
+ expect(unsubscribe).not.toHaveBeenCalled();
+
+ // the latest heard is the one being sampled
+ let latest: HeardTextProgress = { chars: 10, done: false };
+ rerender({ ...props(), heard: () => latest });
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 10));
+
+ latest = { chars: 12, done: false };
+ progress = { chars: 1, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 12));
+ });
+
+ it('skip reveals the full text from then on', () => {
+ const { result } = renderHeardText({
+ heard: () => ({ chars: 3, done: false }),
+ text: TEXT,
+ });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 3));
+ const { skip } = result.current;
+
+ act(() => skip());
+ expect(result.current.heardText).toBe(TEXT);
+ expect(result.current.skip).toBe(skip);
+
+ advance(FRAME_MS * 5);
+ expect(result.current.heardText).toBe(TEXT);
+ });
+
+ describe('when the text is replaced by a different text', () => {
+ const NEXT_TEXT = 'Sure, here is the forecast for tomorrow.';
+
+ it('starts a new reveal instead of carrying over the length already shown', () => {
+ let progress: HeardTextProgress = { chars: 100, done: false };
+ const heard: HeardTextFunction = () => progress;
+ const { rerender, result } = renderHeardText({ heard, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT);
+
+ // the next reply, whose audio has not started yet
+ progress = { chars: 0, done: false };
+ rerender({ heard, text: NEXT_TEXT });
+ expect(result.current).toMatchObject({ heardText: '', isPaced: true });
+
+ progress = { chars: 4, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe('Sure');
+ });
+
+ it('starts a new reveal after the previous text was shown unpaced', () => {
+ let progress: HeardTextProgress | undefined = undefined;
+ const heard: HeardTextFunction = () => progress;
+ const { rerender, result } = renderHeardText({ heard, text: TEXT });
+ expect(result.current).toMatchObject({ heardText: TEXT, isPaced: false });
+
+ progress = { chars: 2, done: false };
+ rerender({ heard, text: NEXT_TEXT });
+ expect(result.current).toMatchObject({ heardText: 'Su', isPaced: true });
+ });
+
+ it('clears a previous skip', () => {
+ const heard: HeardTextFunction = () => ({ chars: 3, done: false });
+ const { rerender, result } = renderHeardText({ heard, text: TEXT });
+ act(() => result.current.skip());
+ expect(result.current.heardText).toBe(TEXT);
+
+ rerender({ heard, text: NEXT_TEXT });
+ expect(result.current.heardText).toBe('Sur');
+ });
+
+ it('reads live again for the new text', () => {
+ const heard: HeardTextFunction = () => ({ chars: 3, done: false });
+ const { rerender, result } = renderHeardText({ heard, live: false, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT);
+
+ rerender({ heard, live: true, text: NEXT_TEXT });
+ expect(result.current.heardText).toBe('Sur');
+ });
+
+ it('keeps the reveal when the text is truncated, and does not reuse the cut length', () => {
+ let progress: HeardTextProgress = { chars: 12, done: false };
+ const heard: HeardTextFunction = () => progress;
+ const { rerender, result } = renderHeardText({ heard, text: TEXT });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 12));
+
+ // e.g. the reply was cut short when the user stopped it
+ rerender({ heard, text: TEXT.slice(0, 8) });
+ expect(result.current.heardText).toBe(TEXT.slice(0, 8));
+
+ // growing again from the truncated text continues from 8, not from 12
+ progress = { chars: 3, done: false };
+ rerender({ heard, text: `${TEXT.slice(0, 8)} again and again` });
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(TEXT.slice(0, 8));
+ });
+ });
+
+ describe('with emoji and other astral characters', () => {
+ // waving hand + skin tone modifier, mathematical script X, grinning face, the
+ // US flag (two regional indicators) and a BMP character: all but the last are
+ // surrogate pairs in UTF-16
+ const ASTRAL_TEXT = 'Hi \u{1F44B}\u{1F3FD} \u{1D4B3}\u{1F600}!\u{1F1FA}\u{1F1F8}é';
+ const CODE_POINTS = Array.from(ASTRAL_TEXT);
+ const LONE_SURROGATE =
+ /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? {
+ expect(CODE_POINTS.length).toBeLessThan(ASTRAL_TEXT.length);
+ let progress: HeardTextProgress = { chars: 0, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: ASTRAL_TEXT });
+
+ for (let chars = 1; chars <= CODE_POINTS.length; chars++) {
+ progress = { chars, done: false };
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(CODE_POINTS.slice(0, chars).join(''));
+ expect(result.current.heardText).not.toMatch(LONE_SURROGATE);
+ }
+ // the whole text once chars reaches the code point length, still speaking
+ expect(result.current.heardText).toBe(ASTRAL_TEXT);
+ });
+
+ it('never produces a lone surrogate for fractional or time based progress', () => {
+ const start = performance.now();
+ // 7.3 code points per second
+ const { result } = renderHeardText({
+ heard: (now) => ({ chars: ((now - start) / 1000) * 7.3, done: false }),
+ text: ASTRAL_TEXT,
+ });
+
+ for (let frame = 0; frame < 150; frame++) {
+ advance(FRAME_MS);
+ expect(result.current.heardText).not.toMatch(LONE_SURROGATE);
+ expect(ASTRAL_TEXT.startsWith(result.current.heardText)).toBe(true);
+ }
+ expect(result.current.heardText).toBe(ASTRAL_TEXT);
+ });
+
+ it('reveals the full text when chars exceeds the code point length', () => {
+ const { result } = renderHeardText({
+ heard: () => ({ chars: ASTRAL_TEXT.length + 10, done: false }),
+ text: ASTRAL_TEXT,
+ });
+ expect(result.current.heardText).toBe(ASTRAL_TEXT);
+ });
+
+ it('never moves the reveal backwards in code points', () => {
+ let progress: HeardTextProgress = { chars: 6, done: false };
+ const { result } = renderHeardText({ heard: () => progress, text: ASTRAL_TEXT });
+ expect(result.current.heardText).toBe(CODE_POINTS.slice(0, 6).join(''));
+
+ progress = { chars: 2, done: false };
+ advance(FRAME_MS * 3);
+ expect(result.current.heardText).toBe(CODE_POINTS.slice(0, 6).join(''));
+ });
+
+ it('keeps code point boundaries when the text grows', () => {
+ const heard: HeardTextFunction = () => ({ chars: 5, done: false });
+ const { rerender, result } = renderHeardText({
+ heard,
+ text: '\u{1F600}\u{1F600}',
+ });
+ expect(result.current.heardText).toBe('\u{1F600}\u{1F600}');
+
+ rerender({
+ heard,
+ text: '\u{1F600}\u{1F600}\u{1F44B}\u{1F44B}\u{1F44B}\u{1F44B}',
+ });
+ advance(FRAME_MS);
+ expect(result.current.heardText).toBe(
+ '\u{1F600}\u{1F600}\u{1F44B}\u{1F44B}\u{1F44B}',
+ );
+ });
+ });
+});
diff --git a/src/plugins/AIComponents/hooks/useHeardText.ts b/src/plugins/AIComponents/hooks/useHeardText.ts
new file mode 100644
index 000000000..29521f6e0
--- /dev/null
+++ b/src/plugins/AIComponents/hooks/useHeardText.ts
@@ -0,0 +1,325 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+/** How much of a text the user has heard so far. */
+export type HeardTextProgress = {
+ /**
+ * Number of characters of the text the user has heard, counted in Unicode code points
+ * (as `Array.from(text).length` counts them, not UTF-16 code units like `text.length`).
+ * May be fractional; it is floored before rendering. A value at or above the text's
+ * code point length reveals the whole text.
+ */
+ chars: number;
+ /** `true` once the speech for the text has finished (played out, interrupted or skipped). */
+ done: boolean;
+};
+
+/**
+ * Returns how much of the text the user has heard by `now` (a `performance.now()`
+ * timestamp).
+ *
+ * Return `undefined` only for a text that is not spoken at all: the full text is then
+ * shown, and the reveal never moves back past what has been shown. A text that will be
+ * spoken but whose audio has not started, or that the voice source does not know yet,
+ * must return `{ chars: 0, done: false }`, or it is shown in full before it is heard.
+ *
+ * Also called during render, so keep it free of side effects.
+ */
+export type HeardTextFunction = (now: number) => HeardTextProgress | undefined;
+
+/**
+ * Registers a listener that is called whenever the speech progress source changes
+ * (playback started, paused, finished...). Returns an unsubscribe function.
+ */
+export type HeardTextSubscribe = (listener: () => void) => () => void;
+
+export type UseHeardTextParams = {
+ /** The speech progress of `text`, see `HeardTextFunction`. */
+ heard: HeardTextFunction;
+ /** The full text to reveal. */
+ text: string;
+ /**
+ * `true` while the text is still being written. The full text is only revealed
+ * automatically once `heard` reports `done` and `generating` is falsy.
+ */
+ generating?: boolean;
+ /**
+ * Defaults to `true`. `false` reveals the full text at once, for example for a message
+ * loaded from history. Read when the hook mounts and when `text` is replaced by a
+ * different text, not on every change.
+ */
+ live?: boolean;
+ /**
+ * Optional change notifications from the speech progress source. Without it, `heard`
+ * is only checked again while the text is paced and its speech is not done, and when
+ * `text` or `generating` change. Subscribed once; to switch to another source, remount
+ * the component (for example with a `key`).
+ */
+ subscribe?: HeardTextSubscribe;
+};
+
+export type UseHeardTextResult = {
+ /** The part of the text the user has heard (the full text when not paced). */
+ heardText: string;
+ /** `false` when `heard` returned `undefined`; `heardText` is then the full text. */
+ isPaced: boolean;
+ /** Reveals the full text from now on (e.g. when the user stops the AI response). */
+ skip: () => void;
+};
+
+type PacingState = {
+ /** Number of code points revealed. Never decreases while the text continues. */
+ count: number;
+ isPaced: boolean;
+ /** No more ticking is needed until the inputs change or the source notifies. */
+ settled: boolean;
+};
+
+/** The reveal of one text. Starts over when the text is replaced by a different text. */
+type RevealState = PacingState & {
+ /** `live` as read when the reveal started. */
+ live: boolean;
+ skipped: boolean;
+ /** The text `count` refers to. */
+ text: string;
+};
+
+const now = () =>
+ typeof performance !== 'undefined' && typeof performance.now === 'function'
+ ? performance.now()
+ : Date.now();
+
+const scheduleFrame = (callback: () => void): (() => void) => {
+ if (typeof requestAnimationFrame === 'function') {
+ const id = requestAnimationFrame(callback);
+ return () => cancelAnimationFrame(id);
+ }
+ const id = setTimeout(callback, 16);
+ return () => clearTimeout(id);
+};
+
+/**
+ * UTF-16 offsets of the code point boundaries of `text`: `offsets[n]` is the
+ * `String.prototype.slice` end index of the first `n` code points, so
+ * `offsets.length - 1` is the code point length. Slicing at these offsets never splits
+ * a surrogate pair.
+ */
+const getCodePointOffsets = (text: string): number[] => {
+ const offsets = [0];
+ let offset = 0;
+ for (const codePoint of text) {
+ offset += codePoint.length;
+ offsets.push(offset);
+ }
+ return offsets;
+};
+
+const sample = ({
+ generating,
+ heard,
+ live,
+ previousCount,
+ skipped,
+ textLength,
+}: {
+ generating: boolean;
+ heard: HeardTextFunction;
+ live: boolean;
+ previousCount: number;
+ skipped: boolean;
+ /** Length of the text in code points. */
+ textLength: number;
+}): PacingState => {
+ const progress = heard(now());
+
+ if (!progress) {
+ // Not paced: the full text is shown, so pacing that starts later must not move the
+ // reveal backwards past what has already been shown.
+ return { count: Math.max(previousCount, textLength), isPaced: false, settled: true };
+ }
+
+ if (!live || skipped || (progress.done && !generating)) {
+ return { count: Math.max(previousCount, textLength), isPaced: true, settled: true };
+ }
+
+ // `+Infinity` means the whole text has been heard; NaN, negative and non-numeric
+ // counts mean nothing has
+ const flooredChars = Math.floor(progress.chars);
+ const heardChars = flooredChars > 0 ? flooredChars : 0;
+
+ return {
+ count: Math.max(previousCount, Math.min(heardChars, textLength)),
+ isPaced: true,
+ settled: progress.done,
+ };
+};
+
+const startReveal = ({
+ generating,
+ heard,
+ live,
+ text,
+ textLength,
+}: {
+ generating: boolean;
+ heard: HeardTextFunction;
+ live: boolean;
+ text: string;
+ textLength: number;
+}): RevealState => ({
+ ...sample({ generating, heard, live, previousCount: 0, skipped: false, textLength }),
+ live,
+ skipped: false,
+ text,
+});
+
+/**
+ * Whether `text` continues the reveal of `previous.text`: it extends or truncates the part
+ * already shown. Anything else (the next reply rendered by the same component) is a new
+ * text, whose reveal must not start from the length shown for the previous one.
+ */
+const continuesReveal = (previous: RevealState, text: string) => {
+ const offsets = getCodePointOffsets(previous.text);
+ const shown = previous.text.slice(
+ 0,
+ offsets[Math.min(previous.count, offsets.length - 1)],
+ );
+ return text.startsWith(shown) || shown.startsWith(text);
+};
+
+const noop = () => undefined;
+
+/**
+ * Reveals a text at the pace the user hears it spoken, for example the reply of a voice
+ * AI agent that is written into the chat faster than the agent speaks it. The speech
+ * progress comes from the app as a plain function, so the hook works with any voice
+ * backend and has no dependency on the chat SDK.
+ *
+ * - Reveals the first `floor(chars)` Unicode code points of `text` (emoji and other
+ * astral characters are never split) and never moves backwards while `text` grows or
+ * is truncated. When `text` is replaced by a different text (the next reply rendered
+ * by the same component), the reveal starts over: the length shown, `skip()` and
+ * `live` are reset.
+ * - While the text is paced and its speech is not done, samples `heard` on every
+ * animation frame (a 16 ms timeout without `requestAnimationFrame`). It stops ticking
+ * when the speech is done or the component unmounts; `subscribe` notifications, and
+ * changes of `text` or `generating`, make it check again.
+ * - Reveals the full text once `heard` reports `done` and `generating` is falsy, when
+ * `live` is `false`, and after `skip()`.
+ * - When `heard` returns `undefined`, `isPaced` is `false` and `heardText` is the full
+ * text. If it becomes paced later, the reveal continues from the length already shown.
+ *
+ * `heard` and `subscribe` are read through refs, so passing new function identities on
+ * every render (inline arrow functions) does not restart the pacing. `subscribe` is
+ * called once on mount (and again only if it goes from absent to present) and its
+ * unsubscribe function runs on unmount; to switch to another speech source, remount the
+ * component. `heard` is also called during render, so keep it free of side effects.
+ *
+ * @example
+ * ```tsx
+ * import { useHeardText } from 'stream-chat-react/ai-components';
+ *
+ * const SpokenReply = ({ generating, session, text, turnId }) => {
+ * const { heardText } = useHeardText({
+ * generating,
+ * heard: (now) => session.speech.heard(turnId, now),
+ * subscribe: (listener) => session.speech.subscribe(listener),
+ * text,
+ * });
+ * return {heardText}
;
+ * };
+ * ```
+ */
+export const useHeardText = ({
+ generating = false,
+ heard,
+ live = true,
+ subscribe,
+ text,
+}: UseHeardTextParams): UseHeardTextResult => {
+ const codePointOffsets = useMemo(() => getCodePointOffsets(text), [text]);
+ const textLength = codePointOffsets.length - 1;
+
+ const heardRef = useRef(heard);
+ heardRef.current = heard;
+ const subscribeRef = useRef(subscribe);
+ subscribeRef.current = subscribe;
+
+ const [revealState, setRevealState] = useState(() =>
+ startReveal({ generating, heard, live, text, textLength }),
+ );
+ let reveal = revealState;
+ if (revealState.text !== text) {
+ // Updated during render so that the new text never renders with the previous reveal.
+ reveal = continuesReveal(revealState, text)
+ ? { ...revealState, count: Math.min(revealState.count, textLength), text }
+ : startReveal({ generating, heard, live, text, textLength });
+ setRevealState(reveal);
+ }
+ const revealRef = useRef(reveal);
+ revealRef.current = reveal;
+ const wakeRef = useRef<() => void>(noop);
+
+ const { live: isLive, skipped } = reveal;
+
+ useEffect(() => {
+ let cancelFrame: (() => void) | undefined;
+ let disposed = false;
+
+ const tick = () => {
+ cancelFrame = undefined;
+ if (disposed) return;
+
+ const previous = revealRef.current;
+ const next = sample({
+ generating,
+ heard: heardRef.current,
+ live: isLive,
+ previousCount: previous.count,
+ skipped,
+ textLength,
+ });
+
+ if (
+ next.count !== previous.count ||
+ next.isPaced !== previous.isPaced ||
+ next.settled !== previous.settled
+ ) {
+ const updated = { ...previous, ...next };
+ revealRef.current = updated;
+ setRevealState(updated);
+ }
+
+ if (!next.settled) cancelFrame = scheduleFrame(tick);
+ };
+
+ wakeRef.current = () => {
+ if (!disposed && !cancelFrame) tick();
+ };
+ tick();
+
+ return () => {
+ disposed = true;
+ cancelFrame?.();
+ wakeRef.current = noop;
+ };
+ }, [generating, isLive, skipped, text, textLength]);
+
+ const hasSubscribe = Boolean(subscribe);
+ useEffect(() => {
+ if (!hasSubscribe) return;
+ return subscribeRef.current?.(() => wakeRef.current());
+ }, [hasSubscribe]);
+
+ const skip = useCallback(
+ () => setRevealState((previous) => ({ ...previous, skipped: true })),
+ [],
+ );
+
+ return {
+ heardText: reveal.isPaced
+ ? text.slice(0, codePointOffsets[Math.min(reveal.count, textLength)])
+ : text,
+ isPaced: reveal.isPaced,
+ skip,
+ };
+};
diff --git a/src/plugins/AIComponents/index.ts b/src/plugins/AIComponents/index.ts
index d9719b8be..8ff685868 100644
--- a/src/plugins/AIComponents/index.ts
+++ b/src/plugins/AIComponents/index.ts
@@ -1,5 +1,6 @@
export * from './AIMarkdown';
export * from './AIMessageComposer';
export * from './AIStateIndicator/AIStateIndicator';
+export * from './hooks/useHeardText';
export * from './StreamingMessage/StreamingMessage';
export * from './VoiceVisualization';