From 44b1e693c9a15c8cca31556eb864eee2273c62e3 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 26 Oct 2026 10:24:34 +0100 Subject: [PATCH 1/2] feat(ai-components): add VoiceWave and VoiceGlow voice-agent visualizations Port of #3339 from the v14 line: VoiceWave, VoiceGlow and useAudioLevel in stream-chat-react/ai-components, with v15 namespaced i18n keys (aiComponents.voiceVisualization.*), and a voice mode in the ai-chatbot example built on v15's channel and composer APIs. --- ai-docs/i18n-v15-key-map.json | 26 +- examples/ai-chatbot/README.md | 8 +- examples/ai-chatbot/implementation.md | 17 + .../ChatContainer/ChatContainer.tsx | 20 +- .../MessageInputBar/MessageInputBar.tsx | 71 +-- .../src/components/VoiceMode/VoiceMode.scss | 116 +++++ .../src/components/VoiceMode/VoiceMode.tsx | 108 ++++ .../components/VoiceMode/VoiceModeButton.tsx | 35 ++ .../components/VoiceMode/VoiceModeContext.tsx | 34 ++ .../src/components/VoiceMode/index.ts | 3 + .../VoiceMode/useVoiceConversation.ts | 272 ++++++++++ examples/ai-chatbot/src/components/index.scss | 3 + .../src/components/useSendAIMessage.ts | 92 ++++ src/i18n/__tests__/catalog.fixture.json | 6 + src/i18n/keys.ts | 6 + src/plugins/AIComponents/README.md | 93 +++- .../VoiceVisualization/VoiceGlow.tsx | 54 ++ .../VoiceVisualization/VoiceGlowCss.tsx | 345 +++++++++++++ .../VoiceVisualizationView.tsx | 265 ++++++++++ .../VoiceVisualization/VoiceWave.tsx | 32 ++ .../AIComponents/VoiceVisualization/bounce.ts | 56 +++ .../VoiceVisualization/cssGlow.ts | 177 +++++++ .../AIComponents/VoiceVisualization/driver.ts | 221 ++++++++ .../VoiceVisualization/hitArea.ts | 22 + .../AIComponents/VoiceVisualization/index.ts | 8 + .../AIComponents/VoiceVisualization/math.ts | 63 +++ .../VoiceVisualization/palette.ts | 115 +++++ .../renderers/glowWebGLRenderer.ts | 430 ++++++++++++++++ .../VoiceVisualization/renderers/types.ts | 20 + .../renderers/waveRenderer.ts | 159 ++++++ .../VoiceVisualization/simplex.ts | 84 ++++ .../AIComponents/VoiceVisualization/types.ts | 56 +++ .../VoiceVisualization/useAudioLevel.ts | 81 +++ .../VoiceVisualization/useVoiceStateLabel.ts | 19 + .../__tests__/VoiceGlowCss.test.tsx | 415 +++++++++++++++ .../__tests__/VoiceVisualization.test.tsx | 401 +++++++++++++++ .../AIComponents/__tests__/a11y.test.tsx | 22 +- .../__tests__/classNames.test.tsx | 15 +- .../__tests__/glowWebGLRenderer.test.ts | 140 ++++++ .../AIComponents/__tests__/hydration.test.tsx | 39 +- .../AIComponents/__tests__/i18n.test.tsx | 48 +- .../AIComponents/__tests__/ssr.test.tsx | 21 +- .../__tests__/styleTokens.test.ts | 47 ++ .../__tests__/useAudioLevel.test.ts | 116 +++++ .../__tests__/voiceDriver.test.ts | 257 ++++++++++ .../__tests__/voicePalette.test.ts | 168 +++++++ .../__tests__/voiceWaveRenderer.test.ts | 131 +++++ src/plugins/AIComponents/index.ts | 1 + src/plugins/AIComponents/styling/_tokens.scss | 8 + src/plugins/AIComponents/styling/index.scss | 2 + .../AIComponents/styling/voice-glow-css.scss | 475 ++++++++++++++++++ .../styling/voice-visualization.scss | 78 +++ 52 files changed, 5422 insertions(+), 79 deletions(-) create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceMode.scss create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceMode.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx create mode 100644 examples/ai-chatbot/src/components/VoiceMode/index.ts create mode 100644 examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts create mode 100644 examples/ai-chatbot/src/components/useSendAIMessage.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/VoiceWave.tsx create mode 100644 src/plugins/AIComponents/VoiceVisualization/bounce.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/cssGlow.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/driver.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/hitArea.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/index.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/math.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/palette.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/glowWebGLRenderer.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/types.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/renderers/waveRenderer.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/simplex.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/types.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/useAudioLevel.ts create mode 100644 src/plugins/AIComponents/VoiceVisualization/useVoiceStateLabel.ts create mode 100644 src/plugins/AIComponents/__tests__/VoiceGlowCss.test.tsx create mode 100644 src/plugins/AIComponents/__tests__/VoiceVisualization.test.tsx create mode 100644 src/plugins/AIComponents/__tests__/glowWebGLRenderer.test.ts create mode 100644 src/plugins/AIComponents/__tests__/useAudioLevel.test.ts create mode 100644 src/plugins/AIComponents/__tests__/voiceDriver.test.ts create mode 100644 src/plugins/AIComponents/__tests__/voicePalette.test.ts create mode 100644 src/plugins/AIComponents/__tests__/voiceWaveRenderer.test.ts create mode 100644 src/plugins/AIComponents/styling/voice-glow-css.scss create mode 100644 src/plugins/AIComponents/styling/voice-visualization.scss diff --git a/ai-docs/i18n-v15-key-map.json b/ai-docs/i18n-v15-key-map.json index 5db06c4bb..2ac8960ee 100644 --- a/ai-docs/i18n-v15-key-map.json +++ b/ai-docs/i18n-v15-key-map.json @@ -1,6 +1,6 @@ { "$comment": "Migration table: natural-language translation key (v14) -> namespaced key (v15). Hand-reviewed; nothing regenerates it. Integrators who passed translationsForLanguage/registerTranslation dictionaries keyed on the old strings should use this to rename their keys. See ai-docs/i18n-v15-migration.md.", - "count": 621, + "count": 627, "keys": { "aria/Active": { "key": "a11y.accessibleLabel.active.ariaLabel", @@ -186,6 +186,30 @@ "key": "aiComponents.stateIndicator.workingMyMagic.text", "prose": true }, + "Connecting": { + "key": "aiComponents.voiceVisualization.connecting.text", + "prose": true + }, + "Listening": { + "key": "aiComponents.voiceVisualization.listening.text", + "prose": true + }, + "Ready": { + "key": "aiComponents.voiceVisualization.ready.text", + "prose": true + }, + "Speaking": { + "key": "aiComponents.voiceVisualization.speaking.text", + "prose": true + }, + "Thinking": { + "key": "aiComponents.voiceVisualization.thinking.text", + "prose": true + }, + "aria/Voice agent: {{ state }}": { + "key": "aiComponents.voiceVisualization.voiceAgent.ariaLabel", + "prose": true + }, "Generating...": { "key": "aiState.indicator.generating.label", "prose": true diff --git a/examples/ai-chatbot/README.md b/examples/ai-chatbot/README.md index 87328f730..e8752f31c 100644 --- a/examples/ai-chatbot/README.md +++ b/examples/ai-chatbot/README.md @@ -1,7 +1,7 @@ # AI chatbot example A Next.js chatbot built with `stream-chat-react` and its `stream-chat-react/ai-components` entry point -(streaming messages, AI state indicator, AI message composer). Styles come from +(streaming messages, AI state indicator, AI message composer, voice agent visualizations). Styles come from `stream-chat-react/css/ai-components.css`. ## Getting started @@ -33,4 +33,10 @@ expose `POST /start-ai-agent` and `POST /summarize`, as called from `src/compone - **One copy of `@stream-io/state-store`.** The example keeps `installConfig.hoistingLimits: workspaces`, so the app installs its own `stream-chat` and `@stream-io/state-store` while the linked `stream-chat-react` workspace resolves the copies at the repository root. `stream-chat`, `@stream-io/i18n` and `stream-chat-react` hand each other store instances, so exactly one copy of `@stream-io/state-store` must be loaded. The app dedupes `stream-chat` and `@stream-io/state-store` through the Turbopack `resolveAlias` in `next.config.ts` plus the matching `paths` in `tsconfig.json`. - **Sidebar pagination.** The sidebar uses infinite scroll: v15's `ChannelList` has no load-more button. +## Voice mode + +The waveform button in the composer opens voice mode: a hands-free conversation in which the AI agent is shown as a `VoiceGlow` and your microphone as a `VoiceWave`. It listens to you, sends what you said, and reads the answer aloud before listening again. + +It needs a browser with the Web Speech API's `SpeechRecognition` (Chrome, Edge or Safari; the button is hidden elsewhere) and access to the microphone. Reading answers aloud uses `speechSynthesis`; where it is missing, the answer is only shown in the chat. + See [implementation.md](./implementation.md) for implementation notes. diff --git a/examples/ai-chatbot/implementation.md b/examples/ai-chatbot/implementation.md index 0b5b6fa73..cdc77e16e 100644 --- a/examples/ai-chatbot/implementation.md +++ b/examples/ai-chatbot/implementation.md @@ -342,6 +342,23 @@ examples/ai-chatbot/ - Posts to `/summarize` endpoint with OpenAI platform - Returns summary string from API response +### 18. **useSendAIMessage.ts** + +- The send path shared by `MessageInputBar` and voice mode: creates a new conversation with `channel.ensureWatched()`, starts the AI agent when it is not watching yet, sends through `composer.send()` (with attachments), records the message against the rate limit and summarizes a new conversation +- Also holds `availableModels`, `defaultPlatformModel` and `isWatchedByAI` + +### 19. **VoiceMode** + +A hands-free conversation with the agent, built on `VoiceGlow`, `VoiceWave`, `useAudioLevel` and `useSpeechToText` from `stream-chat-react/ai-components`. + +- `VoiceModeButton` sits in the composer next to the dictation button and appears only where the browser has `SpeechRecognition` (Chrome, Edge, Safari) +- `VoiceModeProvider` (in `ChatContainer`) holds whether voice mode is open; `VoiceMode` is mounted inside `Channel` only while open, so the microphone is released when it closes +- The overlay shows a large `VoiceGlow` for the agent and a `VoiceWave` for the user's microphone, fed by `useAudioLevel` on the `getUserMedia` stream +- `useVoiceConversation` runs the turn loop: listen (dictation ends on silence), send through `useSendAIMessage`, wait for the agent's `ai_indicator` events, read the finished answer aloud with `speechSynthesis` (markdown stripped, code skipped), then listen again +- Voice agent state: `connecting` while the microphone is requested or the agent is being started, `listening` during the user's turn, `thinking` while the agent works, `speaking` while the answer is read aloud, `idle` when paused or stopped +- Pause/resume and end buttons; Escape closes it; the rate limit and a denied microphone stop it with a message +- `--str-chat__ai-voice-backdrop` is set to the demo background in `index.scss`, so the visualizations pick their light or dark look from the demo theme rather than the SDK theme + --- ## Next.js Architecture diff --git a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx index f4985985a..c4f3ce85d 100644 --- a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx +++ b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx @@ -15,6 +15,7 @@ import { MessageBubble } from '../MessageBubble'; import { MessageInputBar } from '../MessageInputBar'; import { AIStateIndicator } from '../AIStateIndicator'; import { TopNavBar } from '../TopNavBar'; +import { VoiceMode, VoiceModeProvider } from '../VoiceMode'; import './ChatContainer.scss'; interface ChatContainerProps { @@ -44,14 +45,17 @@ export const ChatContainer = ({ onToggleSidebar }: ChatContainerProps) => { MessageComposerUI: MessageInputBar, }} > - {channel && ( - - - - - - - )} + + {channel && ( + + + + + + + + )} + ); diff --git a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx index 2ffefc413..0f754e5dd 100644 --- a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx +++ b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx @@ -3,7 +3,6 @@ import { AIMessageComposer } from 'stream-chat-react/ai-components'; import { useEffect, useState } from 'react'; import { - type Channel, isImageFile, type LocalUploadAttachment, type UploadRequestFn, @@ -14,31 +13,16 @@ import { useChatContext, useMessageComposerController, } from 'stream-chat-react'; -import { startAiAgent, summarizeConversation } from '@/components/api'; -import { - checkRateLimit, - formatTimeRemaining, - recordMessage, -} from '@/components/rateLimitUtils'; +import { checkRateLimit, formatTimeRemaining } from '@/components/rateLimitUtils'; +import { availableModels, useSendAIMessage } from '@/components/useSendAIMessage'; +import { VoiceModeButton } from '@/components/VoiceMode'; import './MessageInputBar.scss'; -const isWatchedByAI = (channel: Channel) => - Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); - -const availableModels = [ - { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, - { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, - { - platform: 'gemini', - value: 'gemini-flash-latest', - label: 'Gemini Flash (latest)', - }, -] as const; - export const MessageInputBar = () => { const { client } = useChatContext(); const channel = useChannel(); const composer = useMessageComposerController(); + const sendAIMessage = useSendAIMessage(); const { attachments } = useAttachmentsForPreview(); const [selectedPlatformModel, setSelectedPlatformModel] = useState(); @@ -127,48 +111,14 @@ export const MessageInputBar = () => { const platformModel = formData.get('platform-model'); setSelectedPlatformModel(platformModel as string); - composer.textComposer.setText(message as string); - - // A new conversation exists only locally until now; `Channel` no longer creates it. - // Both setup steps run before `send()`, so a failure here leaves the draft untouched. - if (!channel.initialized) { - await channel.ensureWatched(); - } - - const [platform, model] = (platformModel as string).split('|'); - - if (!isWatchedByAI(channel)) { - await startAiAgent(channel, model, platform); - } - - // `send()` composes the message, shows it optimistically, releases the composer and - // reports a failed send itself (the message stays in the list marked `failed`). - const result = await composer.send(); - - // `nothing-to-send` keeps the composer's contents (e.g. an upload is still running). - if (result === 'nothing-to-send') return; - - target.reset(); - - if (result !== 'sent') return; - - // Record message after successful send - recordMessage(channel.id!); + const sent = await sendAIMessage({ + onComposed: () => target.reset(), + platformModel: platformModel as string, + text: message as string, + }); // Update rate limit state - const newState = checkRateLimit(channel.id!); - setRateLimitState(newState); - - if (!channel.data?.custom?.summary) { - const summary = await summarizeConversation(message as string).catch(() => { - console.warn('Failed to summarize conversation'); - return null; - }); - - if (typeof summary === 'string' && summary.length > 0) { - await channel.update({ data: { custom: { summary } } }); - } - } + if (sent) setRateLimitState(checkRateLimit(channel.id!)); }} > @@ -205,6 +155,7 @@ export const MessageInputBar = () => {
+ = { + error: 'Voice mode stopped', + listening: 'Listening', + paused: 'Paused', + speaking: 'Speaking', + starting: 'Connecting', + waiting: 'Thinking', +}; + +const VoiceConversation = () => { + const { close } = useVoiceMode(); + const { agentState, error, micLevel, pause, phase, reply, resume, transcript } = + useVoiceConversation(); + const closeRef = useRef(null); + + useEffect(() => { + closeRef.current?.focus(); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') close(); + }; + document.addEventListener('keydown', onKeyDown); + return () => document.removeEventListener('keydown', onKeyDown); + }, [close]); + + const paused = phase === 'paused'; + const caption = + phase === 'error' + ? error + : phase === 'speaking' + ? reply + : phase === 'listening' + ? transcript || 'Go ahead, I’m listening.' + : phase === 'waiting' + ? transcript + : ''; + + return ( +
+
+ +

+ {STATUS[phase]} +

+

{caption}

+
+ + {/* The glow is the agent; the wave under it is the user's microphone. */} + + +
+ + +
+
+ ); +}; + +/** + * A hands-free conversation with the agent over the chat: it listens, sends what was said, + * shows the agent thinking and reads its answer aloud. Mounted only while open, so the + * microphone is released as soon as it closes. + */ +export const VoiceMode = () => { + const { isOpen } = useVoiceMode(); + return isOpen ? : null; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx new file mode 100644 index 000000000..c28a69f93 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx @@ -0,0 +1,35 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useVoiceMode } from './VoiceModeContext'; + +const supportsVoiceMode = () => + typeof window !== 'undefined' && + !!( + (window as Window & { SpeechRecognition?: unknown }).SpeechRecognition ?? + (window as Window & { webkitSpeechRecognition?: unknown }).webkitSpeechRecognition + ) && + !!navigator.mediaDevices?.getUserMedia; + +/** Opens voice mode; shown only where the browser can listen (Chrome, Edge, Safari). */ +export const VoiceModeButton = () => { + const { isOpen, open } = useVoiceMode(); + // Support is known only in the browser, so the button appears after hydration. + const [isSupported, setIsSupported] = useState(false); + useEffect(() => setIsSupported(supportsVoiceMode()), []); + + if (!isSupported) return null; + + return ( + + ); +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx new file mode 100644 index 000000000..9018bc8a8 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx @@ -0,0 +1,34 @@ +'use client'; + +import { createContext, type ReactNode, useContext, useMemo, useState } from 'react'; + +type VoiceModeContextValue = { + close: () => void; + isOpen: boolean; + open: () => void; +}; + +const VoiceModeContext = createContext(undefined); + +export const useVoiceMode = () => { + const context = useContext(VoiceModeContext); + if (!context) { + throw new Error('useVoiceMode must be used within VoiceModeProvider'); + } + return context; +}; + +export const VoiceModeProvider = ({ children }: { children: ReactNode }) => { + const [isOpen, setIsOpen] = useState(false); + + const value = useMemo( + () => ({ + close: () => setIsOpen(false), + isOpen, + open: () => setIsOpen(true), + }), + [isOpen], + ); + + return {children}; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/index.ts b/examples/ai-chatbot/src/components/VoiceMode/index.ts new file mode 100644 index 000000000..cfe736dc0 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/index.ts @@ -0,0 +1,3 @@ +export * from './VoiceMode'; +export * from './VoiceModeButton'; +export * from './VoiceModeContext'; diff --git a/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts new file mode 100644 index 000000000..384357a5e --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts @@ -0,0 +1,272 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { AIStates, type Channel } from 'stream-chat'; +import { useAIState, useChannel, useChatContext } from 'stream-chat-react'; +import { + useAudioLevel, + useSpeechToText, + type VoiceAgentState, +} from 'stream-chat-react/ai-components'; +import { checkRateLimit } from '@/components/rateLimitUtils'; +import { isWatchedByAI, useSendAIMessage } from '@/components/useSendAIMessage'; + +/** + * Where the conversation is: + * - `starting`: asking for the microphone; + * - `listening`: dictating the user's turn; + * - `waiting`: the turn was sent and the agent is working on its answer; + * - `speaking`: reading the answer aloud; + * - `paused`: the user muted voice mode; + * - `error`: it can't go on (no microphone, rate limit, ...). + */ +export type VoicePhase = + | 'starting' + | 'listening' + | 'waiting' + | 'speaking' + | 'paused' + | 'error'; + +const WORKING_STATES: string[] = [ + AIStates.Thinking, + AIStates.ExternalSources, + AIStates.Generating, +]; + +/** Long answers are cut, as browsers stop reading aloud after a while. */ +const MAX_SPOKEN_LENGTH = 1200; +/** How long the agent may take to start on an answer before the turn is given up. */ +const ANSWER_TIMEOUT = 20_000; + +const canSpeak = () => typeof window !== 'undefined' && 'speechSynthesis' in window; + +/** Markdown as plain prose for reading aloud: code is skipped, formatting dropped. */ +export const toSpeech = (markdown: string) => + markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^\s{0,3}(#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') + .replace(/[*_~|]/g, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_SPOKEN_LENGTH); + +const lastAgentMessage = (channel: Channel, userId?: string) => { + const messages = channel.messagePaginator.state.getLatestValue().items ?? []; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].user?.id !== userId) return messages[i]; + } + return undefined; +}; + +/** + * A hands-free conversation with the AI agent on the current channel: listen to the user, + * send what they said, wait for the agent's answer, read it aloud, then listen again. + */ +export const useVoiceConversation = () => { + const { client } = useChatContext(); + const channel = useChannel(); + const { aiState } = useAIState(channel); + const sendAIMessage = useSendAIMessage(); + + const [phase, setPhase] = useState('starting'); + const [mic, setMic] = useState(null); + const [transcript, setTranscript] = useState(''); + const [reply, setReply] = useState(''); + const [error, setError] = useState(null); + const [agentStarting, setAgentStarting] = useState(false); + + const phaseRef = useRef(phase); + useEffect(() => { + phaseRef.current = phase; + }, [phase]); + const transcriptRef = useRef(''); + const sawAgentWorkRef = useRef(false); + // The agent's last answer when a turn was sent, to tell a new answer from it. + const previousAnswerIdRef = useRef(undefined); + const wasListeningRef = useRef(false); + + const micLevel = useAudioLevel(mic); + + const fail = useCallback((message: string) => { + setError(message); + setPhase('error'); + }, []); + + const { isListening, isSupported, startListening, stopListening } = useSpeechToText({ + onError: (message) => { + // Silence and aborts end a turn on their own; anything else stops voice mode. + if (/no speech|aborted/i.test(message)) return; + fail(message); + }, + onTranscript: (text) => { + transcriptRef.current = text; + setTranscript(text); + }, + }); + + // The microphone, for the level of the user's voice; released when voice mode closes. + useEffect(() => { + let stream: MediaStream | undefined; + let cancelled = false; + if (!navigator.mediaDevices?.getUserMedia) { + fail('This browser cannot use a microphone.'); + return; + } + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((granted) => { + if (cancelled) { + granted.getTracks().forEach((track) => track.stop()); + return; + } + stream = granted; + setMic(granted); + setPhase((current) => (current === 'starting' ? 'listening' : current)); + }) + .catch(() => fail('Voice mode needs access to your microphone.')); + return () => { + cancelled = true; + stream?.getTracks().forEach((track) => track.stop()); + }; + }, [fail]); + + // Start dictating whenever it is the user's turn and recognition is not running. + useEffect(() => { + if (phase !== 'listening' || isListening || !isSupported) return; + // Recognition can refuse to restart in the same tick it ended. + const timeout = setTimeout(() => { + try { + transcriptRef.current = ''; + setTranscript(''); + startListening(); + } catch { + // Already started; the next end restarts it. + } + }, 150); + return () => clearTimeout(timeout); + }, [isListening, isSupported, phase, startListening]); + + const submit = useCallback( + (text: string) => { + if (checkRateLimit(channel.id!).isLimited) { + fail('Limit reached, 10 messages per conversation.'); + return; + } + sawAgentWorkRef.current = false; + previousAnswerIdRef.current = lastAgentMessage(channel, client.userID)?.id; + setReply(''); + setAgentStarting(!isWatchedByAI(channel)); + setPhase('waiting'); + sendAIMessage({ text }).catch(() => fail('Your message could not be sent.')); + }, + [channel, client.userID, fail, sendAIMessage], + ); + + // A turn ends when recognition stops: send what was said, or listen again after silence. + useEffect(() => { + if (isListening) { + wasListeningRef.current = true; + return; + } + if (!wasListeningRef.current) return; + wasListeningRef.current = false; + if (phaseRef.current !== 'listening') return; + const text = transcriptRef.current.trim(); + if (text) submit(text); + }, [isListening, submit]); + + const speak = useCallback((markdown: string) => { + const text = toSpeech(markdown); + if (!text || !canSpeak()) { + setPhase('listening'); + return; + } + setReply(text); + setPhase('speaking'); + const utterance = new SpeechSynthesisUtterance(text); + const done = () => { + if (phaseRef.current === 'speaking') setPhase('listening'); + }; + utterance.onend = done; + utterance.onerror = done; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + }, []); + + // The agent's indicator: working while it thinks and writes, idle again once the answer is in. + useEffect(() => { + if (phaseRef.current !== 'waiting') return; + if (WORKING_STATES.includes(aiState)) { + sawAgentWorkRef.current = true; + setAgentStarting(false); + return; + } + if (!sawAgentWorkRef.current) return; + speak(lastAgentMessage(channel, client.userID)?.text ?? ''); + }, [aiState, channel, client.userID, speak]); + + // A turn the agent is never seen working on (no indicator events, or missed ones) still ends: + // with its answer if one came in, otherwise with an error, rather than waiting forever. + useEffect(() => { + if (phase !== 'waiting') return; + const timeout = setTimeout(() => { + if (sawAgentWorkRef.current) return; + const answer = lastAgentMessage(channel, client.userID); + if (answer && answer.id !== previousAnswerIdRef.current) speak(answer.text ?? ''); + else fail('The agent did not answer.'); + }, ANSWER_TIMEOUT); + return () => clearTimeout(timeout); + }, [channel, client.userID, fail, phase, speak]); + + const pause = useCallback(() => { + setPhase('paused'); + stopListening(); + if (canSpeak()) window.speechSynthesis.cancel(); + }, [stopListening]); + + const resume = useCallback(() => setPhase('listening'), []); + + // Nothing keeps talking or listening once voice mode closes. + useEffect( + () => () => { + if (canSpeak()) window.speechSynthesis.cancel(); + }, + [], + ); + useEffect(() => { + if (phase === 'paused' || phase === 'error') stopListening(); + }, [phase, stopListening]); + + const agentState: VoiceAgentState = (() => { + switch (phase) { + case 'starting': + return 'connecting'; + case 'listening': + return 'listening'; + case 'waiting': + if (agentStarting) return 'connecting'; + // Without read-aloud, the answer being written is the agent speaking. + return aiState === AIStates.Generating && !canSpeak() ? 'speaking' : 'thinking'; + case 'speaking': + return 'speaking'; + default: + return 'idle'; + } + })(); + + return { + agentState, + error, + isSupported, + micLevel, + pause, + phase, + reply, + resume, + transcript, + }; +}; diff --git a/examples/ai-chatbot/src/components/index.scss b/examples/ai-chatbot/src/components/index.scss index 73a79dadc..111182a0a 100644 --- a/examples/ai-chatbot/src/components/index.scss +++ b/examples/ai-chatbot/src/components/index.scss @@ -91,6 +91,9 @@ --str-chat__ai-syntax-error: #f87171; --str-chat__ai-syntax-whitespace: hsla(0, 0%, 100%, 0.1); + /* VoiceWave and VoiceGlow sit on the demo's background, not the SDK theme's */ + --str-chat__ai-voice-backdrop: var(--ai-demo-bg-primary); + color-scheme: dark; color: var(--ai-demo-text-primary); background-color: var(--ai-demo-bg-primary); diff --git a/examples/ai-chatbot/src/components/useSendAIMessage.ts b/examples/ai-chatbot/src/components/useSendAIMessage.ts new file mode 100644 index 000000000..f45aad17a --- /dev/null +++ b/examples/ai-chatbot/src/components/useSendAIMessage.ts @@ -0,0 +1,92 @@ +'use client'; + +import { useCallback } from 'react'; +import type { Channel } from 'stream-chat'; +import { useChannel, useMessageComposerController } from 'stream-chat-react'; +import { startAiAgent, summarizeConversation } from '@/components/api'; +import { recordMessage } from '@/components/rateLimitUtils'; + +export const availableModels = [ + { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, + { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, + { + platform: 'gemini', + value: 'gemini-flash-latest', + label: 'Gemini Flash (latest)', + }, +] as const; + +export const defaultPlatformModel = `${availableModels[0].platform}|${availableModels[0].value}`; + +export const isWatchedByAI = (channel: Channel) => + Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); + +type SendAIMessageOptions = { + text: string; + /** `platform|model`, as the model select submits it. */ + platformModel?: string; + /** Called once the composer has taken the message (it was sent or failed to send). */ + onComposed?: () => void; +}; + +/** + * Sends a message to the AI agent: creates a new conversation on the server, starts the agent on + * the channel if it isn't watching yet, sends the message (with any attachments), records it + * against the rate limit and gives a new conversation its summary. Shared by the composer and + * voice mode. + * + * Resolves to whether a message was sent. + */ +export const useSendAIMessage = () => { + const channel = useChannel(); + const composer = useMessageComposerController(); + + return useCallback( + async ({ + text, + platformModel = defaultPlatformModel, + onComposed, + }: SendAIMessageOptions) => { + composer.textComposer.setText(text); + + // A new conversation exists only locally until now; `Channel` no longer creates it. + // Both setup steps run before `send()`, so a failure here leaves the draft untouched. + if (!channel.initialized) { + await channel.ensureWatched(); + } + + const [platform, model] = platformModel.split('|'); + + if (!isWatchedByAI(channel)) { + await startAiAgent(channel, model, platform); + } + + // `send()` composes the message, shows it optimistically, releases the composer and + // reports a failed send itself (the message stays in the list marked `failed`). + const result = await composer.send(); + + // `nothing-to-send` keeps the composer's contents (e.g. an upload is still running). + if (result === 'nothing-to-send') return false; + + onComposed?.(); + + if (result !== 'sent') return false; + + recordMessage(channel.id!); + + if (!channel.data?.custom?.summary) { + const summary = await summarizeConversation(text).catch(() => { + console.warn('Failed to summarize conversation'); + return null; + }); + + if (typeof summary === 'string' && summary.length > 0) { + await channel.update({ data: { custom: { summary } } }); + } + } + + return true; + }, + [channel, composer], + ); +}; diff --git a/src/i18n/__tests__/catalog.fixture.json b/src/i18n/__tests__/catalog.fixture.json index 0cf75f406..0f6be7550 100644 --- a/src/i18n/__tests__/catalog.fixture.json +++ b/src/i18n/__tests__/catalog.fixture.json @@ -48,6 +48,12 @@ "aiComponents.stateIndicator.summoningMyInnerGenius.text": "Summoning my inner genius", "aiComponents.stateIndicator.thinkingReallyHard.text": "Thinking really hard", "aiComponents.stateIndicator.workingMyMagic.text": "Working my magic", + "aiComponents.voiceVisualization.connecting.text": "Connecting", + "aiComponents.voiceVisualization.listening.text": "Listening", + "aiComponents.voiceVisualization.ready.text": "Ready", + "aiComponents.voiceVisualization.speaking.text": "Speaking", + "aiComponents.voiceVisualization.thinking.text": "Thinking", + "aiComponents.voiceVisualization.voiceAgent.ariaLabel": "Voice agent: {{ state }}", "aiState.indicator.generating.label": "Generating...", "aiState.indicator.thinking.label": "Thinking...", "attachment.actions.giphyActions.ariaLabel": "Giphy actions", diff --git a/src/i18n/keys.ts b/src/i18n/keys.ts index e174bc5a2..362e51269 100644 --- a/src/i18n/keys.ts +++ b/src/i18n/keys.ts @@ -59,6 +59,12 @@ export type TranslationCatalog = { 'aiComponents.stateIndicator.summoningMyInnerGenius.text': 'Summoning my inner genius'; 'aiComponents.stateIndicator.thinkingReallyHard.text': 'Thinking really hard'; 'aiComponents.stateIndicator.workingMyMagic.text': 'Working my magic'; + 'aiComponents.voiceVisualization.connecting.text': 'Connecting'; + 'aiComponents.voiceVisualization.listening.text': 'Listening'; + 'aiComponents.voiceVisualization.ready.text': 'Ready'; + 'aiComponents.voiceVisualization.speaking.text': 'Speaking'; + 'aiComponents.voiceVisualization.thinking.text': 'Thinking'; + 'aiComponents.voiceVisualization.voiceAgent.ariaLabel': 'Voice agent: {{ state }}'; 'aiState.indicator.generating.label': 'Generating...'; 'aiState.indicator.thinking.label': 'Thinking...'; 'attachment.actions.giphyActions.ariaLabel': 'Giphy actions'; diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index 272f64307..5aa0d47fc 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -31,9 +31,9 @@ Load `ai-components.css` in addition to `stream-chat-react/css/index.css`. From `stream-chat-react/ai-components`: -- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator` -- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText` -- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions` +- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator`, `VoiceWave`, `VoiceGlow` +- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useAudioLevel` +- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `VoiceAgentState`, `VoiceGlowRendering`, `VoiceVisualizationProps`, `VoiceWaveProps`, `VoiceGlowProps` `useMessageTextStreaming` is not exported from here; import it from the main entry point (`stream-chat-react`). @@ -185,6 +185,12 @@ Every key the components use: | `aiComponents.stateIndicator.summoningMyInnerGenius.text` | Summoning my inner genius | `Summoning my inner genius` | | `aiComponents.stateIndicator.thinkingReallyHard.text` | Thinking really hard | `Thinking really hard` | | `aiComponents.stateIndicator.workingMyMagic.text` | Working my magic | `Working my magic` | +| `aiComponents.voiceVisualization.connecting.text` | Connecting | `Connecting` | +| `aiComponents.voiceVisualization.listening.text` | Listening | `Listening` | +| `aiComponents.voiceVisualization.ready.text` | Ready | `Ready` | +| `aiComponents.voiceVisualization.speaking.text` | Speaking | `Speaking` | +| `aiComponents.voiceVisualization.thinking.text` | Thinking | `Thinking` | +| `aiComponents.voiceVisualization.voiceAgent.ariaLabel` | Voice agent: {{ state }} | `aria/Voice agent: {{ state }}` | | `messageComposer.sendButton.send.ariaLabel` (core) | Send | `aria/Send` | | `fileUpload.uploadButton.fileUpload.ariaLabel` (core) | File upload | `aria/File upload` | | `common.retryUpload.ariaLabel` (core) | Retry upload | `aria/Retry upload` | @@ -193,7 +199,7 @@ The last three are core `stream-chat-react` keys that the components reuse, so a ## Migrating from `stream-chat-react` v14 -If you already use `stream-chat-react/ai-components` on v14 (shipped from 14.13.0), the component API is identical on v15: same exports, props, class names and theming contract. Only the i18n keys changed. v14 used the English sentence (or an `aria/…` string) as the key; v15 uses namespaced keys. If you registered translations for these components, rename them with the table above or with `ai-docs/i18n-v15-key-map.json`. If you never translated them, there is nothing to do. +If you already use `stream-chat-react/ai-components` on v14 (shipped from 14.13.0; `VoiceWave`, `VoiceGlow` and `useAudioLevel` from 14.14.0), the component API is identical on v15: same exports, props, class names and theming contract. Only the i18n keys changed. v14 used the English sentence (or an `aria/…` string) as the key; v15 uses namespaced keys. If you registered translations for these components, rename them with the table above or with `ai-docs/i18n-v15-key-map.json`. If you never translated them, there is nothing to do. v15 bundles English only. If your app runs in another language and relied on the translations 14.13 shipped for these components, those strings now fall back to English on v15 until you register translations yourself with `i18n.registerTranslation(lang, dict)`. The v14 values live in the 14.x `src/i18n/*.json` files of the 14.13.0 release (for example `git show v14.13.0:src/i18n/de.json`); map the keys with the table above or `ai-docs/i18n-v15-key-map.json`. @@ -226,6 +232,8 @@ Precedence for every token except the code palette is: your `--str-chat__ai-*` o | `table-text` | table cell text | `text-secondary` | `#495057` | | `scrollbar-track` | attachment list scrollbar track | `background-core-surface-subtle` | `#f1f1f1` | | `scrollbar-thumb` | attachment list scrollbar thumb | `border-core-default` | `#ddd` | +| `voice-color` | `VoiceWave` / `VoiceGlow` color when `colors` is unset | `accent-primary` | `#005fff` | +| `voice-backdrop` | what `VoiceWave` / `VoiceGlow` sit on (light or dark) | `background-core-elevation-0` | `#ffffff` | | `syntax-*` | code palette (see below) | none | built-in palette | The code palette tokens are `--str-chat__ai-syntax-text`, `-selection-bg`, `-comment`, `-punctuation`, `-number`, `-keyword`, `-tag`, `-string`, `-function`, `-cyan`, `-error` and `-whitespace`. @@ -334,6 +342,73 @@ function MessageContent({ content }) { } ``` +### `VoiceWave` and `VoiceGlow` + +Two animated visualizations of a voice agent, the web counterparts of `AIVoiceWave` and `AIVoiceGlow` in the Android and iOS AI components, with the same motion. `VoiceWave` is a single string carrying a wave packet that erupts with speech; `VoiceGlow` is a soft globe whose color masses orbit inside it and swell with speech. + +Both draw on a `` inside a `div` with `role="img"` and an accessible name such as "Voice agent: Speaking". They have no size of their own and draw in the middle of the box they are given (`VoiceWave` about 80% as wide as the shorter side, `VoiceGlow` about half), so size them with CSS. The defaults are `width: 100%; height: 8rem` for the wave and `12rem` square for the glow. The wave's line is as thick as on Android and iOS, 0.6% of the shorter side and never under 1.5 device pixels, and drawn at the screen's own density up to 3x; the soft glow is drawn at 1x, which looks the same at a fraction of the cost. + +#### Props + +| Name | Type | Required | Description | +| -------------- | ------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `state` | `'idle' \| 'connecting' \| 'listening' \| 'thinking' \| 'speaking'` | yes | What the agent is doing. The states differ only in motion, never in color, and a change eases from one motion into the next. | +| `colors` | `string[]` | no | One to five CSS colors, flowing through the shape. Past five are ignored. When omitted or empty, `--str-chat__ai-voice-color` is used, which follows the SDK accent color. | +| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Defaults to 1. | +| `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | +| `rendering` | `'auto' \| 'css'` | no | `VoiceGlow` only. `auto` (default) draws with a WebGL shader and falls back to `css` when WebGL is unavailable or its context is lost; `css` builds the glow from CSS gradients animated by the browser's compositor (see below). | + +Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Only the shape takes the pointer, as on iOS: the globe (within 0.32 of the shorter side from the centre) or the string. Elsewhere in the box, clicks and hovers reach whatever is behind it. On the shape, pointer events are watched and never stopped: a tap bounces it and a hovering mouse or pen ripples it, while your own `onClick` on the root or a parent still fires. + +Behavior: + +- The animation runs only while the element is on screen and the tab is visible. +- Every WebGL glow on a page draws with one shared WebGL context, so the shader compiles once and a page with many glows stays under the browser's limit on live contexts. +- It fades in over 0.3 s once its first frame is drawn (instantly with reduced motion), so it never flashes empty while a shader compiles. +- With `prefers-reduced-motion: reduce`, the motion slows to a crawl and taps and hover are ignored. +- The palette and the light or dark look follow the theme (`--str-chat__ai-voice-color` and `--str-chat__ai-voice-backdrop`), re-read about once a second, so a theme switch needs no prop. +- `VoiceWave` moves as on Android and iOS, smoothed for speech: its height rises quickly and settles slowly, its syllables swing it a little less, and as it grows its crests spread out rather than steepen into spikes. While idle, connecting or thinking it is the native wave exactly. + +`rendering='css'`, also the fallback when WebGL is unavailable, composes the glow the way the shader does, as gradients that are painted once and animated only through `transform` and `opacity`, so the browser's compositor runs the motion. It keeps moving while the main thread is busy (for example while a long answer streams in), needs no canvas or WebGL, and runs no script per frame unless `audioLevel` is given. It lacks the shader's fine marbling and its second, fainter riding mass: a tap bounces the glow and sends out a ring instead of a ripple, and a hovering pointer does nothing. Off screen, its animations pause. It needs `@property`, container query units and `color-mix()` (Chrome 111, Safari 16.4, Firefox 128 and later). + +- Under SSR the markup is rendered and drawing starts after hydration. + +#### Example + +```tsx +import { useEffect, useState } from 'react'; +import { + useAudioLevel, + type VoiceAgentState, + VoiceGlow, + VoiceWave, +} from 'stream-chat-react/ai-components'; + +const VoiceAgent = ({ state }: { state: VoiceAgentState }) => { + const [mic, setMic] = useState(null); + const micLevel = useAudioLevel(mic); + + useEffect(() => { + let stream: MediaStream | undefined; + navigator.mediaDevices.getUserMedia({ audio: true }).then((granted) => { + stream = granted; + setMic(granted); + }); + return () => stream?.getTracks().forEach((track) => track.stop()); + }, []); + + return ( + <> + + + + ); +}; +``` + ### `StreamingMessage` The `StreamingMessage` is a component that displays text with a typewriter animation effect, similar to ChatGPT. It's ideal for streaming AI responses. It's a simplified wrapper with typewriter animation effect around the `AIMarkdown` component. @@ -543,6 +618,16 @@ function VoiceInput() { } ``` +### `useAudioLevel` + +Reads the live level of a `MediaStream`'s audio (a microphone from `getUserMedia({ audio: true })`, or a remote track) for the `audioLevel` prop of `VoiceWave` and `VoiceGlow`. + +```ts +const useAudioLevel: (stream: MediaStream | null | undefined) => () => number | null; +``` + +It returns a stable getter that measures the level when called, from 0 to 1, so nothing re-renders as the level changes. It measures over the last 2048 samples (about 43 ms at 48 kHz), a few pitch periods of a voice, so the level holds steady through a vowel. The getter returns `null` while there is no stream or no Web Audio support. The audio graph is torn down when the stream changes and on unmount; the stream's tracks are left running, since they belong to you. + ## Examples ### Basic Chat Interface diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx new file mode 100644 index 000000000..2aa60a436 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx @@ -0,0 +1,54 @@ +import { useCallback, useState } from 'react'; +import clsx from 'clsx'; + +import { glowContains } from './hitArea'; +import { createGlowWebGLRenderer } from './renderers/glowWebGLRenderer'; +import type { VoiceGlowRendering, VoiceVisualizationProps } from './types'; +import { VoiceGlowCss } from './VoiceGlowCss'; +import { VoiceVisualizationView } from './VoiceVisualizationView'; + +export type VoiceGlowProps = VoiceVisualizationProps & { + /** How the glow is drawn. See `VoiceGlowRendering`. Defaults to `auto`. */ + rendering?: VoiceGlowRendering; +}; + +/** + * A voice agent as a soft, airy globe: color masses orbit inside it and swell with speech, and + * the gaps between them let the background show through. Tapping the globe bounces it; drawn + * with WebGL, it also ripples from the touch and under a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the box it is given, about half as wide + * as its shorter side, so give it a square. + */ +export const VoiceGlow = ({ + className, + rendering = 'auto', + ...props +}: VoiceGlowProps) => { + // Set when WebGL cannot start or its context is lost; the CSS glow takes over from then on. + const [webGLFailed, setWebGLFailed] = useState(false); + + const createRenderer = useCallback((canvas: HTMLCanvasElement) => { + const renderer = createGlowWebGLRenderer(canvas, () => setWebGLFailed(true)); + if (!renderer) setWebGLFailed(true); + return renderer; + }, []); + + if (rendering === 'css' || webGLFailed) { + return ; + } + + return ( + + ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx new file mode 100644 index 000000000..2f3299b8c --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx @@ -0,0 +1,345 @@ +import { + type CSSProperties, + type PointerEvent, + type ReactNode, + useEffect, + useRef, + useState, +} from 'react'; +import clsx from 'clsx'; + +import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; +import { + CSS_GLOW_RATES, + cssGlowPalette, + cssGlowRateKind, + massBackground, + MASSES, + orbitDistance, + orbitMask, + ORBITS, + RIDING_BODY, +} from './cssGlow'; +import { ease, MIC, REDUCED, scales } from './driver'; +import { glowContains, toLocalPoint } from './hitArea'; +import { audioLevelInput, expressivityInput } from './math'; +import { FALLBACK_COLOR, readVoiceTheme, toHsvPalette } from './palette'; +import type { VoiceVisualizationProps } from './types'; +import { useVoiceStateLabel } from './useVoiceStateLabel'; + +/** How often the theme is read again, so a switch to dark mode is followed. */ +const THEME_INTERVAL = 1000; +const RIPPLE_DURATION = 900; + +const block = 'str-chat__ai-voice-glow'; + +const Layer = ({ + children, + className, + style, +}: { + children?: ReactNode; + className: string; + style?: CSSProperties; +}) => ( +
+ {children} +
+); + +const orbitStyle = (orbit: number) => + ({ + '--str-ai-glow-direction': ORBITS[orbit].direction, + '--str-ai-glow-period': `${ORBITS[orbit].period}s`, + '--str-ai-glow-start': `${ORBITS[orbit].start}deg`, + }) as CSSProperties; + +const swayStyle = (orbit: number) => + ({ '--str-ai-glow-sway-period': `${ORBITS[orbit].sway}s` }) as CSSProperties; + +/** The masses the palette fills: the fourth and fifth only with that many colors. */ +const massesFor = (colorCount: number) => + MASSES.filter((_, index) => index < 3 || index < colorCount); + +/** The opaque color field: a base and the masses mixed over it in order. */ +const ColorField = ({ colorCount }: { colorCount: number }) => ( + + + + {massesFor(colorCount).map((mass, index) => ( + + +
+ + + ))} + + +); + +/** + * The color field seen through opacity that rides the first orbit: the mask turns with the + * orbit, and the inverse turns inside it keep the colors where they are. + */ +const RidingBody = ({ colorCount }: { colorCount: number }) => { + const { alpha, orbit, reach } = RIDING_BODY; + return ( + + + +
+ + + + + + + +
+ + + + ); +}; + +type Ripple = { id: number; x: number; y: number }; + +/** + * VoiceGlow drawn with CSS: the shader's composition, with every gradient painted once and only + * `transform` and `opacity` animated, so the browser's compositor runs the motion. It keeps + * moving while the page is busy, needs no canvas or WebGL, and costs no JavaScript per frame + * unless `audioLevel` is given. Off screen, its animations pause and its script rests. + * + * Script sets what the stylesheet cannot know: the palette (read from the theme when `colors` + * is not given), the playback rate of each animation for the state, and the live audio level. + */ +export const VoiceGlowCss = ({ + audioLevel, + className, + colors, + expressivity, + onPointerUp, + state, + style, + ...restProps +}: VoiceVisualizationProps) => { + const label = useVoiceStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const bounceRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const rippleTimers = useRef(new Set>()); + const [ripples, setRipples] = useState([]); + const [onScreen, setOnScreen] = useState(true); + const { amplitude, speed } = scales(expressivityInput(expressivity)); + const colorsKey = (colors ?? []).join('|'); + const colorCount = Math.min(colors?.length || 1, 5); + const hasAudioLevel = !!audioLevel; + + useEffect(() => { + audioLevelRef.current = audioLevel; + }); + + // Off screen, the animations pause (see the stylesheet) and the script below rests. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof IntersectionObserver !== 'function') return; + const observer = new IntersectionObserver((entries) => + setOnScreen(entries.some((entry) => entry.isIntersecting)), + ); + observer.observe(root); + return () => observer.disconnect(); + }, []); + + // The palette, from `colors` or the theme, and whether the glow sits on a dark backdrop. Read + // again about once a second while on screen, so a theme switch is followed without a prop. + useEffect(() => { + const root = rootRef.current; + if (!onScreen || !root || typeof getComputedStyle !== 'function') return; + const own = toHsvPalette(colorsKey ? colorsKey.split('|') : []); + let applied = ''; + const apply = () => { + const theme = readVoiceTheme(root); + const palette = own.length + ? own + : theme.palette.length + ? theme.palette + : toHsvPalette([FALLBACK_COLOR]); + const properties = cssGlowPalette(palette, theme.dark); + const next = JSON.stringify(properties) + theme.dark; + if (next === applied) return; + applied = next; + for (const [name, value] of Object.entries(properties)) { + root.style.setProperty(name, value); + } + root.setAttribute('data-dark', String(!!theme.dark)); + // Fades in once the palette is in, so the theme color it starts from never shows. + root.setAttribute('data-ready', 'true'); + }; + apply(); + const interval = setInterval(apply, THEME_INTERVAL); + const scheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + scheme?.addEventListener?.('change', apply); + return () => { + clearInterval(interval); + scheme?.removeEventListener?.('change', apply); + }; + }, [colorsKey, onScreen]); + + // Each state plays the animations at its own rates. Changing a playback rate keeps an + // animation where it is, where changing its duration would make it jump; a turn and its + // inverse always get the same rate, so they stay in step. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof root.getAnimations !== 'function') return; + const rates = CSS_GLOW_RATES[state]; + const slow = reduceMotion ? REDUCED : 1; + for (const animation of root.getAnimations({ subtree: true })) { + const kind = cssGlowRateKind((animation as CSSAnimation).animationName ?? ''); + if (kind) animation.updatePlaybackRate(rates[kind] * speed * slow); + } + // Masses come and go with the palette, so their animations need rates too. + }, [colorsKey, reduceMotion, speed, state]); + + // A live audio level, eased like the driver eases it, is the only value written per frame, + // and only while the glow is on screen. + useEffect(() => { + const root = rootRef.current; + if ( + !hasAudioLevel || + !onScreen || + !root || + typeof requestAnimationFrame !== 'function' + ) + return; + let frame = 0; + let last = 0; + let level = 0; + let written = -1; + let live: boolean | null = null; + const tick = (now: number) => { + const target = audioLevelInput(audioLevelRef.current?.()); + const dt = last ? Math.min((now - last) / 1000, 0.05) : 0; + last = now; + level += ((target ?? 0) - level) * ease(MIC, dt); + if (Math.abs(level - written) > 0.002) { + written = level; + root.style.setProperty('--str-ai-glow-level', level.toFixed(3)); + } + if (live !== (target !== null)) { + live = target !== null; + root.setAttribute('data-live', String(live)); + } + frame = requestAnimationFrame(tick); + }; + frame = requestAnimationFrame(tick); + return () => { + cancelAnimationFrame(frame); + root.style.removeProperty('--str-ai-glow-level'); + root.removeAttribute('data-live'); + }; + }, [hasAudioLevel, onScreen]); + + useEffect(() => { + const timers = rippleTimers.current; + return () => timers.forEach(clearTimeout); + }, []); + + // A tap inside the globe bounces it and sends a ring out from the touch. Nothing is + // prevented or stopped, so a click handler on the glow or a parent still gets it. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + if (reduceMotion) return; + const { x, y } = toLocalPoint(event); + if (!glowContains(x, y)) return; + const bounce = bounceRef.current; + if (bounce) { + bounce.classList.remove(`${block}__bounce--active`); + // Restarts the bounce when taps come faster than it lasts. + void bounce.offsetWidth; + bounce.classList.add(`${block}__bounce--active`); + } + const rect = event.currentTarget.getBoundingClientRect(); + const ripple = { + id: Date.now() + Math.random(), + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; + setRipples((current) => [...current, ripple]); + const timer = setTimeout(() => { + rippleTimers.current.delete(timer); + setRipples((current) => current.filter(({ id }) => id !== ripple.id)); + }, RIPPLE_DURATION); + rippleTimers.current.add(timer); + }; + + return ( +
+ + +
+ + + + + +
+ +
+ + + + + + +
+ + {/* Only the globe takes the pointer; around it, events reach what is behind. */} + + ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx new file mode 100644 index 000000000..986851639 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx @@ -0,0 +1,265 @@ +import { type PointerEvent, useEffect, useRef } from 'react'; +import clsx from 'clsx'; + +import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; +import { useStableCallback } from '../../../utils/useStableCallback'; +import { type VoiceDriverInput, VoiceVisualizationDriver } from './driver'; +import { audioLevelInput, expressivityInput } from './math'; +import { toLocalPoint } from './hitArea'; +import { FALLBACK_COLOR, readVoiceTheme, toHsvPalette } from './palette'; +import type { VoiceRenderer } from './renderers/types'; +import type { Hsv, VoiceVisualizationProps } from './types'; +import { useVoiceStateLabel } from './useVoiceStateLabel'; + +/** How many frames pass between two reads of the theme, about once a second at 60 Hz. */ +const THEME_FRAMES = 60; + +type ViewProps = VoiceVisualizationProps & { + /** Whether a point, centred and normalized to the shorter side with y up, is on the shape. */ + contains: (x: number, y: number) => boolean; + /** Makes the renderer for a freshly mounted canvas, or null when it cannot draw there. */ + createRenderer: (canvas: HTMLCanvasElement) => VoiceRenderer | null; + /** Remounts the canvas and remakes the renderer when it changes. */ + rendererKey: string; + /** + * The most device pixels per CSS pixel the canvas gets: a blurred shape looks the same at 1, + * at a quarter of the pixels of a 2x display, while a thin line needs the screen's own. + */ + maxPixelRatio: number; +}; + +/** + * What VoiceWave and VoiceGlow share: the motion advanced once a frame, the canvas kept at the + * element's size, the theme's colors, reduced motion, the pointer and the accessible name. + * + * The loop runs only while the element is on screen, reads `audioLevel` once a frame and + * redraws without re-rendering React. + */ +export const VoiceVisualizationView = ({ + audioLevel, + className, + colors, + contains, + createRenderer, + expressivity, + maxPixelRatio, + onPointerLeave, + onPointerMove, + onPointerUp, + rendererKey, + state, + ...restProps +}: ViewProps) => { + const label = useVoiceStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const canvasRef = useRef(null); + const driverRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const themeRef = useRef<{ dark: number; palette: Hsv[] }>({ dark: 0, palette: [] }); + const paletteRef = useRef<{ key: string; palette: Hsv[] }>({ key: '', palette: [] }); + const propsRef = useRef({ colors, expressivity, reduceMotion, state }); + + /** The theme's accent color and whether it sits on a dark backdrop. */ + const readTheme = useStableCallback(() => { + const root = rootRef.current; + if (!root || typeof getComputedStyle !== 'function') return; + themeRef.current = readVoiceTheme(root); + }); + + /** The driver's input from the latest props and the theme, keeping the audio level it has. */ + const applyInput = useStableCallback(() => { + // The first input settles the driver, so read the theme before it starts on a fallback. + if (!driverRef.current) readTheme(); + const { colors, expressivity, reduceMotion, state } = propsRef.current; + const key = (colors ?? []).join('|'); + if (paletteRef.current.key !== key) { + paletteRef.current = { key, palette: toHsvPalette(colors ?? []) }; + } + const own = paletteRef.current.palette; + const theme = themeRef.current.palette; + const input: VoiceDriverInput = { + audioLevel: driverRef.current?.input.audioLevel ?? null, + colors: own.length ? own : theme.length ? theme : toHsvPalette([FALLBACK_COLOR]), + expressivity: expressivityInput(expressivity), + reduceMotion, + state, + }; + if (driverRef.current) driverRef.current.input = input; + else + driverRef.current = new VoiceVisualizationDriver({ + ...input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }); + }); + + useEffect(() => { + audioLevelRef.current = audioLevel; + propsRef.current = { colors, expressivity, reduceMotion, state }; + applyInput(); + }); + + const makeRenderer = useStableCallback(createRenderer); + + useEffect(() => { + const root = rootRef.current; + const canvas = canvasRef.current; + if (!root || !canvas || typeof requestAnimationFrame !== 'function') return; + + const renderer = makeRenderer(canvas); + if (!renderer) return; + + const resize = () => { + const pixelRatio = Math.min(window.devicePixelRatio || 1, maxPixelRatio); + const width = Math.round(root.clientWidth * pixelRatio); + const height = Math.round(root.clientHeight * pixelRatio); + if (canvas.width !== width) canvas.width = width; + if (canvas.height !== height) canvas.height = height; + }; + resize(); + const resizeObserver = + typeof ResizeObserver === 'function' ? new ResizeObserver(resize) : null; + resizeObserver?.observe(root); + + // A new pixel ratio alone (another monitor, browser zoom) needs a new backing size too. The + // query matches only the ratio it was made for, so it is made again after each change. + let ratioQuery: MediaQueryList | null = null; + const onRatioChange = () => { + resize(); + watchRatio(); + }; + const watchRatio = () => { + ratioQuery?.removeEventListener?.('change', onRatioChange); + ratioQuery = + typeof window.matchMedia === 'function' + ? window.matchMedia(`(resolution: ${window.devicePixelRatio || 1}dppx)`) + : null; + ratioQuery?.addEventListener?.('change', onRatioChange); + }; + watchRatio(); + + let frameId = 0; + let last = 0; + let frames = 0; + let visible = true; + + const tick = (now: number) => { + frameId = 0; + const driver = driverRef.current; + if (!driver) return; + const seconds = last === 0 ? 0 : (now - last) / 1000; + if (last !== 0) { + driver.input = { + ...driver.input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }; + driver.advance(seconds); + } + last = now; + if (++frames % THEME_FRAMES === 0) { + readTheme(); + applyInput(); + } + renderer.draw({ + dark: themeRef.current.dark, + driver, + height: canvas.height, + seconds, + width: canvas.width, + }); + // The canvas fades in once it has something to show, as on iOS, rather than flashing + // empty while a shader compiles or the first frame lands. + if (!canvas.dataset.ready) canvas.dataset.ready = 'true'; + schedule(); + }; + + const schedule = () => { + if (visible && !frameId) frameId = requestAnimationFrame(tick); + }; + + const stop = () => { + if (frameId) cancelAnimationFrame(frameId); + frameId = 0; + // The first frame back shows where it was rather than leaping over the pause. + last = 0; + }; + + // Off screen, nothing is drawn; hidden tabs already pause requestAnimationFrame. + const intersectionObserver = + typeof IntersectionObserver === 'function' + ? new IntersectionObserver((entries) => { + visible = entries.some((entry) => entry.isIntersecting); + if (visible) schedule(); + else stop(); + }) + : null; + intersectionObserver?.observe(root); + + const colorScheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + const onSchemeChange = () => { + readTheme(); + applyInput(); + }; + colorScheme?.addEventListener?.('change', onSchemeChange); + + schedule(); + + return () => { + stop(); + resizeObserver?.disconnect(); + ratioQuery?.removeEventListener?.('change', onRatioChange); + intersectionObserver?.disconnect(); + colorScheme?.removeEventListener?.('change', onSchemeChange); + renderer.dispose?.(); + }; + }, [applyInput, makeRenderer, maxPixelRatio, readTheme, rendererKey]); + + // Pointer events are only watched, never prevented or stopped, so a click handler on the + // element or a parent still gets them. They come only from the shape's hit area: the rest of + // the box lets them through to whatever is behind it. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + const { x, y } = toLocalPoint(event); + if (contains(x, y)) driverRef.current?.tap(x, y); + }; + + const handlePointerMove = (event: PointerEvent) => { + onPointerMove?.(event); + // Only a hovering mouse or pen ripples it; a finger has no hover. + if (event.pointerType === 'touch') return; + const { x, y } = toLocalPoint(event); + const hovering = event.buttons === 0 && contains(x, y); + driverRef.current?.hover(hovering ? x : null, hovering ? y : null); + }; + + const handlePointerLeave = (event: PointerEvent) => { + onPointerLeave?.(event); + driverRef.current?.hover(null, null); + }; + + return ( +
+