diff --git a/ai-docs/i18n-v15-key-map.json b/ai-docs/i18n-v15-key-map.json index 5db06c4bb..2ac8960ee 100644 --- a/ai-docs/i18n-v15-key-map.json +++ b/ai-docs/i18n-v15-key-map.json @@ -1,6 +1,6 @@ { "$comment": "Migration table: natural-language translation key (v14) -> namespaced key (v15). Hand-reviewed; nothing regenerates it. Integrators who passed translationsForLanguage/registerTranslation dictionaries keyed on the old strings should use this to rename their keys. See ai-docs/i18n-v15-migration.md.", - "count": 621, + "count": 627, "keys": { "aria/Active": { "key": "a11y.accessibleLabel.active.ariaLabel", @@ -186,6 +186,30 @@ "key": "aiComponents.stateIndicator.workingMyMagic.text", "prose": true }, + "Connecting": { + "key": "aiComponents.voiceVisualization.connecting.text", + "prose": true + }, + "Listening": { + "key": "aiComponents.voiceVisualization.listening.text", + "prose": true + }, + "Ready": { + "key": "aiComponents.voiceVisualization.ready.text", + "prose": true + }, + "Speaking": { + "key": "aiComponents.voiceVisualization.speaking.text", + "prose": true + }, + "Thinking": { + "key": "aiComponents.voiceVisualization.thinking.text", + "prose": true + }, + "aria/Voice agent: {{ state }}": { + "key": "aiComponents.voiceVisualization.voiceAgent.ariaLabel", + "prose": true + }, "Generating...": { "key": "aiState.indicator.generating.label", "prose": true diff --git a/ai-docs/i18n-v15-migration.md b/ai-docs/i18n-v15-migration.md index 5a9aa6bab..e330fa647 100644 --- a/ai-docs/i18n-v15-migration.md +++ b/ai-docs/i18n-v15-migration.md @@ -36,7 +36,7 @@ imported and copied into `Streami18n` at construction, so they shipped even if y ## Renaming your keys -Every old key maps to exactly one new key. The full table (621 rows, including the 18 `aiComponents.*` keys of `stream-chat-react/ai-components`) is +Every old key maps to exactly one new key. The full table (627 rows, including the 24 `aiComponents.*` keys of `stream-chat-react/ai-components`) is [`i18n-v15-key-map.json`](https://github.com/GetStream/stream-chat-react/blob/master/ai-docs/i18n-v15-key-map.json): ```json diff --git a/examples/ai-chatbot/README.md b/examples/ai-chatbot/README.md index 87328f730..e8752f31c 100644 --- a/examples/ai-chatbot/README.md +++ b/examples/ai-chatbot/README.md @@ -1,7 +1,7 @@ # AI chatbot example A Next.js chatbot built with `stream-chat-react` and its `stream-chat-react/ai-components` entry point -(streaming messages, AI state indicator, AI message composer). Styles come from +(streaming messages, AI state indicator, AI message composer, voice agent visualizations). Styles come from `stream-chat-react/css/ai-components.css`. ## Getting started @@ -33,4 +33,10 @@ expose `POST /start-ai-agent` and `POST /summarize`, as called from `src/compone - **One copy of `@stream-io/state-store`.** The example keeps `installConfig.hoistingLimits: workspaces`, so the app installs its own `stream-chat` and `@stream-io/state-store` while the linked `stream-chat-react` workspace resolves the copies at the repository root. `stream-chat`, `@stream-io/i18n` and `stream-chat-react` hand each other store instances, so exactly one copy of `@stream-io/state-store` must be loaded. The app dedupes `stream-chat` and `@stream-io/state-store` through the Turbopack `resolveAlias` in `next.config.ts` plus the matching `paths` in `tsconfig.json`. - **Sidebar pagination.** The sidebar uses infinite scroll: v15's `ChannelList` has no load-more button. +## Voice mode + +The waveform button in the composer opens voice mode: a hands-free conversation in which the AI agent is shown as a `VoiceGlow` and your microphone as a `VoiceWave`. It listens to you, sends what you said, and reads the answer aloud before listening again. + +It needs a browser with the Web Speech API's `SpeechRecognition` (Chrome, Edge or Safari; the button is hidden elsewhere) and access to the microphone. Reading answers aloud uses `speechSynthesis`; where it is missing, the answer is only shown in the chat. + See [implementation.md](./implementation.md) for implementation notes. diff --git a/examples/ai-chatbot/implementation.md b/examples/ai-chatbot/implementation.md index 0b5b6fa73..c5d12277d 100644 --- a/examples/ai-chatbot/implementation.md +++ b/examples/ai-chatbot/implementation.md @@ -342,6 +342,23 @@ examples/ai-chatbot/ - Posts to `/summarize` endpoint with OpenAI platform - Returns summary string from API response +### 18. **useSendAIMessage.ts** + +- The send path shared by `MessageInputBar` and voice mode: creates a new conversation with `channel.ensureWatched()`, starts the AI agent when it is not watching yet, sends through `composer.send()` (with attachments), records the message against the rate limit and summarizes a new conversation in the background (the summary never delays or fails the send) +- Also holds `availableModels`, `defaultPlatformModel` and `isWatchedByAI` + +### 19. **VoiceMode** + +A hands-free conversation with the agent, built on `VoiceGlow`, `VoiceWave`, `useAudioLevel` and `useSpeechToText` from `stream-chat-react/ai-components`. + +- `VoiceModeButton` sits in the composer next to the dictation button and appears only where the browser has `SpeechRecognition` (Chrome, Edge, Safari) +- `VoiceModeProvider` (in `ChatContainer`) holds whether voice mode is open; `VoiceMode` is mounted inside `Channel` only while open, so the microphone is released when it closes +- The overlay shows a large `VoiceGlow` for the agent and a `VoiceWave` for the user's microphone, fed by `useAudioLevel` on the `getUserMedia` stream +- `useVoiceConversation` runs the turn loop: listen (dictation ends on silence), send through `useSendAIMessage` with a `MessageComposer` of its own (so a spoken turn leaves the typed draft and its attachments alone), wait for the agent's `ai_indicator` events, read the finished answer aloud with `speechSynthesis` (markdown stripped, code skipped), then listen again +- Voice agent state: `connecting` while the microphone is requested or the agent is being started, `listening` during the user's turn, `thinking` while the agent works, `speaking` while the answer is read aloud, `idle` when paused or stopped +- Pause/resume and end buttons; Escape closes it; the rate limit, a failed send and a denied microphone stop it with a message; closing it refreshes the composer's rate-limit banner +- `--str-chat__ai-voice-backdrop` is set to the demo background in `index.scss`, so the visualizations pick their light or dark look from the demo theme rather than the SDK theme + --- ## Next.js Architecture diff --git a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx index f4985985a..c4f3ce85d 100644 --- a/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx +++ b/examples/ai-chatbot/src/components/ChatContainer/ChatContainer.tsx @@ -15,6 +15,7 @@ import { MessageBubble } from '../MessageBubble'; import { MessageInputBar } from '../MessageInputBar'; import { AIStateIndicator } from '../AIStateIndicator'; import { TopNavBar } from '../TopNavBar'; +import { VoiceMode, VoiceModeProvider } from '../VoiceMode'; import './ChatContainer.scss'; interface ChatContainerProps { @@ -44,14 +45,17 @@ export const ChatContainer = ({ onToggleSidebar }: ChatContainerProps) => { MessageComposerUI: MessageInputBar, }} > - {channel && ( - - - - - - - )} + + {channel && ( + + + + + + + + )} + ); diff --git a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx index 2ffefc413..f08b37487 100644 --- a/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx +++ b/examples/ai-chatbot/src/components/MessageInputBar/MessageInputBar.tsx @@ -3,7 +3,6 @@ import { AIMessageComposer } from 'stream-chat-react/ai-components'; import { useEffect, useState } from 'react'; import { - type Channel, isImageFile, type LocalUploadAttachment, type UploadRequestFn, @@ -14,31 +13,17 @@ import { useChatContext, useMessageComposerController, } from 'stream-chat-react'; -import { startAiAgent, summarizeConversation } from '@/components/api'; -import { - checkRateLimit, - formatTimeRemaining, - recordMessage, -} from '@/components/rateLimitUtils'; +import { checkRateLimit, formatTimeRemaining } from '@/components/rateLimitUtils'; +import { availableModels, useSendAIMessage } from '@/components/useSendAIMessage'; +import { useVoiceMode, VoiceModeButton } from '@/components/VoiceMode'; import './MessageInputBar.scss'; -const isWatchedByAI = (channel: Channel) => - Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); - -const availableModels = [ - { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, - { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, - { - platform: 'gemini', - value: 'gemini-flash-latest', - label: 'Gemini Flash (latest)', - }, -] as const; - export const MessageInputBar = () => { const { client } = useChatContext(); const channel = useChannel(); const composer = useMessageComposerController(); + const sendAIMessage = useSendAIMessage(); + const { isOpen: isVoiceModeOpen } = useVoiceMode(); const { attachments } = useAttachmentsForPreview(); const [selectedPlatformModel, setSelectedPlatformModel] = useState(); @@ -52,7 +37,8 @@ export const MessageInputBar = () => { remainingMessages: 10, }); - // Check rate limit when channel changes or on mount + // Check rate limit on mount, when the channel changes and when voice mode opens or closes + // (voice mode records its sends too) useEffect(() => { if (!channel?.id) return; @@ -65,7 +51,7 @@ export const MessageInputBar = () => { const interval = setInterval(updateRateLimit, 60000); return () => clearInterval(interval); - }, [channel?.id]); + }, [channel?.id, isVoiceModeOpen]); useEffect(() => { if (!composer) return; @@ -127,48 +113,14 @@ export const MessageInputBar = () => { const platformModel = formData.get('platform-model'); setSelectedPlatformModel(platformModel as string); - composer.textComposer.setText(message as string); - - // A new conversation exists only locally until now; `Channel` no longer creates it. - // Both setup steps run before `send()`, so a failure here leaves the draft untouched. - if (!channel.initialized) { - await channel.ensureWatched(); - } - - const [platform, model] = (platformModel as string).split('|'); - - if (!isWatchedByAI(channel)) { - await startAiAgent(channel, model, platform); - } - - // `send()` composes the message, shows it optimistically, releases the composer and - // reports a failed send itself (the message stays in the list marked `failed`). - const result = await composer.send(); - - // `nothing-to-send` keeps the composer's contents (e.g. an upload is still running). - if (result === 'nothing-to-send') return; - - target.reset(); - - if (result !== 'sent') return; - - // Record message after successful send - recordMessage(channel.id!); + const sent = await sendAIMessage({ + onComposed: () => target.reset(), + platformModel: platformModel as string, + text: message as string, + }); // Update rate limit state - const newState = checkRateLimit(channel.id!); - setRateLimitState(newState); - - if (!channel.data?.custom?.summary) { - const summary = await summarizeConversation(message as string).catch(() => { - console.warn('Failed to summarize conversation'); - return null; - }); - - if (typeof summary === 'string' && summary.length > 0) { - await channel.update({ data: { custom: { summary } } }); - } - } + if (sent) setRateLimitState(checkRateLimit(channel.id!)); }} > @@ -205,6 +157,7 @@ export const MessageInputBar = () => {
+ = { + error: 'Voice mode stopped', + listening: 'Listening', + paused: 'Paused', + speaking: 'Speaking', + starting: 'Connecting', + waiting: 'Thinking', +}; + +const VoiceConversation = () => { + const { close } = useVoiceMode(); + const { agentState, error, micLevel, pause, phase, reply, resume, transcript } = + useVoiceConversation(); + const closeRef = useRef(null); + + useEffect(() => { + closeRef.current?.focus(); + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') close(); + }; + document.addEventListener('keydown', onKeyDown); + return () => document.removeEventListener('keydown', onKeyDown); + }, [close]); + + const paused = phase === 'paused'; + const caption = + phase === 'error' + ? error + : phase === 'speaking' + ? reply + : phase === 'listening' + ? transcript || 'Go ahead, I’m listening.' + : phase === 'waiting' + ? transcript + : ''; + + return ( +
+
+ +

+ {STATUS[phase]} +

+

{caption}

+
+ + {/* The glow is the agent; the wave under it is the user's microphone. */} + + +
+ + +
+
+ ); +}; + +/** + * A hands-free conversation with the agent over the chat: it listens, sends what was said, + * shows the agent thinking and reads its answer aloud. Mounted only while open, so the + * microphone is released as soon as it closes. + */ +export const VoiceMode = () => { + const { isOpen } = useVoiceMode(); + return isOpen ? : null; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx new file mode 100644 index 000000000..c28a69f93 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeButton.tsx @@ -0,0 +1,35 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useVoiceMode } from './VoiceModeContext'; + +const supportsVoiceMode = () => + typeof window !== 'undefined' && + !!( + (window as Window & { SpeechRecognition?: unknown }).SpeechRecognition ?? + (window as Window & { webkitSpeechRecognition?: unknown }).webkitSpeechRecognition + ) && + !!navigator.mediaDevices?.getUserMedia; + +/** Opens voice mode; shown only where the browser can listen (Chrome, Edge, Safari). */ +export const VoiceModeButton = () => { + const { isOpen, open } = useVoiceMode(); + // Support is known only in the browser, so the button appears after hydration. + const [isSupported, setIsSupported] = useState(false); + useEffect(() => setIsSupported(supportsVoiceMode()), []); + + if (!isSupported) return null; + + return ( + + ); +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx new file mode 100644 index 000000000..9018bc8a8 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/VoiceModeContext.tsx @@ -0,0 +1,34 @@ +'use client'; + +import { createContext, type ReactNode, useContext, useMemo, useState } from 'react'; + +type VoiceModeContextValue = { + close: () => void; + isOpen: boolean; + open: () => void; +}; + +const VoiceModeContext = createContext(undefined); + +export const useVoiceMode = () => { + const context = useContext(VoiceModeContext); + if (!context) { + throw new Error('useVoiceMode must be used within VoiceModeProvider'); + } + return context; +}; + +export const VoiceModeProvider = ({ children }: { children: ReactNode }) => { + const [isOpen, setIsOpen] = useState(false); + + const value = useMemo( + () => ({ + close: () => setIsOpen(false), + isOpen, + open: () => setIsOpen(true), + }), + [isOpen], + ); + + return {children}; +}; diff --git a/examples/ai-chatbot/src/components/VoiceMode/index.ts b/examples/ai-chatbot/src/components/VoiceMode/index.ts new file mode 100644 index 000000000..cfe736dc0 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/index.ts @@ -0,0 +1,3 @@ +export * from './VoiceMode'; +export * from './VoiceModeButton'; +export * from './VoiceModeContext'; diff --git a/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts new file mode 100644 index 000000000..65d5e4038 --- /dev/null +++ b/examples/ai-chatbot/src/components/VoiceMode/useVoiceConversation.ts @@ -0,0 +1,288 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { AIStates, type Channel, MessageComposer } from 'stream-chat'; +import { useAIState, useChannel, useChatContext } from 'stream-chat-react'; +import { + useAudioLevel, + useSpeechToText, + type VoiceAgentState, +} from 'stream-chat-react/ai-components'; +import { checkRateLimit } from '@/components/rateLimitUtils'; +import { isWatchedByAI, useSendAIMessage } from '@/components/useSendAIMessage'; + +/** + * Where the conversation is: + * - `starting`: asking for the microphone; + * - `listening`: dictating the user's turn; + * - `waiting`: the turn was sent and the agent is working on its answer; + * - `speaking`: reading the answer aloud; + * - `paused`: the user muted voice mode; + * - `error`: it can't go on (no microphone, rate limit, ...). + */ +export type VoicePhase = + | 'starting' + | 'listening' + | 'waiting' + | 'speaking' + | 'paused' + | 'error'; + +const WORKING_STATES: string[] = [ + AIStates.Thinking, + AIStates.ExternalSources, + AIStates.Generating, +]; + +/** Long answers are cut, as browsers stop reading aloud after a while. */ +const MAX_SPOKEN_LENGTH = 1200; +/** How long the agent may take to start on an answer before the turn is given up. */ +const ANSWER_TIMEOUT = 20_000; + +const canSpeak = () => typeof window !== 'undefined' && 'speechSynthesis' in window; + +/** Markdown as plain prose for reading aloud: code is skipped, formatting dropped. */ +export const toSpeech = (markdown: string) => + markdown + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/^\s{0,3}(#{1,6}|>|[-*+]|\d+\.)\s+/gm, '') + .replace(/[*_~|]/g, '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_SPOKEN_LENGTH); + +/** + * The agent's answer, when it is the channel's latest message. Read from `latestMessage`, which + * follows the newest message whatever window of the list is showing, and keeps up with its edits. + */ +const lastAgentMessage = (channel: Channel, userId?: string) => { + const latest = channel.messagePaginator.latestMessage; + return latest && latest.user?.id !== userId ? latest : undefined; +}; + +/** + * A hands-free conversation with the AI agent on the current channel: listen to the user, + * send what they said, wait for the agent's answer, read it aloud, then listen again. + */ +export const useVoiceConversation = () => { + const { client } = useChatContext(); + const channel = useChannel(); + const { aiState } = useAIState(channel); + const sendAIMessage = useSendAIMessage(); + // Spoken turns go through a composer of their own, so they leave the typed draft and its + // attachments alone. It is never registered, so it keeps no draft of its own. + const voiceComposer = useMemo( + () => + new MessageComposer({ + client, + compositionContext: channel, + config: { drafts: { enabled: false } }, + }), + [channel, client], + ); + + const [phase, setPhase] = useState('starting'); + const [mic, setMic] = useState(null); + const [transcript, setTranscript] = useState(''); + const [reply, setReply] = useState(''); + const [error, setError] = useState(null); + const [agentStarting, setAgentStarting] = useState(false); + + const phaseRef = useRef(phase); + useEffect(() => { + phaseRef.current = phase; + }, [phase]); + const transcriptRef = useRef(''); + const sawAgentWorkRef = useRef(false); + // The agent's last answer when a turn was sent, to tell a new answer from it. + const previousAnswerIdRef = useRef(undefined); + const wasListeningRef = useRef(false); + + const micLevel = useAudioLevel(mic); + + const fail = useCallback((message: string) => { + setError(message); + setPhase('error'); + }, []); + + const { isListening, isSupported, startListening, stopListening } = useSpeechToText({ + onError: (message) => { + // Silence and aborts end a turn on their own; anything else stops voice mode. + if (/no speech|aborted/i.test(message)) return; + fail(message); + }, + onTranscript: (text) => { + transcriptRef.current = text; + setTranscript(text); + }, + }); + + // The microphone, for the level of the user's voice; released when voice mode closes. + useEffect(() => { + let stream: MediaStream | undefined; + let cancelled = false; + if (!navigator.mediaDevices?.getUserMedia) { + fail('This browser cannot use a microphone.'); + return; + } + navigator.mediaDevices + .getUserMedia({ audio: true }) + .then((granted) => { + if (cancelled) { + granted.getTracks().forEach((track) => track.stop()); + return; + } + stream = granted; + setMic(granted); + setPhase((current) => (current === 'starting' ? 'listening' : current)); + }) + .catch(() => fail('Voice mode needs access to your microphone.')); + return () => { + cancelled = true; + stream?.getTracks().forEach((track) => track.stop()); + }; + }, [fail]); + + // Start dictating whenever it is the user's turn and recognition is not running. + useEffect(() => { + if (phase !== 'listening' || isListening || !isSupported) return; + // Recognition can refuse to restart in the same tick it ended. + const timeout = setTimeout(() => { + try { + transcriptRef.current = ''; + setTranscript(''); + startListening(); + } catch { + // Already started; the next end restarts it. + } + }, 150); + return () => clearTimeout(timeout); + }, [isListening, isSupported, phase, startListening]); + + const submit = useCallback( + (text: string) => { + if (checkRateLimit(channel.id!).isLimited) { + fail('Limit reached, 10 messages per conversation.'); + return; + } + sawAgentWorkRef.current = false; + previousAnswerIdRef.current = lastAgentMessage(channel, client.userID)?.id; + setReply(''); + setAgentStarting(!isWatchedByAI(channel)); + setPhase('waiting'); + sendAIMessage({ composer: voiceComposer, text }) + .then((sent) => { + if (!sent) fail('Your message could not be sent.'); + }) + .catch(() => fail('Your message could not be sent.')); + }, + [channel, client.userID, fail, sendAIMessage, voiceComposer], + ); + + // A turn ends when recognition stops: send what was said, or listen again after silence. + useEffect(() => { + if (isListening) { + wasListeningRef.current = true; + return; + } + if (!wasListeningRef.current) return; + wasListeningRef.current = false; + if (phaseRef.current !== 'listening') return; + const text = transcriptRef.current.trim(); + if (text) submit(text); + }, [isListening, submit]); + + const speak = useCallback((markdown: string) => { + const text = toSpeech(markdown); + if (!text || !canSpeak()) { + setPhase('listening'); + return; + } + setReply(text); + setPhase('speaking'); + const utterance = new SpeechSynthesisUtterance(text); + const done = () => { + if (phaseRef.current === 'speaking') setPhase('listening'); + }; + utterance.onend = done; + utterance.onerror = done; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + }, []); + + // The agent's indicator: working while it thinks and writes, idle again once the answer is in. + useEffect(() => { + if (phaseRef.current !== 'waiting') return; + if (WORKING_STATES.includes(aiState)) { + sawAgentWorkRef.current = true; + setAgentStarting(false); + return; + } + if (!sawAgentWorkRef.current) return; + speak(lastAgentMessage(channel, client.userID)?.text ?? ''); + }, [aiState, channel, client.userID, speak]); + + // A turn the agent is never seen working on (no indicator events, or missed ones) still ends: + // with its answer if one came in, otherwise with an error, rather than waiting forever. + useEffect(() => { + if (phase !== 'waiting') return; + const timeout = setTimeout(() => { + if (sawAgentWorkRef.current) return; + const answer = lastAgentMessage(channel, client.userID); + if (answer && answer.id !== previousAnswerIdRef.current) speak(answer.text ?? ''); + else fail('The agent did not answer.'); + }, ANSWER_TIMEOUT); + return () => clearTimeout(timeout); + }, [channel, client.userID, fail, phase, speak]); + + const pause = useCallback(() => { + setPhase('paused'); + stopListening(); + if (canSpeak()) window.speechSynthesis.cancel(); + }, [stopListening]); + + const resume = useCallback(() => setPhase('listening'), []); + + // Nothing keeps talking or listening once voice mode closes. + useEffect( + () => () => { + if (canSpeak()) window.speechSynthesis.cancel(); + }, + [], + ); + useEffect(() => { + if (phase === 'paused' || phase === 'error') stopListening(); + }, [phase, stopListening]); + + const agentState: VoiceAgentState = (() => { + switch (phase) { + case 'starting': + return 'connecting'; + case 'listening': + return 'listening'; + case 'waiting': + if (agentStarting) return 'connecting'; + // Without read-aloud, the answer being written is the agent speaking. + return aiState === AIStates.Generating && !canSpeak() ? 'speaking' : 'thinking'; + case 'speaking': + return 'speaking'; + default: + return 'idle'; + } + })(); + + return { + agentState, + error, + isSupported, + micLevel, + pause, + phase, + reply, + resume, + transcript, + }; +}; diff --git a/examples/ai-chatbot/src/components/index.scss b/examples/ai-chatbot/src/components/index.scss index 73a79dadc..111182a0a 100644 --- a/examples/ai-chatbot/src/components/index.scss +++ b/examples/ai-chatbot/src/components/index.scss @@ -91,6 +91,9 @@ --str-chat__ai-syntax-error: #f87171; --str-chat__ai-syntax-whitespace: hsla(0, 0%, 100%, 0.1); + /* VoiceWave and VoiceGlow sit on the demo's background, not the SDK theme's */ + --str-chat__ai-voice-backdrop: var(--ai-demo-bg-primary); + color-scheme: dark; color: var(--ai-demo-text-primary); background-color: var(--ai-demo-bg-primary); diff --git a/examples/ai-chatbot/src/components/useSendAIMessage.ts b/examples/ai-chatbot/src/components/useSendAIMessage.ts new file mode 100644 index 000000000..118f7d3d1 --- /dev/null +++ b/examples/ai-chatbot/src/components/useSendAIMessage.ts @@ -0,0 +1,97 @@ +'use client'; + +import { useCallback } from 'react'; +import type { Channel, MessageComposer } from 'stream-chat'; +import { useChannel, useMessageComposerController } from 'stream-chat-react'; +import { startAiAgent, summarizeConversation } from '@/components/api'; +import { recordMessage } from '@/components/rateLimitUtils'; + +export const availableModels = [ + { platform: 'openai', value: 'gpt-5.4-mini', label: 'GPT-5.4 mini' }, + { platform: 'openai', value: 'gpt-5.4-nano', label: 'GPT-5.4 nano' }, + { + platform: 'gemini', + value: 'gemini-flash-latest', + label: 'Gemini Flash (latest)', + }, +] as const; + +export const defaultPlatformModel = `${availableModels[0].platform}|${availableModels[0].value}`; + +export const isWatchedByAI = (channel: Channel) => + Object.keys(channel.state.watchers).some((watcher) => watcher.startsWith('ai-bot')); + +type SendAIMessageOptions = { + text: string; + /** `platform|model`, as the model select submits it. */ + platformModel?: string; + /** Called once the composer has taken the message (it was sent or failed to send). */ + onComposed?: () => void; + /** + * The composer to send through. Defaults to the one the message composer UI is bound to; voice + * mode passes its own, so a spoken turn neither replaces the typed draft nor sends its attachments. + */ + composer?: MessageComposer; +}; + +/** + * Sends a message to the AI agent: creates a new conversation on the server, starts the agent on + * the channel if it isn't watching yet, sends the message (with any attachments), records it + * against the rate limit and gives a new conversation its summary in the background. Shared by + * the composer and voice mode. + * + * Resolves to whether a message was sent, as soon as it was: the summary never delays or fails it. + */ +export const useSendAIMessage = () => { + const channel = useChannel(); + const boundComposer = useMessageComposerController(); + + return useCallback( + async ({ + text, + platformModel = defaultPlatformModel, + onComposed, + composer = boundComposer, + }: SendAIMessageOptions) => { + composer.textComposer.setText(text); + + // A new conversation exists only locally until now; `Channel` no longer creates it. + // Both setup steps run before `send()`, so a failure here leaves the draft untouched. + if (!channel.initialized) { + await channel.ensureWatched(); + } + + const [platform, model] = platformModel.split('|'); + + if (!isWatchedByAI(channel)) { + await startAiAgent(channel, model, platform); + } + + // `send()` composes the message, shows it optimistically, releases the composer and + // reports a failed send itself (the message stays in the list marked `failed`). + const result = await composer.send(); + + // `nothing-to-send` keeps the composer's contents (e.g. an upload is still running). + if (result === 'nothing-to-send') return false; + + onComposed?.(); + + if (result !== 'sent') return false; + + recordMessage(channel.id!); + + if (!channel.data?.custom?.summary) { + summarizeConversation(text) + .then((summary) => + typeof summary === 'string' && summary.length > 0 + ? channel.update({ data: { custom: { summary } } }) + : undefined, + ) + .catch(() => console.warn('Failed to summarize conversation')); + } + + return true; + }, + [boundComposer, channel], + ); +}; diff --git a/src/i18n/__tests__/catalog.fixture.json b/src/i18n/__tests__/catalog.fixture.json index 0cf75f406..0f6be7550 100644 --- a/src/i18n/__tests__/catalog.fixture.json +++ b/src/i18n/__tests__/catalog.fixture.json @@ -48,6 +48,12 @@ "aiComponents.stateIndicator.summoningMyInnerGenius.text": "Summoning my inner genius", "aiComponents.stateIndicator.thinkingReallyHard.text": "Thinking really hard", "aiComponents.stateIndicator.workingMyMagic.text": "Working my magic", + "aiComponents.voiceVisualization.connecting.text": "Connecting", + "aiComponents.voiceVisualization.listening.text": "Listening", + "aiComponents.voiceVisualization.ready.text": "Ready", + "aiComponents.voiceVisualization.speaking.text": "Speaking", + "aiComponents.voiceVisualization.thinking.text": "Thinking", + "aiComponents.voiceVisualization.voiceAgent.ariaLabel": "Voice agent: {{ state }}", "aiState.indicator.generating.label": "Generating...", "aiState.indicator.thinking.label": "Thinking...", "attachment.actions.giphyActions.ariaLabel": "Giphy actions", diff --git a/src/i18n/keys.ts b/src/i18n/keys.ts index e174bc5a2..362e51269 100644 --- a/src/i18n/keys.ts +++ b/src/i18n/keys.ts @@ -59,6 +59,12 @@ export type TranslationCatalog = { 'aiComponents.stateIndicator.summoningMyInnerGenius.text': 'Summoning my inner genius'; 'aiComponents.stateIndicator.thinkingReallyHard.text': 'Thinking really hard'; 'aiComponents.stateIndicator.workingMyMagic.text': 'Working my magic'; + 'aiComponents.voiceVisualization.connecting.text': 'Connecting'; + 'aiComponents.voiceVisualization.listening.text': 'Listening'; + 'aiComponents.voiceVisualization.ready.text': 'Ready'; + 'aiComponents.voiceVisualization.speaking.text': 'Speaking'; + 'aiComponents.voiceVisualization.thinking.text': 'Thinking'; + 'aiComponents.voiceVisualization.voiceAgent.ariaLabel': 'Voice agent: {{ state }}'; 'aiState.indicator.generating.label': 'Generating...'; 'aiState.indicator.thinking.label': 'Thinking...'; 'attachment.actions.giphyActions.ariaLabel': 'Giphy actions'; diff --git a/src/plugins/AIComponents/README.md b/src/plugins/AIComponents/README.md index 272f64307..4bb592733 100644 --- a/src/plugins/AIComponents/README.md +++ b/src/plugins/AIComponents/README.md @@ -31,9 +31,9 @@ Load `ai-components.css` in addition to `stream-chat-react/css/index.css`. From `stream-chat-react/ai-components`: -- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator` -- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText` -- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions` +- Components: `AIMarkdown`, `AIMessageComposer` (with the static sub-components `FileInput`, `TextInput`, `SpeechToTextButton`, `SubmitButton`, `ModelSelect`, `AttachmentPreview`, `AttachmentPreview.Item`), `StreamingMessage`, `AIStateIndicator`, `VoiceWave`, `VoiceGlow` +- Hooks: `useAttachments`, `useText`, `useIsDisabled`, `useAIMessageComposerContext`, `useSpeechToText`, `useAudioLevel` +- Types: `AIMessageComposerStore`, `StreamingMessageRef`, `UseMessageTextStreamingProps`, `ToolComponentProps`, `UseSpeechToTextOptions`, `VoiceAgentState`, `VoiceGlowRendering`, `VoiceVisualizationProps`, `VoiceWaveProps`, `VoiceGlowProps` `useMessageTextStreaming` is not exported from here; import it from the main entry point (`stream-chat-react`). @@ -185,6 +185,12 @@ Every key the components use: | `aiComponents.stateIndicator.summoningMyInnerGenius.text` | Summoning my inner genius | `Summoning my inner genius` | | `aiComponents.stateIndicator.thinkingReallyHard.text` | Thinking really hard | `Thinking really hard` | | `aiComponents.stateIndicator.workingMyMagic.text` | Working my magic | `Working my magic` | +| `aiComponents.voiceVisualization.connecting.text` | Connecting | `Connecting` | +| `aiComponents.voiceVisualization.listening.text` | Listening | `Listening` | +| `aiComponents.voiceVisualization.ready.text` | Ready | `Ready` | +| `aiComponents.voiceVisualization.speaking.text` | Speaking | `Speaking` | +| `aiComponents.voiceVisualization.thinking.text` | Thinking | `Thinking` | +| `aiComponents.voiceVisualization.voiceAgent.ariaLabel` | Voice agent: {{ state }} | `aria/Voice agent: {{ state }}` | | `messageComposer.sendButton.send.ariaLabel` (core) | Send | `aria/Send` | | `fileUpload.uploadButton.fileUpload.ariaLabel` (core) | File upload | `aria/File upload` | | `common.retryUpload.ariaLabel` (core) | Retry upload | `aria/Retry upload` | @@ -193,9 +199,9 @@ The last three are core `stream-chat-react` keys that the components reuse, so a ## Migrating from `stream-chat-react` v14 -If you already use `stream-chat-react/ai-components` on v14 (shipped from 14.13.0), the component API is identical on v15: same exports, props, class names and theming contract. Only the i18n keys changed. v14 used the English sentence (or an `aria/…` string) as the key; v15 uses namespaced keys. If you registered translations for these components, rename them with the table above or with `ai-docs/i18n-v15-key-map.json`. If you never translated them, there is nothing to do. +If you already use `stream-chat-react/ai-components` on v14 (shipped from 14.13.0; `VoiceWave`, `VoiceGlow` and `useAudioLevel` from 14.14.0), the component API is identical on v15: same exports, props, class names and theming contract. Only the i18n keys changed. v14 used the English sentence (or an `aria/…` string) as the key; v15 uses namespaced keys. If you registered translations for these components, rename them with the table above or with `ai-docs/i18n-v15-key-map.json`. If you never translated them, there is nothing to do. -v15 bundles English only. If your app runs in another language and relied on the translations 14.13 shipped for these components, those strings now fall back to English on v15 until you register translations yourself with `i18n.registerTranslation(lang, dict)`. The v14 values live in the 14.x `src/i18n/*.json` files of the 14.13.0 release (for example `git show v14.13.0:src/i18n/de.json`); map the keys with the table above or `ai-docs/i18n-v15-key-map.json`. +v15 bundles English only. If your app runs in another language and relied on the translations v14 shipped for these components, those strings now fall back to English on v15 until you register translations yourself with `i18n.registerTranslation(lang, dict)`. The v14 values live in the 14.x `src/i18n/*.json` files; the voice keys first appear in 14.14.0, so read them from that release or later (for example `git show v14.14.0:src/i18n/de.json`); map the keys with the table above or `ai-docs/i18n-v15-key-map.json`. ## Theming @@ -226,6 +232,8 @@ Precedence for every token except the code palette is: your `--str-chat__ai-*` o | `table-text` | table cell text | `text-secondary` | `#495057` | | `scrollbar-track` | attachment list scrollbar track | `background-core-surface-subtle` | `#f1f1f1` | | `scrollbar-thumb` | attachment list scrollbar thumb | `border-core-default` | `#ddd` | +| `voice-color` | `VoiceWave` / `VoiceGlow` color when `colors` is unset | `accent-primary` | `#005fff` | +| `voice-backdrop` | what `VoiceWave` / `VoiceGlow` sit on (light or dark) | `background-core-elevation-0` | `#ffffff` | | `syntax-*` | code palette (see below) | none | built-in palette | The code palette tokens are `--str-chat__ai-syntax-text`, `-selection-bg`, `-comment`, `-punctuation`, `-number`, `-keyword`, `-tag`, `-string`, `-function`, `-cyan`, `-error` and `-whitespace`. @@ -334,6 +342,73 @@ function MessageContent({ content }) { } ``` +### `VoiceWave` and `VoiceGlow` + +Two animated visualizations of a voice agent, the web counterparts of `AIVoiceWave` and `AIVoiceGlow` in the Android and iOS AI components, with the same motion. `VoiceWave` is a single string carrying a wave packet that erupts with speech; `VoiceGlow` is a soft globe whose color masses orbit inside it and swell with speech. + +Both draw on a `` inside a `div` with `role="img"` and an accessible name such as "Voice agent: Speaking". They have no size of their own and draw in the middle of the box they are given (`VoiceWave` about 80% as wide as the shorter side, `VoiceGlow` about half), so size them with CSS. The defaults are `width: 100%; height: 8rem` for the wave and `12rem` square for the glow. The wave's line is as thick as on Android and iOS, 0.6% of the shorter side and never under 1.5 device pixels, and drawn at the screen's own density up to 3x; the soft glow is drawn at 1x, which looks the same at a fraction of the cost. + +#### Props + +| Name | Type | Required | Description | +| -------------- | ------------------------------------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `state` | `'idle' \| 'connecting' \| 'listening' \| 'thinking' \| 'speaking'` | yes | What the agent is doing. The states differ only in motion, never in color, and a change eases from one motion into the next. | +| `colors` | `string[]` | no | One to five CSS colors, flowing through the shape. Past five are ignored. When omitted or empty, `--str-chat__ai-voice-color` is used, which follows the SDK accent color. | +| `expressivity` | `number` | no | From 0 to 2: how far and how fast it moves. Defaults to 1. | +| `audioLevel` | `() => number \| null \| undefined` | no | The level of whoever is talking, from 0 to 1, read once a frame, so a level changing many times a second re-renders nothing. Returning `null` (or omitting it) animates speech on its own. | +| `rendering` | `'auto' \| 'css'` | no | `VoiceGlow` only. `auto` (default) draws with a WebGL shader and falls back to `css` when WebGL is unavailable or its context is lost; `css` builds the glow from CSS gradients animated by the browser's compositor (see below). | + +Any other `div` prop (`className`, `style`, event handlers, ...) is passed to the root. Only the shape takes the pointer, as on iOS: the globe (within 0.32 of the shorter side from the centre) or the string. Elsewhere in the box, clicks and hovers reach whatever is behind it. On the shape, pointer events are watched and never stopped: a tap bounces it and a hovering mouse or pen ripples it, while your own `onClick` on the root or a parent still fires. + +Behavior: + +- The animation runs only while the element is on screen and the tab is visible. +- Every WebGL glow on a page draws with one shared WebGL context, so the shader compiles once and a page with many glows stays under the browser's limit on live contexts. +- It fades in over 0.3 s once its first frame is drawn (instantly with reduced motion), so it never flashes empty while a shader compiles. +- With `prefers-reduced-motion: reduce`, the motion slows to a crawl and taps and hover are ignored. +- The palette and the light or dark look follow the theme (`--str-chat__ai-voice-color` and `--str-chat__ai-voice-backdrop`), re-read about once a second, so a theme switch needs no prop. +- `VoiceWave` moves as on Android and iOS, smoothed for speech: its height rises quickly and settles slowly, its syllables swing it a little less, and as it grows its crests spread out rather than steepen into spikes. While idle, connecting or thinking it is the native wave exactly. + +`rendering='css'`, also the fallback when WebGL is unavailable, composes the glow the way the shader does, as gradients that are painted once and animated only through `transform` and `opacity`, so the browser's compositor runs the motion. It keeps moving while the main thread is busy (for example while a long answer streams in), needs no canvas or WebGL, and runs no script per frame unless `audioLevel` is given. It lacks the shader's fine marbling and its second, fainter riding mass: a tap bounces the glow and sends out a ring instead of a ripple, and a hovering pointer does nothing. Off screen, its animations pause. It needs `@property`, container query units and `color-mix()` (Chrome 111, Safari 16.4, Firefox 128 and later). + +- Under SSR the markup is rendered and drawing starts after hydration. + +#### Example + +```tsx +import { useEffect, useState } from 'react'; +import { + useAudioLevel, + type VoiceAgentState, + VoiceGlow, + VoiceWave, +} from 'stream-chat-react/ai-components'; + +const VoiceAgent = ({ state }: { state: VoiceAgentState }) => { + const [mic, setMic] = useState(null); + const micLevel = useAudioLevel(mic); + + useEffect(() => { + let stream: MediaStream | undefined; + navigator.mediaDevices.getUserMedia({ audio: true }).then((granted) => { + stream = granted; + setMic(granted); + }); + return () => stream?.getTracks().forEach((track) => track.stop()); + }, []); + + return ( + <> + + + + ); +}; +``` + ### `StreamingMessage` The `StreamingMessage` is a component that displays text with a typewriter animation effect, similar to ChatGPT. It's ideal for streaming AI responses. It's a simplified wrapper with typewriter animation effect around the `AIMarkdown` component. @@ -543,6 +618,16 @@ function VoiceInput() { } ``` +### `useAudioLevel` + +Reads the live level of a `MediaStream`'s audio (a microphone from `getUserMedia({ audio: true })`, or a remote track) for the `audioLevel` prop of `VoiceWave` and `VoiceGlow`. + +```ts +const useAudioLevel: (stream: MediaStream | null | undefined) => () => number | null; +``` + +It returns a stable getter that measures the level when called, from 0 to 1, so nothing re-renders as the level changes. It measures over the last 2048 samples (about 43 ms at 48 kHz), a few pitch periods of a voice, so the level holds steady through a vowel. The getter returns `null` while there is no stream or no Web Audio support. The audio graph is torn down when the stream changes and on unmount; the stream's tracks are left running, since they belong to you. + ## Examples ### Basic Chat Interface diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx new file mode 100644 index 000000000..2aa60a436 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlow.tsx @@ -0,0 +1,54 @@ +import { useCallback, useState } from 'react'; +import clsx from 'clsx'; + +import { glowContains } from './hitArea'; +import { createGlowWebGLRenderer } from './renderers/glowWebGLRenderer'; +import type { VoiceGlowRendering, VoiceVisualizationProps } from './types'; +import { VoiceGlowCss } from './VoiceGlowCss'; +import { VoiceVisualizationView } from './VoiceVisualizationView'; + +export type VoiceGlowProps = VoiceVisualizationProps & { + /** How the glow is drawn. See `VoiceGlowRendering`. Defaults to `auto`. */ + rendering?: VoiceGlowRendering; +}; + +/** + * A voice agent as a soft, airy globe: color masses orbit inside it and swell with speech, and + * the gaps between them let the background show through. Tapping the globe bounces it; drawn + * with WebGL, it also ripples from the touch and under a hovering mouse or pen. + * + * It has no size of its own and draws in the middle of the box it is given, about half as wide + * as its shorter side, so give it a square. + */ +export const VoiceGlow = ({ + className, + rendering = 'auto', + ...props +}: VoiceGlowProps) => { + // Set when WebGL cannot start or its context is lost; the CSS glow takes over from then on. + const [webGLFailed, setWebGLFailed] = useState(false); + + const createRenderer = useCallback((canvas: HTMLCanvasElement) => { + const renderer = createGlowWebGLRenderer(canvas, () => setWebGLFailed(true)); + if (!renderer) setWebGLFailed(true); + return renderer; + }, []); + + if (rendering === 'css' || webGLFailed) { + return ; + } + + return ( + + ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx new file mode 100644 index 000000000..2f3299b8c --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceGlowCss.tsx @@ -0,0 +1,345 @@ +import { + type CSSProperties, + type PointerEvent, + type ReactNode, + useEffect, + useRef, + useState, +} from 'react'; +import clsx from 'clsx'; + +import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; +import { + CSS_GLOW_RATES, + cssGlowPalette, + cssGlowRateKind, + massBackground, + MASSES, + orbitDistance, + orbitMask, + ORBITS, + RIDING_BODY, +} from './cssGlow'; +import { ease, MIC, REDUCED, scales } from './driver'; +import { glowContains, toLocalPoint } from './hitArea'; +import { audioLevelInput, expressivityInput } from './math'; +import { FALLBACK_COLOR, readVoiceTheme, toHsvPalette } from './palette'; +import type { VoiceVisualizationProps } from './types'; +import { useVoiceStateLabel } from './useVoiceStateLabel'; + +/** How often the theme is read again, so a switch to dark mode is followed. */ +const THEME_INTERVAL = 1000; +const RIPPLE_DURATION = 900; + +const block = 'str-chat__ai-voice-glow'; + +const Layer = ({ + children, + className, + style, +}: { + children?: ReactNode; + className: string; + style?: CSSProperties; +}) => ( +
+ {children} +
+); + +const orbitStyle = (orbit: number) => + ({ + '--str-ai-glow-direction': ORBITS[orbit].direction, + '--str-ai-glow-period': `${ORBITS[orbit].period}s`, + '--str-ai-glow-start': `${ORBITS[orbit].start}deg`, + }) as CSSProperties; + +const swayStyle = (orbit: number) => + ({ '--str-ai-glow-sway-period': `${ORBITS[orbit].sway}s` }) as CSSProperties; + +/** The masses the palette fills: the fourth and fifth only with that many colors. */ +const massesFor = (colorCount: number) => + MASSES.filter((_, index) => index < 3 || index < colorCount); + +/** The opaque color field: a base and the masses mixed over it in order. */ +const ColorField = ({ colorCount }: { colorCount: number }) => ( + + + + {massesFor(colorCount).map((mass, index) => ( + + +
+ + + ))} + + +); + +/** + * The color field seen through opacity that rides the first orbit: the mask turns with the + * orbit, and the inverse turns inside it keep the colors where they are. + */ +const RidingBody = ({ colorCount }: { colorCount: number }) => { + const { alpha, orbit, reach } = RIDING_BODY; + return ( + + + +
+ + + + + + + +
+ + + + ); +}; + +type Ripple = { id: number; x: number; y: number }; + +/** + * VoiceGlow drawn with CSS: the shader's composition, with every gradient painted once and only + * `transform` and `opacity` animated, so the browser's compositor runs the motion. It keeps + * moving while the page is busy, needs no canvas or WebGL, and costs no JavaScript per frame + * unless `audioLevel` is given. Off screen, its animations pause and its script rests. + * + * Script sets what the stylesheet cannot know: the palette (read from the theme when `colors` + * is not given), the playback rate of each animation for the state, and the live audio level. + */ +export const VoiceGlowCss = ({ + audioLevel, + className, + colors, + expressivity, + onPointerUp, + state, + style, + ...restProps +}: VoiceVisualizationProps) => { + const label = useVoiceStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const bounceRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const rippleTimers = useRef(new Set>()); + const [ripples, setRipples] = useState([]); + const [onScreen, setOnScreen] = useState(true); + const { amplitude, speed } = scales(expressivityInput(expressivity)); + const colorsKey = (colors ?? []).join('|'); + const colorCount = Math.min(colors?.length || 1, 5); + const hasAudioLevel = !!audioLevel; + + useEffect(() => { + audioLevelRef.current = audioLevel; + }); + + // Off screen, the animations pause (see the stylesheet) and the script below rests. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof IntersectionObserver !== 'function') return; + const observer = new IntersectionObserver((entries) => + setOnScreen(entries.some((entry) => entry.isIntersecting)), + ); + observer.observe(root); + return () => observer.disconnect(); + }, []); + + // The palette, from `colors` or the theme, and whether the glow sits on a dark backdrop. Read + // again about once a second while on screen, so a theme switch is followed without a prop. + useEffect(() => { + const root = rootRef.current; + if (!onScreen || !root || typeof getComputedStyle !== 'function') return; + const own = toHsvPalette(colorsKey ? colorsKey.split('|') : []); + let applied = ''; + const apply = () => { + const theme = readVoiceTheme(root); + const palette = own.length + ? own + : theme.palette.length + ? theme.palette + : toHsvPalette([FALLBACK_COLOR]); + const properties = cssGlowPalette(palette, theme.dark); + const next = JSON.stringify(properties) + theme.dark; + if (next === applied) return; + applied = next; + for (const [name, value] of Object.entries(properties)) { + root.style.setProperty(name, value); + } + root.setAttribute('data-dark', String(!!theme.dark)); + // Fades in once the palette is in, so the theme color it starts from never shows. + root.setAttribute('data-ready', 'true'); + }; + apply(); + const interval = setInterval(apply, THEME_INTERVAL); + const scheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + scheme?.addEventListener?.('change', apply); + return () => { + clearInterval(interval); + scheme?.removeEventListener?.('change', apply); + }; + }, [colorsKey, onScreen]); + + // Each state plays the animations at its own rates. Changing a playback rate keeps an + // animation where it is, where changing its duration would make it jump; a turn and its + // inverse always get the same rate, so they stay in step. + useEffect(() => { + const root = rootRef.current; + if (!root || typeof root.getAnimations !== 'function') return; + const rates = CSS_GLOW_RATES[state]; + const slow = reduceMotion ? REDUCED : 1; + for (const animation of root.getAnimations({ subtree: true })) { + const kind = cssGlowRateKind((animation as CSSAnimation).animationName ?? ''); + if (kind) animation.updatePlaybackRate(rates[kind] * speed * slow); + } + // Masses come and go with the palette, so their animations need rates too. + }, [colorsKey, reduceMotion, speed, state]); + + // A live audio level, eased like the driver eases it, is the only value written per frame, + // and only while the glow is on screen. + useEffect(() => { + const root = rootRef.current; + if ( + !hasAudioLevel || + !onScreen || + !root || + typeof requestAnimationFrame !== 'function' + ) + return; + let frame = 0; + let last = 0; + let level = 0; + let written = -1; + let live: boolean | null = null; + const tick = (now: number) => { + const target = audioLevelInput(audioLevelRef.current?.()); + const dt = last ? Math.min((now - last) / 1000, 0.05) : 0; + last = now; + level += ((target ?? 0) - level) * ease(MIC, dt); + if (Math.abs(level - written) > 0.002) { + written = level; + root.style.setProperty('--str-ai-glow-level', level.toFixed(3)); + } + if (live !== (target !== null)) { + live = target !== null; + root.setAttribute('data-live', String(live)); + } + frame = requestAnimationFrame(tick); + }; + frame = requestAnimationFrame(tick); + return () => { + cancelAnimationFrame(frame); + root.style.removeProperty('--str-ai-glow-level'); + root.removeAttribute('data-live'); + }; + }, [hasAudioLevel, onScreen]); + + useEffect(() => { + const timers = rippleTimers.current; + return () => timers.forEach(clearTimeout); + }, []); + + // A tap inside the globe bounces it and sends a ring out from the touch. Nothing is + // prevented or stopped, so a click handler on the glow or a parent still gets it. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + if (reduceMotion) return; + const { x, y } = toLocalPoint(event); + if (!glowContains(x, y)) return; + const bounce = bounceRef.current; + if (bounce) { + bounce.classList.remove(`${block}__bounce--active`); + // Restarts the bounce when taps come faster than it lasts. + void bounce.offsetWidth; + bounce.classList.add(`${block}__bounce--active`); + } + const rect = event.currentTarget.getBoundingClientRect(); + const ripple = { + id: Date.now() + Math.random(), + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; + setRipples((current) => [...current, ripple]); + const timer = setTimeout(() => { + rippleTimers.current.delete(timer); + setRipples((current) => current.filter(({ id }) => id !== ripple.id)); + }, RIPPLE_DURATION); + rippleTimers.current.add(timer); + }; + + return ( +
+ + +
+ + + + + +
+ +
+ + + + + + +
+ + {/* Only the globe takes the pointer; around it, events reach what is behind. */} + + ); +}; diff --git a/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx new file mode 100644 index 000000000..986851639 --- /dev/null +++ b/src/plugins/AIComponents/VoiceVisualization/VoiceVisualizationView.tsx @@ -0,0 +1,265 @@ +import { type PointerEvent, useEffect, useRef } from 'react'; +import clsx from 'clsx'; + +import { useReducedMotionPreference } from '../../../components/MessageList/hooks/useReducedMotionPreference'; +import { useStableCallback } from '../../../utils/useStableCallback'; +import { type VoiceDriverInput, VoiceVisualizationDriver } from './driver'; +import { audioLevelInput, expressivityInput } from './math'; +import { toLocalPoint } from './hitArea'; +import { FALLBACK_COLOR, readVoiceTheme, toHsvPalette } from './palette'; +import type { VoiceRenderer } from './renderers/types'; +import type { Hsv, VoiceVisualizationProps } from './types'; +import { useVoiceStateLabel } from './useVoiceStateLabel'; + +/** How many frames pass between two reads of the theme, about once a second at 60 Hz. */ +const THEME_FRAMES = 60; + +type ViewProps = VoiceVisualizationProps & { + /** Whether a point, centred and normalized to the shorter side with y up, is on the shape. */ + contains: (x: number, y: number) => boolean; + /** Makes the renderer for a freshly mounted canvas, or null when it cannot draw there. */ + createRenderer: (canvas: HTMLCanvasElement) => VoiceRenderer | null; + /** Remounts the canvas and remakes the renderer when it changes. */ + rendererKey: string; + /** + * The most device pixels per CSS pixel the canvas gets: a blurred shape looks the same at 1, + * at a quarter of the pixels of a 2x display, while a thin line needs the screen's own. + */ + maxPixelRatio: number; +}; + +/** + * What VoiceWave and VoiceGlow share: the motion advanced once a frame, the canvas kept at the + * element's size, the theme's colors, reduced motion, the pointer and the accessible name. + * + * The loop runs only while the element is on screen, reads `audioLevel` once a frame and + * redraws without re-rendering React. + */ +export const VoiceVisualizationView = ({ + audioLevel, + className, + colors, + contains, + createRenderer, + expressivity, + maxPixelRatio, + onPointerLeave, + onPointerMove, + onPointerUp, + rendererKey, + state, + ...restProps +}: ViewProps) => { + const label = useVoiceStateLabel(state); + const reduceMotion = useReducedMotionPreference(); + const rootRef = useRef(null); + const canvasRef = useRef(null); + const driverRef = useRef(null); + const audioLevelRef = useRef(audioLevel); + const themeRef = useRef<{ dark: number; palette: Hsv[] }>({ dark: 0, palette: [] }); + const paletteRef = useRef<{ key: string; palette: Hsv[] }>({ key: '', palette: [] }); + const propsRef = useRef({ colors, expressivity, reduceMotion, state }); + + /** The theme's accent color and whether it sits on a dark backdrop. */ + const readTheme = useStableCallback(() => { + const root = rootRef.current; + if (!root || typeof getComputedStyle !== 'function') return; + themeRef.current = readVoiceTheme(root); + }); + + /** The driver's input from the latest props and the theme, keeping the audio level it has. */ + const applyInput = useStableCallback(() => { + // The first input settles the driver, so read the theme before it starts on a fallback. + if (!driverRef.current) readTheme(); + const { colors, expressivity, reduceMotion, state } = propsRef.current; + const key = (colors ?? []).join('|'); + if (paletteRef.current.key !== key) { + paletteRef.current = { key, palette: toHsvPalette(colors ?? []) }; + } + const own = paletteRef.current.palette; + const theme = themeRef.current.palette; + const input: VoiceDriverInput = { + audioLevel: driverRef.current?.input.audioLevel ?? null, + colors: own.length ? own : theme.length ? theme : toHsvPalette([FALLBACK_COLOR]), + expressivity: expressivityInput(expressivity), + reduceMotion, + state, + }; + if (driverRef.current) driverRef.current.input = input; + else + driverRef.current = new VoiceVisualizationDriver({ + ...input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }); + }); + + useEffect(() => { + audioLevelRef.current = audioLevel; + propsRef.current = { colors, expressivity, reduceMotion, state }; + applyInput(); + }); + + const makeRenderer = useStableCallback(createRenderer); + + useEffect(() => { + const root = rootRef.current; + const canvas = canvasRef.current; + if (!root || !canvas || typeof requestAnimationFrame !== 'function') return; + + const renderer = makeRenderer(canvas); + if (!renderer) return; + + const resize = () => { + const pixelRatio = Math.min(window.devicePixelRatio || 1, maxPixelRatio); + const width = Math.round(root.clientWidth * pixelRatio); + const height = Math.round(root.clientHeight * pixelRatio); + if (canvas.width !== width) canvas.width = width; + if (canvas.height !== height) canvas.height = height; + }; + resize(); + const resizeObserver = + typeof ResizeObserver === 'function' ? new ResizeObserver(resize) : null; + resizeObserver?.observe(root); + + // A new pixel ratio alone (another monitor, browser zoom) needs a new backing size too. The + // query matches only the ratio it was made for, so it is made again after each change. + let ratioQuery: MediaQueryList | null = null; + const onRatioChange = () => { + resize(); + watchRatio(); + }; + const watchRatio = () => { + ratioQuery?.removeEventListener?.('change', onRatioChange); + ratioQuery = + typeof window.matchMedia === 'function' + ? window.matchMedia(`(resolution: ${window.devicePixelRatio || 1}dppx)`) + : null; + ratioQuery?.addEventListener?.('change', onRatioChange); + }; + watchRatio(); + + let frameId = 0; + let last = 0; + let frames = 0; + let visible = true; + + const tick = (now: number) => { + frameId = 0; + const driver = driverRef.current; + if (!driver) return; + const seconds = last === 0 ? 0 : (now - last) / 1000; + if (last !== 0) { + driver.input = { + ...driver.input, + audioLevel: audioLevelInput(audioLevelRef.current?.()), + }; + driver.advance(seconds); + } + last = now; + if (++frames % THEME_FRAMES === 0) { + readTheme(); + applyInput(); + } + renderer.draw({ + dark: themeRef.current.dark, + driver, + height: canvas.height, + seconds, + width: canvas.width, + }); + // The canvas fades in once it has something to show, as on iOS, rather than flashing + // empty while a shader compiles or the first frame lands. + if (!canvas.dataset.ready) canvas.dataset.ready = 'true'; + schedule(); + }; + + const schedule = () => { + if (visible && !frameId) frameId = requestAnimationFrame(tick); + }; + + const stop = () => { + if (frameId) cancelAnimationFrame(frameId); + frameId = 0; + // The first frame back shows where it was rather than leaping over the pause. + last = 0; + }; + + // Off screen, nothing is drawn; hidden tabs already pause requestAnimationFrame. + const intersectionObserver = + typeof IntersectionObserver === 'function' + ? new IntersectionObserver((entries) => { + visible = entries.some((entry) => entry.isIntersecting); + if (visible) schedule(); + else stop(); + }) + : null; + intersectionObserver?.observe(root); + + const colorScheme = + typeof window.matchMedia === 'function' + ? window.matchMedia('(prefers-color-scheme: dark)') + : null; + const onSchemeChange = () => { + readTheme(); + applyInput(); + }; + colorScheme?.addEventListener?.('change', onSchemeChange); + + schedule(); + + return () => { + stop(); + resizeObserver?.disconnect(); + ratioQuery?.removeEventListener?.('change', onRatioChange); + intersectionObserver?.disconnect(); + colorScheme?.removeEventListener?.('change', onSchemeChange); + renderer.dispose?.(); + }; + }, [applyInput, makeRenderer, maxPixelRatio, readTheme, rendererKey]); + + // Pointer events are only watched, never prevented or stopped, so a click handler on the + // element or a parent still gets them. They come only from the shape's hit area: the rest of + // the box lets them through to whatever is behind it. + const handlePointerUp = (event: PointerEvent) => { + onPointerUp?.(event); + const { x, y } = toLocalPoint(event); + if (contains(x, y)) driverRef.current?.tap(x, y); + }; + + const handlePointerMove = (event: PointerEvent) => { + onPointerMove?.(event); + // Only a hovering mouse or pen ripples it; a finger has no hover. + if (event.pointerType === 'touch') return; + const { x, y } = toLocalPoint(event); + const hovering = event.buttons === 0 && contains(x, y); + driverRef.current?.hover(hovering ? x : null, hovering ? y : null); + }; + + const handlePointerLeave = (event: PointerEvent) => { + onPointerLeave?.(event); + driverRef.current?.hover(null, null); + }; + + return ( +
+