From 933de2d591dec89f88e56f15e40a3d6e19bc47a5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 23:50:08 +0000 Subject: [PATCH 01/16] docs: add AUDIOCHAT_DESIGN.md (voice input design: Whisper Phase 1, local Phase 2) Co-Authored-By: jamisonmoore@gmail.com --- docs/voice/AUDIOCHAT_DESIGN.md | 215 +++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 docs/voice/AUDIOCHAT_DESIGN.md diff --git a/docs/voice/AUDIOCHAT_DESIGN.md b/docs/voice/AUDIOCHAT_DESIGN.md new file mode 100644 index 00000000000..c9c0fcad00c --- /dev/null +++ b/docs/voice/AUDIOCHAT_DESIGN.md @@ -0,0 +1,215 @@ +# AUDIOCHAT_DESIGN.md + +Title: Roo-Code Voice Input (Audio Chat) — Phase 1 Design and Phase 2 Plan + +Status: Draft +Owner: @BinaryBeastMaster +Scope: Planning/Analysis (no implementation yet) +Last updated: 2025-08-11 + +Overview +Roo-Code will add optional voice input to the chat box. The MVP integrates a cloud STT provider (OpenAI Whisper Realtime) with configurable auto-send on silence. Phase 2 adds a local/offline STT option to eliminate ongoing costs, while keeping professional cloud providers selectable. + +Free-tier summary (recurring vs. trial credits) +- OpenAI Whisper (Realtime) + - Recurring monthly free tier: None + - Trials/credits: None advertised; pay-as-you-go after enabling billing + - Source: https://openai.com/api/pricing +- Deepgram + - Recurring monthly free tier: None + - Trials/credits: One-time $200 signup credit; then pay-as-you-go (no minimums) + - Source: https://deepgram.com/pricing +- AssemblyAI + - Recurring monthly free tier: None + - Trials/credits: One-time $50 credit for evaluation (limited concurrency) + - Source: https://www.assemblyai.com/pricing/ +- Google Cloud Speech-to-Text v2 + - Recurring monthly free tier: None specific to STT v2 + - Trials/credits: New GCP users often get general $300 credits (not recurring; can be used for STT) + - Source: https://cloud.google.com/speech-to-text/pricing + +Notes +- Terms and prices can change; verify during setup. +- Cloud streaming STT is generally inexpensive (~$0.24–$0.96 per hour depending on provider/model and account plan). + +Goals and Non-goals +- Goals + - Enable dictation into the chat input with live interim and final transcripts. + - Make auto-send on silence configurable with a user-defined silence delay. + - Keep voice fully optional and off by default; no breaking changes to existing chat UX. + - Provide a provider-agnostic interface to allow adding more providers later. +- Non-goals (Phase 1) + - Wake word detection, diarization, translation, and advanced NLP. + - Multi-channel audio routing, noise profiling, or room calibration. + +User experience (Phase 1) +- Mic entry points + - Mic button in ChatTextArea toolbar with tooltip reflecting state. + - Push-to-Talk (PTT) hotkey by default (e.g., hold Cmd/Ctrl+M); configurable binding. +- Transcript display + - Interim transcript appears in the textarea styled subtly (gray/italic). + - Final transcript replaces interim seamlessly. +- Sending behavior + - Not auto-send by default; user can edit and press Enter. + - Optional send-on-release (off by default). + - Optional auto-send on silence after a configurable delay (default 3000 ms). +- Visual states + - Idle → Recording → Streaming → Waiting for silence → Sent + - Small input level indicator (waveform/level bar). +- Accessibility + - Keyboard-only operation, ARIA labels, visible focus states. +- Device selection + - Input device dropdown (default system mic); remember selection per workspace. + +Settings schema (initial) +- voice.sttProvider: "openai-realtime" | "local" | future providers +- voice.apiKey: stored in VS Code SecretStorage +- voice.language: "auto" | BCP-47 (default "en-US") +- voice.autoSendOnSilence: boolean +- voice.silenceDelayMs: number (default 3000) +- voice.pushToTalk: boolean (default true) +- voice.sendOnRelease: boolean (default false) +- voice.inputNoiseSuppression: boolean +- voice.inputAutoGainControl: boolean +- voice.inputEchoCancellation: boolean +- voice.localServerUrl: string (Phase 2) +- voice.punctuationEnabled: boolean (provider-dependent; default true) +- voice.profanityFilterEnabled: boolean (provider-dependent; default false) +- voice.sessionMaxMinutes: number (default 5) + +Provider selection and roadmap +- Phase 1 default: OpenAI Whisper Realtime over WebSocket +- Future cloud providers: Deepgram, AssemblyAI, Google STT v2 (as optional “professional” choices) +- Phase 2 local/offline: faster-whisper or whisper-cpp via a local server (HTTP or WebSocket) + +Architecture and integration points +- Webview UI (React) + - webview-ui/src/components/chat/ChatTextArea.tsx + - Add mic button, device selector, PTT handling, interim/final rendering, state indicators. + - webview-ui/src/utils/vscode.ts + - Use postMessage to send sttStart/sttChunk/sttStop and receive insertTextIntoTextarea + voice state updates. +- Extension host (VS Code) + - src/core/webview/webviewMessageHandler.ts + - Handle sttStart/sttChunk/sttStop from the webview; route to SttSession. + - src/core/webview/ClineProvider.ts + - Manage per-webview SttSession lifecycle and push transcript updates to the webview (insertTextIntoTextarea). + - src/shared/WebviewMessage.ts + - Add types: "sttStart" | "sttChunk" | "sttStop" | "voiceState" + - Extend insertTextIntoTextarea payload to include mode: "interim" | "final". + - New modules + - src/services/stt/providers/openaiRealtime.ts + - Minimal WS client for Realtime Whisper. + - src/services/stt/session.ts + - Provider-independent session controller (VAD/silence, timers, auto-send, error handling). + +Event and data flow +1) Webview captures mic audio via getUserMedia({ audio: { noiseSuppression, autoGainControl, echoCancellation, deviceId? } }) +2) Webview posts: + - sttStart: { sampleRate, encoding: "pcm16", language?, deviceId? } + - sttChunk: ArrayBuffer PCM16 frames at 20–50 ms cadence + - sttStop: {} +3) Extension opens provider WS on sttStart (auth via SecretStorage API key), forwards sttChunk frames. +4) Provider returns interim and final transcript events; SttSession relays insertTextIntoTextarea with mode = "interim" | "final". +5) Endpointing: + - SttSession runs energy-based VAD + hangover. + - On silence detected, start silenceDelayMs timer; if no speech resumes and autoSendOnSilence = true, auto-send current transcript; else just finalize and stop. +6) Webview updates UI state (voiceState) and shows transcript; if auto-send triggers, the chat input is cleared after send. + +Silence detection and endpointing +- Client-side VAD: + - Compute RMS/energy per audio frame, thresholds with hysteresis. + - Hangover to prevent rapid toggling; debounce with silenceDelayMs. +- Provider endpointing (if available): + - When provider offers endpointing signals, treat them as hints; still apply user-configured delay as a debounce for auto-send. +- Config exposure: + - Only silenceDelayMs is exposed; internal thresholds remain fixed to reduce complexity. + +Quality controls +- Language + - Prefer "auto"; allow manual language override via setting. +- Punctuation/casing and profanity filter (provider-dependent) + - Toggleable; default punctuation on, profanity filter off. +- Browser audio controls + - Toggle echo cancellation, noise suppression, and AGC when supported. + +Reliability and error handling +- Errors surfaced clearly: + - Mic permission denied, no input device, missing/invalid API key, billing required, network/WS disconnect, rate limited, session max duration reached. +- Retry and backoff: + - Retry transient network errors with exponential backoff; cap attempts per session; do not duplicate sends. +- Safe fallbacks: + - If provider fails mid-session, preserve any interim transcript in the textarea; stop cleanly and inform the user. +- Session timeout: + - Stop automatically after voice.sessionMaxMinutes to cap billing and avoid hanging sessions. + +Privacy, compliance, transparency +- First-run disclosure: + - A one-time note that audio is streamed to the selected provider; link to provider policy. +- Data retention: + - Link doc page explaining provider retention policies; no audio is stored by Roo-Code. +- Optional redaction: + - Client-side redaction list for common secret patterns before streaming; off by default. + +Telemetry (opt-in) +- Collect only non-content event counts (no audio, no transcripts): + - start, stop, failure, auto-send triggered, send-on-release used +- Disabled by default; respect VS Code telemetry settings. + +Provider abstraction +- Interface (per provider) + - startStream(opts) → { sendPcm(frame), stop(), onTranscript(cb), onError(cb) } +- Provider config map + - Flags for server-side endpointing, max chunk size, keepalive, language option hints. +- Backpressure + - Drop or coalesce frames under load to maintain real-time behavior; never block UI thread. + +Testing strategy +- Unit tests + - VAD/endpointing behavior around silenceDelayMs, including edge cases (low-volume, long pauses). + - Message contracts for sttStart/sttChunk/sttStop; insertTextIntoTextarea mode handling. + - Error surfaces and teardown pathways. +- Integration tests + - Mock provider (local WS) emitting interim/final transcripts. + - End-to-end flow: webview mic capture → transcripts → optional auto-send. +- Manual test checklist + - Mic permission flow, device switching, hotkey PTT, interim/final rendering, error injection, network toggle, session timeout. + +Phase 2: Local/offline and “professional” options +- Local server (faster-whisper/whisper-cpp) + - Docs for CPU vs. GPU, model recommendations, expected latency, RAM/VRAM needs. + - Health check endpoint; auto-detect availability to enable "local" in provider selection. +- Cloud providers + - Add Deepgram/AssemblyAI/Google STT with provider modules and per-provider settings. +- Benchmarks + - Snapshot doc of latency/accuracy/cost across cloud vs. local on a standard sample set. +- Power-user settings + - Chunk size, downsampling, max utterance duration, reconnect behavior. + +“Start without paying” note (docs) +- Use trial credits to evaluate: + - Deepgram: $200 signup credit + - AssemblyAI: $50 credit (eval concurrency limits) + - Google Cloud: $300 general credits on new accounts (usable for STT) +- Then switch to local mode in Phase 2 for ongoing zero cost. +- Setup checklist: + - Enter API key (SecretStorage), enable Voice, set auto-send and silence delay, verify mic permissions, and select provider/local. + +Open questions +- Default PTT hotkey (Cmd/Ctrl+M proposed) — confirm or adjust. +- Default language value ("en-US" vs. "auto") — proposed default "en-US" with "auto" available. +- Default sessionMaxMinutes (5 proposed). + +Risks and mitigations +- Browser audio quirks: gate controls behind feature detection; provide informative fallbacks. +- Network instability: retries with backoff and stop gracefully; preserve interim text. +- Provider changes: abstract via provider interface; document flags and endpoints. + +Rollout plan +- Feature-flag gated; off by default. +- Add “Voice” section to Extension Settings and docs. +- Ship Phase 1 with Whisper first; monitor issues; iterate. +- Phase 2: local server docs + provider, then additional cloud providers. + +Changelog (planned) +- Phase 1: Add voice input (optional), Whisper Realtime provider, silence auto-send option, silence delay, PTT, device selector, error handling, basic tests, telemetry opt-in. +- Phase 2: Local provider option, more providers, benchmarks, advanced configs. From 8ca7816f082924ede60242df1d1f8fe9e7b7e70f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 23:59:44 +0000 Subject: [PATCH 02/16] ci: disable Discord PR notifier workflow (no webhook configured) Co-Authored-By: jamisonmoore@gmail.com --- .github/workflows/discord-pr-notify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/discord-pr-notify.yml b/.github/workflows/discord-pr-notify.yml index 88c918edfe3..da7de82f402 100644 --- a/.github/workflows/discord-pr-notify.yml +++ b/.github/workflows/discord-pr-notify.yml @@ -8,7 +8,7 @@ on: jobs: notify: runs-on: ubuntu-latest - if: github.head_ref != 'changeset-release/main' + if: false steps: - name: Send Discord Notification run: | From 4897128dddb7d2d08e8873cc234b3d09f7a11084 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 00:16:43 +0000 Subject: [PATCH 03/16] feat(voice): add Settings UI section with Voice/STT API key stored securely via provider settings Co-Authored-By: jamisonmoore@gmail.com --- packages/types/src/global-settings.ts | 1 + packages/types/src/provider-settings.ts | 5 ++ src/package.json | 3 +- .../src/components/settings/SettingsView.tsx | 3 ++ .../src/components/settings/VoiceSettings.tsx | 52 +++++++++++++++++++ 5 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 webview-ui/src/components/settings/VoiceSettings.tsx diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index 39480e5a3d7..75734d464f2 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -202,6 +202,7 @@ export const SECRET_STATE_KEYS = [ "sambaNovaApiKey", "fireworksApiKey", "ioIntelligenceApiKey", + "voiceApiKey", ] as const satisfies readonly (keyof ProviderSettings)[] export type SecretState = Pick diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index e6b5c4ca260..0c723a519a6 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -282,6 +282,10 @@ const ioIntelligenceSchema = apiModelIdProviderModelSchema.extend({ ioIntelligenceApiKey: z.string().optional(), }) +const voiceSchema = z.object({ + voiceApiKey: z.string().optional(), +}) + const defaultSchema = z.object({ apiProvider: z.undefined(), }) @@ -354,6 +358,7 @@ export const providerSettingsSchema = z.object({ ...zaiSchema.shape, ...fireworksSchema.shape, ...ioIntelligenceSchema.shape, + ...voiceSchema.shape, ...codebaseIndexProviderSchema.shape, }) diff --git a/src/package.json b/src/package.json index f6007aef34f..c1767505102 100644 --- a/src/package.json +++ b/src/package.json @@ -173,8 +173,7 @@ { "command": "roo-cline.acceptInput", "title": "%command.acceptInput.title%", - "category": "%configuration.title%" - } + }, ], "menus": { "editor/context": [ diff --git a/webview-ui/src/components/settings/SettingsView.tsx b/webview-ui/src/components/settings/SettingsView.tsx index 630b59485d7..b9505b3b0c7 100644 --- a/webview-ui/src/components/settings/SettingsView.tsx +++ b/webview-ui/src/components/settings/SettingsView.tsx @@ -22,6 +22,7 @@ import { Globe, Info, MessageSquare, + Mic, LucideIcon, } from "lucide-react" @@ -62,6 +63,7 @@ import { ContextManagementSettings } from "./ContextManagementSettings" import { TerminalSettings } from "./TerminalSettings" import { ExperimentalSettings } from "./ExperimentalSettings" import { LanguageSettings } from "./LanguageSettings" +import { VoiceSettings } from "./VoiceSettings" import { About } from "./About" import { Section } from "./Section" import PromptsSettings from "./PromptsSettings" @@ -89,6 +91,7 @@ const sectionNames = [ "prompts", "experimental", "language", + "voice", "about", ] as const diff --git a/webview-ui/src/components/settings/VoiceSettings.tsx b/webview-ui/src/components/settings/VoiceSettings.tsx new file mode 100644 index 00000000000..40e83bca355 --- /dev/null +++ b/webview-ui/src/components/settings/VoiceSettings.tsx @@ -0,0 +1,52 @@ +import { HTMLAttributes, useCallback } from "react" +import { Mic } from "lucide-react" + +import type { ProviderSettings } from "@roo-code/types" + +import { useAppTranslation } from "@src/i18n/TranslationContext" +import { cn } from "@src/lib/utils" +import { VSCodeTextField } from "@vscode/webview-ui-toolkit/react" + +import { SectionHeader } from "./SectionHeader" +import { Section } from "./Section" + +type VoiceSettingsProps = HTMLAttributes & { + apiConfiguration: ProviderSettings + setApiConfigurationField: (field: keyof ProviderSettings, value: ProviderSettings[keyof ProviderSettings]) => void +} + +export const VoiceSettings = ({ apiConfiguration, setApiConfigurationField, className, ...props }: VoiceSettingsProps) => { + const { t } = useAppTranslation() + + const handleInput = useCallback( + (field: keyof ProviderSettings) => + (e: Event | any) => { + const value = (e.target as HTMLInputElement).value + setApiConfigurationField(field, value) + }, + [setApiConfigurationField], + ) + + return ( +
+ +
+ +
{t("settings:sections.voice")}
+
+ + +
+ + + +
+
+ ) +} From b28bf2f0062c0a7e68c48484410ee8b20a8032e1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 00:32:08 +0000 Subject: [PATCH 04/16] feat(voice): add mic button near Send; Voice settings under Experimental; add STT message types; fix package.json JSON parse error Co-Authored-By: jamisonmoore@gmail.com --- src/package.json | 4 +- src/shared/ExtensionMessage.ts | 7 ++++ src/shared/WebviewMessage.ts | 8 ++++ .../src/components/chat/ChatTextArea.tsx | 41 ++++++++++++++++++- .../src/components/settings/SettingsView.tsx | 10 ++++- 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/package.json b/src/package.json index c1767505102..232b501c12c 100644 --- a/src/package.json +++ b/src/package.json @@ -172,8 +172,8 @@ }, { "command": "roo-cline.acceptInput", - "title": "%command.acceptInput.title%", - }, + "title": "%command.acceptInput.title%" + } ], "menus": { "editor/context": [ diff --git a/src/shared/ExtensionMessage.ts b/src/shared/ExtensionMessage.ts index f9ac305e07f..76a1ab527a0 100644 --- a/src/shared/ExtensionMessage.ts +++ b/src/shared/ExtensionMessage.ts @@ -118,6 +118,7 @@ export interface ExtensionMessage { | "codeIndexSettingsSaved" | "codeIndexSecretStatus" | "showDeleteMessageDialog" + | "voiceState" | "showEditMessageDialog" | "commands" | "insertTextIntoTextarea" @@ -195,6 +196,12 @@ export interface ExtensionMessage { messageTs?: number context?: string commands?: Command[] + voice?: { + isRecording?: boolean + isStreaming?: boolean + silenceCountdownMs?: number + error?: string + } } export type ExtensionState = Pick< diff --git a/src/shared/WebviewMessage.ts b/src/shared/WebviewMessage.ts index 2d94896bf5b..969d223a536 100644 --- a/src/shared/WebviewMessage.ts +++ b/src/shared/WebviewMessage.ts @@ -210,6 +210,9 @@ export interface WebviewMessage { | "openCommandFile" | "deleteCommand" | "createCommand" + | "sttStart" + | "sttChunk" + | "sttStop" | "insertTextIntoTextarea" text?: string editedMessageContent?: string @@ -238,6 +241,11 @@ export interface WebviewMessage { slug?: string modeConfig?: ModeConfig timeout?: number + sttSampleRate?: number + sttEncoding?: "pcm16" + sttData?: ArrayBuffer | number[] | Uint8Array + sttLanguage?: string + insertMode?: "interim" | "final" payload?: WebViewMessagePayload source?: "global" | "project" requestId?: string diff --git a/webview-ui/src/components/chat/ChatTextArea.tsx b/webview-ui/src/components/chat/ChatTextArea.tsx index 5135eca2f2c..0d28fdba431 100644 --- a/webview-ui/src/components/chat/ChatTextArea.tsx +++ b/webview-ui/src/components/chat/ChatTextArea.tsx @@ -26,7 +26,7 @@ import ModeSelector from "./ModeSelector" import { ApiConfigSelector } from "./ApiConfigSelector" import { MAX_IMAGES_PER_MESSAGE } from "./ChatView" import ContextMenu from "./ContextMenu" -import { VolumeX, Image, WandSparkles, SendHorizontal } from "lucide-react" +import { VolumeX, Image, WandSparkles, SendHorizontal, Mic } from "lucide-react" import { IndexingStatusBadge } from "./IndexingStatusBadge" import { SlashCommandsPopover } from "./SlashCommandsPopover" import { cn } from "@/lib/utils" @@ -244,6 +244,23 @@ const ChatTextArea = forwardRef( setInputValue(t("chat:enhancePromptDescription")) } }, [inputValue, setInputValue, t]) + const [isRecording, setIsRecording] = useState(false) + const handleToggleMic = useCallback(() => { + if (!isRecording) { + setIsRecording(true) + const message: WebviewMessage = { + type: "sttStart", + sttSampleRate: 16000, + sttEncoding: "pcm16", + } + vscode.postMessage(message) + } else { + setIsRecording(false) + const stopMsg: WebviewMessage = { type: "sttStop" } + vscode.postMessage(stopMsg) + } + }, [isRecording]) + const allModes = useMemo(() => getAllModes(customModes), [customModes]) @@ -1114,7 +1131,27 @@ const ChatTextArea = forwardRef( {!isEditMode && ( -
+
+ + +
) From 74d68ac4c359728e0f7f7712944341a4c05f5243 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 02:37:30 +0000 Subject: [PATCH 09/16] style(settings): align Voice (Experimental) row with others; remove extra Section wrapper Co-Authored-By: jamisonmoore@gmail.com --- .../src/components/settings/SettingsView.tsx | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/webview-ui/src/components/settings/SettingsView.tsx b/webview-ui/src/components/settings/SettingsView.tsx index 2c27e74d356..fc09b126d89 100644 --- a/webview-ui/src/components/settings/SettingsView.tsx +++ b/webview-ui/src/components/settings/SettingsView.tsx @@ -726,14 +726,12 @@ const SettingsView = forwardRef(({ onDone, t setExperimentEnabled={setExperimentEnabled} experiments={experiments} /> -
- -
+ )} From 8f6672e5afef071f97892d9aa806348706dbd6c7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 02:49:33 +0000 Subject: [PATCH 10/16] feat(voice settings): add STT provider & mic capture method dropdowns; clarify Speech vs STT Co-Authored-By: jamisonmoore@gmail.com --- packages/types/src/provider-settings.ts | 2 + .../src/components/settings/VoiceSettings.tsx | 39 ++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index 0c723a519a6..eefb33b17fa 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -284,6 +284,8 @@ const ioIntelligenceSchema = apiModelIdProviderModelSchema.extend({ const voiceSchema = z.object({ voiceApiKey: z.string().optional(), + voiceSttProvider: z.enum(["openai-realtime", "local"]).optional(), + voiceMicCaptureMethod: z.enum(["built-in", "vscode-speech"]).optional(), }) const defaultSchema = z.object({ diff --git a/webview-ui/src/components/settings/VoiceSettings.tsx b/webview-ui/src/components/settings/VoiceSettings.tsx index 3b42f3456bb..95d7760ec2e 100644 --- a/webview-ui/src/components/settings/VoiceSettings.tsx +++ b/webview-ui/src/components/settings/VoiceSettings.tsx @@ -5,7 +5,7 @@ import type { ProviderSettings } from "@roo-code/types" import { useAppTranslation } from "@src/i18n/TranslationContext" import { cn } from "@src/lib/utils" -import { VSCodeTextField, VSCodeCheckbox } from "@vscode/webview-ui-toolkit/react" +import { VSCodeTextField, VSCodeCheckbox, VSCodeDropdown, VSCodeOption } from "@vscode/webview-ui-toolkit/react" import { SectionHeader } from "./SectionHeader" import { Section } from "./Section" @@ -59,6 +59,43 @@ export const VoiceSettings = ({ {voiceEnabled && (
+
+ + setApiConfigurationField("voiceSttProvider", e.target.value)} + className="w-full"> + + OpenAI Realtime (Transcription) + + + Local (coming soon) + + +
+ “VS Code Speech” is a mic capture helper; the STT provider actually does the + transcription. +
+
+ +
+ + setApiConfigurationField("voiceMicCaptureMethod", e.target.value)} + className="w-full"> + + Built-in (webview) + + + VS Code Speech extension + + +
+ If built-in capture is blocked, you’ll be prompted to install “VS Code Speech”. +
+
+
Date: Tue, 12 Aug 2025 02:57:44 +0000 Subject: [PATCH 11/16] refactor(voice settings): make mic capture automatic (built-in then VS Code Speech); remove mic-capture selector; clarify UI copy; schema: drop voiceMicCaptureMethod Co-Authored-By: jamisonmoore@gmail.com --- packages/types/src/provider-settings.ts | 1 - .../src/components/settings/VoiceSettings.tsx | 23 ++++--------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index eefb33b17fa..69f0547293f 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -285,7 +285,6 @@ const ioIntelligenceSchema = apiModelIdProviderModelSchema.extend({ const voiceSchema = z.object({ voiceApiKey: z.string().optional(), voiceSttProvider: z.enum(["openai-realtime", "local"]).optional(), - voiceMicCaptureMethod: z.enum(["built-in", "vscode-speech"]).optional(), }) const defaultSchema = z.object({ diff --git a/webview-ui/src/components/settings/VoiceSettings.tsx b/webview-ui/src/components/settings/VoiceSettings.tsx index 95d7760ec2e..78991e7ac51 100644 --- a/webview-ui/src/components/settings/VoiceSettings.tsx +++ b/webview-ui/src/components/settings/VoiceSettings.tsx @@ -73,27 +73,14 @@ export const VoiceSettings = ({
- “VS Code Speech” is a mic capture helper; the STT provider actually does the - transcription. + Mic capture runs automatically. “VS Code Speech” is a mic-capture helper; the STT + provider below performs the transcription.
-
- - setApiConfigurationField("voiceMicCaptureMethod", e.target.value)} - className="w-full"> - - Built-in (webview) - - - VS Code Speech extension - - -
- If built-in capture is blocked, you’ll be prompted to install “VS Code Speech”. -
+
+ Mic capture is automatic: the chat webview tries built‑in capture first; if blocked, you’ll + be prompted to install “VS Code Speech”. The STT Provider handles the transcription.
From 8974020473eb1ee3822fc030eddbc7aa701a9d40 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 03:13:53 +0000 Subject: [PATCH 12/16] fix(voice): wire STT webview messages in handler (sttStart/Chunk/Stop, voiceEnsureSpeechExtension) so mic click initiates streaming Co-Authored-By: jamisonmoore@gmail.com --- src/core/webview/webviewMessageHandler.ts | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index cb782d38832..95f7b642f03 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -2604,6 +2604,66 @@ export const webviewMessageHandler = async ( break } + case "sttStart": { + try { + const vw = (provider as any).view?.webview + if (vw) { + await provider.sttStart(vw, { + sampleRate: message.sttSampleRate ?? 16000, + encoding: message.sttEncoding ?? "pcm16", + language: message.sttLanguage, + }) + } + } catch (error) { + provider.log(`Error in sttStart: ${error instanceof Error ? error.message : String(error)}`) + await provider.postMessageToWebview({ + type: "voiceState", + voice: { error: "stt_start_failed", isRecording: false, isStreaming: false }, + }) + } + break + } + case "sttChunk": { + try { + const vw = (provider as any).view?.webview + if (vw && message.sttData) { + provider.sttChunk(vw, message.sttData) + } + } catch (error) { + provider.log(`Error in sttChunk: ${error instanceof Error ? error.message : String(error)}`) + } + break + } + case "sttStop": { + try { + const vw = (provider as any).view?.webview + if (vw) { + provider.sttStop(vw) + } + } catch (error) { + provider.log(`Error in sttStop: ${error instanceof Error ? error.message : String(error)}`) + } + break + } + case "voiceEnsureSpeechExtension": { + try { + const installed = await provider.ensureVsCodeSpeechInstalled() + await provider.postMessageToWebview({ + type: "voiceState", + voice: { speechExtensionInstalled: installed === true }, + }) + } catch (error) { + provider.log( + `Error ensuring VS Code Speech extension: ${error instanceof Error ? error.message : String(error)}`, + ) + await provider.postMessageToWebview({ + type: "voiceState", + voice: { speechExtensionInstalled: false, error: "speech_extension_install_failed" }, + }) + } + break + } + case "insertTextIntoTextarea": { const text = message.text if (text) { From 35187e2bfef4965b5393c26dba5699bfb59322e3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 03:42:47 +0000 Subject: [PATCH 13/16] fix(voice): close getStateToPostToWebview; include voiceEnabled in getState; mic UI wiring and Save activation Co-Authored-By: jamisonmoore@gmail.com --- src/core/webview/ClineProvider.ts | 4 ++++ webview-ui/src/components/chat/ChatTextArea.tsx | 4 ++++ webview-ui/src/components/settings/SettingsView.tsx | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 2b5e75819bd..574cfb44af8 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -1800,6 +1800,7 @@ export class ClineProvider maxDiagnosticMessages, includeTaskHistoryInEnhance, remoteControlEnabled, + voiceEnabled, } = await this.getState() const telemetryKey = process.env.POSTHOG_API_KEY @@ -1928,6 +1929,7 @@ export class ClineProvider maxDiagnosticMessages: maxDiagnosticMessages ?? 50, includeTaskHistoryInEnhance: includeTaskHistoryInEnhance ?? false, remoteControlEnabled: remoteControlEnabled ?? false, + voiceEnabled: voiceEnabled ?? false, } } @@ -2057,6 +2059,8 @@ export class ClineProvider terminalZshOhMy: stateValues.terminalZshOhMy ?? false, terminalZshP10k: stateValues.terminalZshP10k ?? false, terminalZdotdir: stateValues.terminalZdotdir ?? false, + voiceEnabled: stateValues.voiceEnabled ?? false, + terminalCompressProgressBar: stateValues.terminalCompressProgressBar ?? true, mode: stateValues.mode ?? defaultModeSlug, language: stateValues.language ?? formatLanguage(vscode.env.language), diff --git a/webview-ui/src/components/chat/ChatTextArea.tsx b/webview-ui/src/components/chat/ChatTextArea.tsx index a9ce0bde57b..98a8fdb0432 100644 --- a/webview-ui/src/components/chat/ChatTextArea.tsx +++ b/webview-ui/src/components/chat/ChatTextArea.tsx @@ -960,6 +960,10 @@ const ChatTextArea = forwardRef( setIsTtsPlaying(true) } else if (message.type === "ttsStop") { setIsTtsPlaying(false) + } else if (message.type === "voiceState") { + if (message.voice?.speechExtensionInstalled) { + setSpeechExtensionInstalled(true) + } } }) diff --git a/webview-ui/src/components/settings/SettingsView.tsx b/webview-ui/src/components/settings/SettingsView.tsx index fc09b126d89..8899a2ce758 100644 --- a/webview-ui/src/components/settings/SettingsView.tsx +++ b/webview-ui/src/components/settings/SettingsView.tsx @@ -231,7 +231,7 @@ const SettingsView = forwardRef(({ onDone, t const previousValue = prevState.apiConfiguration?.[field] const isInitialSync = previousValue === undefined && value !== undefined - if (field === "voiceApiKey") { + if (field === "voiceApiKey" || field === "voiceSttProvider") { setChangeDetected(true) } else if (!isInitialSync) { setChangeDetected(true) From f87250edfefde7fd058c934a3cb2e9dbd21d8118 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 03:56:50 +0000 Subject: [PATCH 14/16] =?UTF-8?q?docs(devhost):=20add=20troubleshooting=20?= =?UTF-8?q?section=20for=20F5=20when=20Dev=20Host=20opens=20without=20Roo?= =?UTF-8?q?=E2=80=91Code=20loaded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: jamisonmoore@gmail.com --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 1b5e8fe2ecd..d2150ad6c8b 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,30 @@ Press `F5` (or go to **Run** → **Start Debugging**) in VSCode. This will open - Changes to the webview will appear immediately. - Changes to the core extension will also hot reload automatically. +### Troubleshooting F5: Dev Host opens without Roo‑Code + +If pressing F5 opens an Extension Development Host window but Roo‑Code is not loaded: + +1. Make sure you opened the repo root as the workspace (File → Open Folder… → Roo‑Code). +2. Use the “Run and Debug” panel and pick “Run Extension” (the Roo‑Code config). It uses: + - args: `--extensionDevelopmentPath=${workspaceFolder}/src` + - default build task: “watch” (builds webview and extension bundle) +3. Approve builds on first install (Windows often requires this): + - In a terminal at the repo root, run: + - `pnpm approve-builds` (press “a” to select all, then Enter) + - `pnpm -w install` + - Then F5 again. +4. In the Dev Host, run “Roo: Focus Panel” (or “Roo: New Task”) to activate the extension. +5. If it still doesn’t appear, install the VSIX inside the Dev Host: + - Extensions → “…” → Install from VSIX… → pick the file from `bin/roo-cline-.vsix` + - Then try “Roo: Focus Panel” again. + +Logs to check: + +- View → Output → “Roo Code” +- Command Palette → “Developer: Toggle Developer Tools” → Console (Extension Host) +- In the Roo chat webview: “Developer: Open Webview Developer Tools” → Console + ### Automated VSIX Installation To build and install the extension as a VSIX package directly into VSCode: From 5019ca821971bebaa10f5a1d9ac2f64be1a6bf9f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 04:15:50 +0000 Subject: [PATCH 15/16] fix(activation): ensure Dev Host activation via onView/onCommand and add palette focus command; docs: clarify F5 activation and logs Co-Authored-By: jamisonmoore@gmail.com --- README.md | 7 +++++-- src/extension.ts | 2 +- src/package.json | 11 ++++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d2150ad6c8b..f10e007e41f 100644 --- a/README.md +++ b/README.md @@ -162,14 +162,17 @@ If pressing F5 opens an Extension Development Host window but Roo‑Code is not - `pnpm approve-builds` (press “a” to select all, then Enter) - `pnpm -w install` - Then F5 again. -4. In the Dev Host, run “Roo: Focus Panel” (or “Roo: New Task”) to activate the extension. +4. Activate Roo‑Code in the Dev Host: + - Command Palette → run “Roo: Focus Panel” (activates via command) + - Or open the Roo view from the Activity Bar (activates via view) + - Confirm “Roo Code (Development)” is listed in the Extensions panel 5. If it still doesn’t appear, install the VSIX inside the Dev Host: - Extensions → “…” → Install from VSIX… → pick the file from `bin/roo-cline-.vsix` - Then try “Roo: Focus Panel” again. Logs to check: -- View → Output → “Roo Code” +- View → Output → “Roo Code” (look for “extension activated”) - Command Palette → “Developer: Toggle Developer Tools” → Console (Extension Host) - In the Roo chat webview: “Developer: Open Webview Developer Tools” → Console diff --git a/src/extension.ts b/src/extension.ts index 3df954fa0a7..5b9328e542b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -58,7 +58,7 @@ export async function activate(context: vscode.ExtensionContext) { extensionContext = context outputChannel = vscode.window.createOutputChannel(Package.outputChannel) context.subscriptions.push(outputChannel) - outputChannel.appendLine(`${Package.name} extension activated - ${JSON.stringify(Package)}`) + outputChannel.appendLine(`${Package.name} extension activated`) // Migrate old settings to new await migrateSettings(context, outputChannel) diff --git a/src/package.json b/src/package.json index 232b501c12c..af09bca554a 100644 --- a/src/package.json +++ b/src/package.json @@ -47,7 +47,11 @@ ], "activationEvents": [ "onLanguage", - "onStartupFinished" + "onStartupFinished", + "onView:roo-cline.SidebarProvider", + "onCommand:roo-cline.focusPanel", + "onCommand:roo-cline.newTask", + "onCommand:roo-cline.settingsButtonClicked" ], "main": "./dist/extension.js", "contributes": { @@ -173,6 +177,11 @@ { "command": "roo-cline.acceptInput", "title": "%command.acceptInput.title%" + }, + { + "command": "roo-cline.focusPanel", + "title": "%command.focusPanel.title%", + "category": "%configuration.title%" } ], "menus": { From 0af1b3b25250a33ed35335c023f898aa79834e01 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 04:19:41 +0000 Subject: [PATCH 16/16] i18n: add missing command.focusPanel.title for VSIX packaging Co-Authored-By: jamisonmoore@gmail.com --- src/package.nls.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/package.nls.json b/src/package.nls.json index 36ef72a8238..69890b113ea 100644 --- a/src/package.nls.json +++ b/src/package.nls.json @@ -37,5 +37,6 @@ "settings.customStoragePath.description": "Custom storage path. Leave empty to use the default location. Supports absolute paths (e.g. 'D:\\RooCodeStorage')", "settings.enableCodeActions.description": "Enable Roo Code quick fixes", "settings.autoImportSettingsPath.description": "Path to a RooCode configuration file to automatically import on extension startup. Supports absolute paths and paths relative to the home directory (e.g. '~/Documents/roo-code-settings.json'). Leave empty to disable auto-import.", - "settings.useAgentRules.description": "Enable loading of AGENTS.md files for agent-specific rules (see https://agent-rules.org/)" + "settings.useAgentRules.description": "Enable loading of AGENTS.md files for agent-specific rules (see https://agent-rules.org/)", + "command.focusPanel.title": "Roo: Focus Panel" }