diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0e6db87b109..d8bfe3c7da4 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -89,6 +89,7 @@ import { persistServerRuntimeState, } from "./serverRuntimeState.ts"; import { orchestrationHttpApiLayer } from "./orchestration/http.ts"; +import { voiceHttpApiLayer } from "./voice/http.ts"; import * as NetService from "@t3tools/shared/Net"; import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale"; @@ -352,6 +353,7 @@ export const makeRoutesLayer = Layer.mergeAll( Layer.provide(authHttpApiLayer), Layer.provide(connectHttpApiLayer), Layer.provide(orchestrationHttpApiLayer), + Layer.provide(voiceHttpApiLayer), Layer.provide(serverEnvironmentHttpApiLayer), Layer.provide(environmentAuthenticatedAuthLayer), ), diff --git a/apps/server/src/voice/VoiceTranscription.test.ts b/apps/server/src/voice/VoiceTranscription.test.ts new file mode 100644 index 00000000000..2697bb2750a --- /dev/null +++ b/apps/server/src/voice/VoiceTranscription.test.ts @@ -0,0 +1,98 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { CodexSettings } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; + +import { + CODEX_TRANSCRIBE_URL, + readCodexVoiceCredentials, + transcribeCodexAudio, +} from "./VoiceTranscription.ts"; + +const decodeCodexSettings = Schema.decodeSync(CodexSettings); + +it.layer(NodeServices.layer)("VoiceTranscription credentials", (it) => { + it.effect("reads ChatGPT OAuth credentials from the selected shadow home", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sharedHome = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-shared-" }); + const shadowHome = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-shadow-" }); + yield* fileSystem.writeFileString( + path.join(shadowHome, "auth.json"), + '{"auth_mode":"chatgpt","tokens":{"access_token":"oauth-token","account_id":"account-123"}}', + ); + + const credentials = yield* readCodexVoiceCredentials( + decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }), + ); + + expect(credentials).toEqual({ accessToken: "oauth-token", accountId: "account-123" }); + }), + ); + + it.effect("rejects Codex API-key auth", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const homePath = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-api-key-" }); + yield* fileSystem.writeFileString( + path.join(homePath, "auth.json"), + '{"auth_mode":"apikey","OPENAI_API_KEY":"secret"}', + ); + + const error = yield* readCodexVoiceCredentials(decodeCodexSettings({ homePath })).pipe( + Effect.flip, + ); + expect(error._tag).toBe("EnvironmentHttpForbiddenError"); + }), + ); +}); + +it.effect("posts audio to the Codex app transcription endpoint with OAuth headers", () => + Effect.gen(function* () { + let observed = false; + const clientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.sync(() => { + observed = true; + expect(request.url).toBe(CODEX_TRANSCRIBE_URL); + expect(request.method).toBe("POST"); + expect(request.headers.authorization).toBe("Bearer oauth-token"); + expect(request.headers["chatgpt-account-id"]).toBe("account-123"); + expect(request.headers["x-openai-attach-auth"]).toBe("1"); + expect(request.headers["x-openai-attach-integrity-state"]).toBe("1"); + expect(request.headers["x-codex-base64"]).toBe("1"); + expect(request.headers.originator).toBe("Codex Desktop"); + expect(request.headers["user-agent"]).toContain("Chrome/136.0.0.0"); + expect(request.headers["oai-did"]).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u, + ); + expect(request.headers.origin).toBe("https://chatgpt.com"); + expect(request.body._tag).toBe("FormData"); + if (request.body._tag === "FormData") { + const file = request.body.formData.get("file"); + expect(file).toBeInstanceOf(Blob); + expect((file as Blob).type).toBe("audio/webm"); + } + return HttpClientResponse.fromWeb(request, Response.json({ text: "hello from voice" })); + }), + ), + ); + + const result = yield* transcribeCodexAudio({ + credentials: { accessToken: "oauth-token", accountId: "account-123" }, + audio: new Uint8Array([1, 2, 3]), + mimeType: "audio/webm", + }).pipe(Effect.provide(clientLayer)); + + expect(observed).toBe(true); + expect(result).toEqual({ text: "hello from voice" }); + }), +); diff --git a/apps/server/src/voice/VoiceTranscription.ts b/apps/server/src/voice/VoiceTranscription.ts new file mode 100644 index 00000000000..296a7cea93d --- /dev/null +++ b/apps/server/src/voice/VoiceTranscription.ts @@ -0,0 +1,164 @@ +import { + CodexSettings, + EnvironmentHttpBadRequestError, + EnvironmentHttpForbiddenError, + EnvironmentHttpInternalServerError, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { HttpBody, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; +import * as NodeCrypto from "node:crypto"; + +import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; +import { deriveProviderInstanceConfigMap } from "../provider/Layers/ProviderInstanceRegistryHydration.ts"; +import { ServerSettingsService } from "../serverSettings.ts"; + +const MAX_VOICE_AUDIO_BYTES = 25 * 1024 * 1024; +export const CODEX_TRANSCRIBE_URL = "https://chatgpt.com/backend-api/transcribe"; + +const CODEX_BROWSER_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"; +const CODEX_TRANSCRIPTION_HEADERS = { + "X-OpenAI-Attach-Auth": "1", + "X-OpenAI-Attach-Integrity-State": "1", + "x-codex-base64": "1", + originator: "Codex Desktop", + "User-Agent": CODEX_BROWSER_USER_AGENT, + "sec-ch-ua": '"Chromium";v="136", "Google Chrome";v="136", "Not=A?Brand";v="24"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"macOS"', + "oai-did": NodeCrypto.randomUUID(), + "OAI-Language": "en", + Origin: "https://chatgpt.com", + Referer: "https://chatgpt.com/", +} as const; + +const CodexVoiceAuth = Schema.Struct({ + auth_mode: Schema.Literal("chatgpt"), + tokens: Schema.Struct({ + access_token: Schema.NonEmptyString, + account_id: Schema.NonEmptyString, + }), +}); + +const CodexTranscriptionResponse = Schema.Struct({ + text: Schema.String, +}); +const decodeCodexVoiceAuth = Schema.decodeEffect(Schema.fromJsonString(CodexVoiceAuth)); +const decodeCodexSettings = Schema.decodeUnknownEffect(CodexSettings); + +interface CodexVoiceCredentials { + readonly accessToken: string; + readonly accountId: string; +} + +export const readCodexVoiceCredentials = Effect.fn("VoiceTranscription.readCodexVoiceCredentials")( + function* (config: CodexSettings) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const layout = yield* resolveCodexHomeLayout(config); + const authPath = path.join(layout.effectiveHomePath ?? layout.sharedHomePath, "auth.json"); + const encoded = yield* fileSystem.readFileString(authPath).pipe( + Effect.mapError( + () => + new EnvironmentHttpForbiddenError({ + message: "Codex OAuth credentials are unavailable.", + }), + ), + ); + const auth = yield* decodeCodexVoiceAuth(encoded).pipe( + Effect.mapError( + () => + new EnvironmentHttpForbiddenError({ + message: "Voice input requires a Codex ChatGPT OAuth login.", + }), + ), + ); + return { + accessToken: auth.tokens.access_token, + accountId: auth.tokens.account_id, + } satisfies CodexVoiceCredentials; + }, +); + +const extensionForMimeType = (mimeType: string): string => { + if (mimeType.includes("mp4")) return "m4a"; + if (mimeType.includes("ogg")) return "ogg"; + if (mimeType.includes("wav")) return "wav"; + return "webm"; +}; + +export const transcribeCodexAudio = Effect.fn("VoiceTranscription.transcribeCodexAudio")( + function* (input: { + readonly credentials: CodexVoiceCredentials; + readonly audio: Uint8Array; + readonly mimeType: string; + }) { + if (input.audio.byteLength === 0) { + return yield* new EnvironmentHttpBadRequestError({ message: "The recording is empty." }); + } + if (input.audio.byteLength > MAX_VOICE_AUDIO_BYTES) { + return yield* new EnvironmentHttpBadRequestError({ + message: "The recording exceeds the 25 MB voice input limit.", + }); + } + if (!/^(audio|video)\//u.test(input.mimeType)) { + return yield* new EnvironmentHttpBadRequestError({ + message: "The recording has an unsupported media type.", + }); + } + + const formData = new FormData(); + formData.append( + "file", + new Blob([input.audio], { type: input.mimeType }), + `recording.${extensionForMimeType(input.mimeType)}`, + ); + const httpClient = yield* HttpClient.HttpClient; + return yield* HttpClientRequest.post(CODEX_TRANSCRIBE_URL, { + body: HttpBody.formData(formData), + }).pipe( + HttpClientRequest.bearerToken(input.credentials.accessToken), + HttpClientRequest.setHeader("ChatGPT-Account-Id", input.credentials.accountId), + HttpClientRequest.setHeaders(CODEX_TRANSCRIPTION_HEADERS), + httpClient.execute, + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(CodexTranscriptionResponse)), + Effect.mapError( + () => new EnvironmentHttpInternalServerError({ message: "Voice transcription failed." }), + ), + ); + }, +); + +export const transcribeVoice = Effect.fn("VoiceTranscription.transcribe")(function* (input: { + readonly providerInstanceId: ProviderInstanceId; + readonly audio: Uint8Array; + readonly mimeType: string; +}) { + const settingsService = yield* ServerSettingsService; + const settings = yield* settingsService.getSettings.pipe( + Effect.mapError( + () => new EnvironmentHttpInternalServerError({ message: "Voice transcription failed." }), + ), + ); + const instance = deriveProviderInstanceConfigMap(settings)[input.providerInstanceId]; + if (!instance || instance.driver !== "codex") { + return yield* new EnvironmentHttpForbiddenError({ + message: "Voice input requires a Codex provider instance.", + }); + } + const config = yield* decodeCodexSettings(instance.config ?? {}).pipe( + Effect.mapError( + () => + new EnvironmentHttpForbiddenError({ + message: "The selected Codex provider configuration is invalid.", + }), + ), + ); + const credentials = yield* readCodexVoiceCredentials(config); + return yield* transcribeCodexAudio({ ...input, credentials }); +}); diff --git a/apps/server/src/voice/http.ts b/apps/server/src/voice/http.ts new file mode 100644 index 00000000000..663bee50a0b --- /dev/null +++ b/apps/server/src/voice/http.ts @@ -0,0 +1,17 @@ +import { AuthOrchestrationOperateScope, EnvironmentHttpApi } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; + +import { annotateEnvironmentRequest, requireEnvironmentScope } from "../auth/http.ts"; +import { transcribeVoice } from "./VoiceTranscription.ts"; + +export const voiceHttpApiLayer = HttpApiBuilder.group(EnvironmentHttpApi, "voice", (handlers) => + handlers.handle( + "transcribe", + Effect.fn("environment.voice.transcribe")(function* (args) { + yield* annotateEnvironmentRequest(args.endpoint.name); + yield* requireEnvironmentScope(AuthOrchestrationOperateScope); + return yield* transcribeVoice(args.payload); + }), + ), +); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 6ec2e631d19..d0870d69420 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -22,6 +22,7 @@ import { connectionStatusText, type EnvironmentConnectionPresentation, } from "@t3tools/client-runtime/connection"; +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; import { serializeComposerFileLink } from "@t3tools/shared/composerTrigger"; import { createModelSelection, normalizeModelSlug } from "@t3tools/shared/model"; import { @@ -73,6 +74,7 @@ import { type ComposerCommandItem, ComposerCommandMenu } from "./ComposerCommand import { ComposerPendingApprovalActions } from "./ComposerPendingApprovalActions"; import { CompactComposerControlsMenu } from "./CompactComposerControlsMenu"; import { ComposerPrimaryActions } from "./ComposerPrimaryActions"; +import { ComposerVoiceInput } from "./ComposerVoiceInput"; import { ComposerPendingApprovalPanel } from "./ComposerPendingApprovalPanel"; import { ComposerPendingUserInputPanel } from "./ComposerPendingUserInputPanel"; import { ComposerPlanFollowUpBanner } from "./ComposerPlanFollowUpBanner"; @@ -1554,6 +1556,23 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }; }, [composerCursor, composerTerminalContexts, promptRef]); + const insertVoiceTranscription = useCallback( + (transcription: string) => { + const snapshot = readComposerSnapshot(); + const insertionPoint = snapshot.expandedCursor; + const needsLeadingSpace = + insertionPoint > 0 && !/\s/u.test(snapshot.value[insertionPoint - 1] ?? ""); + const needsTrailingSpace = + insertionPoint < snapshot.value.length && !/\s/u.test(snapshot.value[insertionPoint] ?? ""); + applyPromptReplacement( + insertionPoint, + insertionPoint, + `${needsLeadingSpace ? " " : ""}${transcription}${needsTrailingSpace ? " " : ""}`, + ); + }, + [applyPromptReplacement, readComposerSnapshot], + ); + const resolveActiveComposerTrigger = useCallback((): { snapshot: { value: string; cursor: number; expandedCursor: number }; trigger: ComposerTrigger | null; @@ -2575,6 +2594,27 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) } className="flex shrink-0 flex-nowrap items-center justify-end gap-2" > + 0 || + projectSelectionRequired || + environmentUnavailable !== null + } + onTranscribed={insertVoiceTranscription} + /> { + const candidates = ["audio/webm;codecs=opus", "audio/mp4", "audio/ogg;codecs=opus"]; + return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)); +}; + +const errorMessage = (error: unknown): string => { + if (typeof error === "object" && error !== null) { + if ("message" in error && typeof error.message === "string") return error.message; + if ("cause" in error) return errorMessage(error.cause); + } + return "Voice transcription failed."; +}; + +export const ComposerVoiceInput = memo(function ComposerVoiceInput(props: { + providerInstanceId: ProviderInstanceId; + hasCodexOauth: boolean; + disabled: boolean; + onTranscribed: (text: string) => void; +}) { + const recorderRef = useRef(null); + const streamRef = useRef(null); + const operationRef = useRef(0); + const [state, setState] = useState<"idle" | "requesting" | "recording" | "transcribing">("idle"); + const captureSupported = + typeof navigator !== "undefined" && + navigator.mediaDevices?.getUserMedia !== undefined && + typeof MediaRecorder !== "undefined"; + const unavailableReason = !props.hasCodexOauth + ? "Voice input requires a Codex ChatGPT OAuth login" + : !captureSupported + ? "Voice input is not supported in this browser" + : props.disabled + ? "Voice input is unavailable while the composer is disabled" + : null; + + const releaseCapture = useCallback((stopRecorder = false) => { + const recorder = recorderRef.current; + if (recorder) { + recorder.ondataavailable = null; + recorder.onstop = null; + // oxlint-disable-next-line unicorn/prefer-add-event-listener -- this component owns the recorder and its single handlers + recorder.onerror = null; + if (stopRecorder && recorder.state !== "inactive") recorder.stop(); + } + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + recorderRef.current = null; + }, []); + + useEffect( + () => () => { + operationRef.current += 1; + releaseCapture(true); + }, + [releaseCapture], + ); + + const transcribe = async (audioBlob: Blob, operation: number) => { + setState("transcribing"); + try { + const audio = new Uint8Array(await audioBlob.arrayBuffer()); + const result = await runPrimaryHttp( + PrimaryEnvironmentHttpClient.pipe( + Effect.flatMap((client) => + client.voice.transcribe({ + payload: { + providerInstanceId: props.providerInstanceId, + audio, + mimeType: audioBlob.type || "audio/webm", + }, + headers: {}, + }), + ), + ), + ); + const text = result.text.trim(); + if (operation === operationRef.current && text.length > 0) props.onTranscribed(text); + } catch (error) { + if (operation === operationRef.current) { + toastManager.add({ type: "error", title: errorMessage(error) }); + } + } finally { + if (operation === operationRef.current) setState("idle"); + } + }; + + const stopRecording = () => { + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + }; + + const startRecording = async () => { + const operation = ++operationRef.current; + setState("requesting"); + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (operation !== operationRef.current) { + stream.getTracks().forEach((track) => track.stop()); + return; + } + streamRef.current = stream; + const mimeType = preferredRecorderMimeType(); + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + const chunks: Blob[] = []; + recorderRef.current = recorder; + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunks.push(event.data); + }; + recorder.onstop = () => { + const blob = new Blob(chunks, { type: recorder.mimeType || "audio/webm" }); + releaseCapture(); + void transcribe(blob, operation); + }; + // oxlint-disable-next-line unicorn/prefer-add-event-listener -- this component owns the recorder and its single handlers + recorder.onerror = () => { + releaseCapture(); + setState("idle"); + toastManager.add({ type: "error", title: "Microphone recording failed." }); + }; + recorder.start(); + setState("recording"); + } catch (error) { + if (operation !== operationRef.current) return; + releaseCapture(); + setState("idle"); + toastManager.add({ + type: "error", + title: + error instanceof DOMException && error.name === "NotAllowedError" + ? "Microphone access was denied." + : "Could not access the microphone.", + }); + } + }; + + const label = + state === "recording" + ? "Stop recording" + : state === "requesting" + ? "Starting voice input" + : state === "transcribing" + ? "Transcribing voice input" + : (unavailableReason ?? "Start voice input"); + + return ( + + void startRecording()} + /> + } + > + {state === "requesting" || state === "transcribing" ? ( + + {label} + + ); +}); diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 2d40dad60cc..93e66f51f26 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -25,6 +25,7 @@ import { ServerAuthSessionMethod, } from "./auth.ts"; import { AuthSessionId, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { ProviderInstanceId } from "./providerInstance.ts"; import { ExecutionEnvironmentDescriptor } from "./environment.ts"; import { ClientOrchestrationCommand, @@ -550,8 +551,27 @@ export class EnvironmentConnectHttpApi extends HttpApiGroup.make("connect") }), ) {} +export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice").add( + HttpApiEndpoint.post("transcribe", "/api/voice/transcribe", { + headers: OptionalBearerHeaders, + payload: Schema.Struct({ + providerInstanceId: ProviderInstanceId, + audio: Schema.Uint8ArrayFromBase64, + mimeType: TrimmedNonEmptyString, + }), + success: Schema.Struct({ text: Schema.String }), + error: [ + EnvironmentHttpBadRequestError, + EnvironmentHttpForbiddenError, + EnvironmentHttpInternalServerError, + EnvironmentScopeRequiredError, + ], + }).middleware(EnvironmentAuthenticatedAuth), +) {} + export class EnvironmentHttpApi extends HttpApi.make("environment") .add(EnvironmentMetadataHttpApi) .add(EnvironmentAuthHttpApi) .add(EnvironmentOrchestrationHttpApi) + .add(EnvironmentVoiceHttpApi) .add(EnvironmentConnectHttpApi) {}