feat: add Codex OAuth voice-to-text input

apps/server/src/server.ts

@@ -89,6 +89,7 @@ import {
   persistServerRuntimeState,
 } from "./serverRuntimeState.ts";
 import { orchestrationHttpApiLayer } from "./orchestration/http.ts";
+import { voiceHttpApiLayer } from "./voice/http.ts";
 import * as NetService from "@t3tools/shared/Net";
 import * as RelayClient from "@t3tools/shared/relayClient";
 import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale";
@@ -352,6 +353,7 @@ export const makeRoutesLayer = Layer.mergeAll(
       Layer.provide(authHttpApiLayer),
       Layer.provide(connectHttpApiLayer),
       Layer.provide(orchestrationHttpApiLayer),
+      Layer.provide(voiceHttpApiLayer),
       Layer.provide(serverEnvironmentHttpApiLayer),
       Layer.provide(environmentAuthenticatedAuthLayer),
     ),

apps/server/src/voice/VoiceTranscription.test.ts

@@ -0,0 +1,98 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { expect, it } from "@effect/vitest";
+import { CodexSettings } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+import { HttpClient, HttpClientResponse } from "effect/unstable/http";
+
+import {
+  CODEX_TRANSCRIBE_URL,
+  readCodexVoiceCredentials,
+  transcribeCodexAudio,
+} from "./VoiceTranscription.ts";
+
+const decodeCodexSettings = Schema.decodeSync(CodexSettings);
+
+it.layer(NodeServices.layer)("VoiceTranscription credentials", (it) => {
+  it.effect("reads ChatGPT OAuth credentials from the selected shadow home", () =>
+    Effect.gen(function* () {
+      const fileSystem = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const sharedHome = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-shared-" });
+      const shadowHome = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-shadow-" });
+      yield* fileSystem.writeFileString(
+        path.join(shadowHome, "auth.json"),
+        '{"auth_mode":"chatgpt","tokens":{"access_token":"oauth-token","account_id":"account-123"}}',
+      );
+
+      const credentials = yield* readCodexVoiceCredentials(
+        decodeCodexSettings({ homePath: sharedHome, shadowHomePath: shadowHome }),
+      );
+
+      expect(credentials).toEqual({ accessToken: "oauth-token", accountId: "account-123" });
+    }),
+  );
+
+  it.effect("rejects Codex API-key auth", () =>
+    Effect.gen(function* () {
+      const fileSystem = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const homePath = yield* fileSystem.makeTempDirectoryScoped({ prefix: "voice-api-key-" });
+      yield* fileSystem.writeFileString(
+        path.join(homePath, "auth.json"),
+        '{"auth_mode":"apikey","OPENAI_API_KEY":"secret"}',
+      );
+
+      const error = yield* readCodexVoiceCredentials(decodeCodexSettings({ homePath })).pipe(
+        Effect.flip,
+      );
+      expect(error._tag).toBe("EnvironmentHttpForbiddenError");
+    }),
+  );
+});
+
+it.effect("posts audio to the Codex app transcription endpoint with OAuth headers", () =>
+  Effect.gen(function* () {
+    let observed = false;
+    const clientLayer = Layer.succeed(
+      HttpClient.HttpClient,
+      HttpClient.make((request) =>
+        Effect.sync(() => {
+          observed = true;
+          expect(request.url).toBe(CODEX_TRANSCRIBE_URL);
+          expect(request.method).toBe("POST");
+          expect(request.headers.authorization).toBe("Bearer oauth-token");
+          expect(request.headers["chatgpt-account-id"]).toBe("account-123");
+          expect(request.headers["x-openai-attach-auth"]).toBe("1");
+          expect(request.headers["x-openai-attach-integrity-state"]).toBe("1");
+          expect(request.headers["x-codex-base64"]).toBe("1");
+          expect(request.headers.originator).toBe("Codex Desktop");
+          expect(request.headers["user-agent"]).toContain("Chrome/136.0.0.0");
+          expect(request.headers["oai-did"]).toMatch(
+            /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
+          );
+          expect(request.headers.origin).toBe("https://chatgpt.com");
+          expect(request.body._tag).toBe("FormData");
+          if (request.body._tag === "FormData") {
+            const file = request.body.formData.get("file");
+            expect(file).toBeInstanceOf(Blob);
+            expect((file as Blob).type).toBe("audio/webm");
+          }
+          return HttpClientResponse.fromWeb(request, Response.json({ text: "hello from voice" }));
+        }),
+      ),
+    );
+
+    const result = yield* transcribeCodexAudio({
+      credentials: { accessToken: "oauth-token", accountId: "account-123" },
+      audio: new Uint8Array([1, 2, 3]),
+      mimeType: "audio/webm",
+    }).pipe(Effect.provide(clientLayer));
+
+    expect(observed).toBe(true);
+    expect(result).toEqual({ text: "hello from voice" });
+  }),
+);

apps/server/src/voice/VoiceTranscription.ts

@@ -0,0 +1,164 @@
+import {
+  CodexSettings,
+  EnvironmentHttpBadRequestError,
+  EnvironmentHttpForbiddenError,
+  EnvironmentHttpInternalServerError,
+  type ProviderInstanceId,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+import { HttpBody, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
+import * as NodeCrypto from "node:crypto";
+
+import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts";
+import { deriveProviderInstanceConfigMap } from "../provider/Layers/ProviderInstanceRegistryHydration.ts";
+import { ServerSettingsService } from "../serverSettings.ts";
+
+const MAX_VOICE_AUDIO_BYTES = 25 * 1024 * 1024;
+export const CODEX_TRANSCRIBE_URL = "https://chatgpt.com/backend-api/transcribe";
+
+const CODEX_BROWSER_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36";
+const CODEX_TRANSCRIPTION_HEADERS = {
+  "X-OpenAI-Attach-Auth": "1",
+  "X-OpenAI-Attach-Integrity-State": "1",
+  "x-codex-base64": "1",
+  originator: "Codex Desktop",
+  "User-Agent": CODEX_BROWSER_USER_AGENT,
+  "sec-ch-ua": '"Chromium";v="136", "Google Chrome";v="136", "Not=A?Brand";v="24"',
+  "sec-ch-ua-mobile": "?0",
+  "sec-ch-ua-platform": '"macOS"',
+  "oai-did": NodeCrypto.randomUUID(),
+  "OAI-Language": "en",
+  Origin: "https://chatgpt.com",
+  Referer: "https://chatgpt.com/",
+} as const;
+
+const CodexVoiceAuth = Schema.Struct({
+  auth_mode: Schema.Literal("chatgpt"),
+  tokens: Schema.Struct({
+    access_token: Schema.NonEmptyString,
+    account_id: Schema.NonEmptyString,
+  }),
+});
+
+const CodexTranscriptionResponse = Schema.Struct({
+  text: Schema.String,
+});
+const decodeCodexVoiceAuth = Schema.decodeEffect(Schema.fromJsonString(CodexVoiceAuth));
+const decodeCodexSettings = Schema.decodeUnknownEffect(CodexSettings);
+
+interface CodexVoiceCredentials {
+  readonly accessToken: string;
+  readonly accountId: string;
+}
+
+export const readCodexVoiceCredentials = Effect.fn("VoiceTranscription.readCodexVoiceCredentials")(
+  function* (config: CodexSettings) {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    const layout = yield* resolveCodexHomeLayout(config);
+    const authPath = path.join(layout.effectiveHomePath ?? layout.sharedHomePath, "auth.json");
+    const encoded = yield* fileSystem.readFileString(authPath).pipe(
+      Effect.mapError(
+        () =>
+          new EnvironmentHttpForbiddenError({
+            message: "Codex OAuth credentials are unavailable.",
+          }),
+      ),
+    );
+    const auth = yield* decodeCodexVoiceAuth(encoded).pipe(
+      Effect.mapError(
+        () =>
+          new EnvironmentHttpForbiddenError({
+            message: "Voice input requires a Codex ChatGPT OAuth login.",
+          }),
+      ),
+    );
+    return {
+      accessToken: auth.tokens.access_token,
+      accountId: auth.tokens.account_id,
+    } satisfies CodexVoiceCredentials;
+  },
+);
+
+const extensionForMimeType = (mimeType: string): string => {
+  if (mimeType.includes("mp4")) return "m4a";
+  if (mimeType.includes("ogg")) return "ogg";
+  if (mimeType.includes("wav")) return "wav";
+  return "webm";
+};
+
+export const transcribeCodexAudio = Effect.fn("VoiceTranscription.transcribeCodexAudio")(
+  function* (input: {
+    readonly credentials: CodexVoiceCredentials;
+    readonly audio: Uint8Array;
+    readonly mimeType: string;
+  }) {
+    if (input.audio.byteLength === 0) {
+      return yield* new EnvironmentHttpBadRequestError({ message: "The recording is empty." });
+    }
+    if (input.audio.byteLength > MAX_VOICE_AUDIO_BYTES) {
+      return yield* new EnvironmentHttpBadRequestError({
+        message: "The recording exceeds the 25 MB voice input limit.",
+      });
+    }
+    if (!/^(audio|video)\//u.test(input.mimeType)) {
+      return yield* new EnvironmentHttpBadRequestError({
+        message: "The recording has an unsupported media type.",
+      });
+    }
+
+    const formData = new FormData();
+    formData.append(
+      "file",
+      new Blob([input.audio], { type: input.mimeType }),
+      `recording.${extensionForMimeType(input.mimeType)}`,
+    );
+    const httpClient = yield* HttpClient.HttpClient;
+    return yield* HttpClientRequest.post(CODEX_TRANSCRIBE_URL, {
+      body: HttpBody.formData(formData),
+    }).pipe(
+      HttpClientRequest.bearerToken(input.credentials.accessToken),
+      HttpClientRequest.setHeader("ChatGPT-Account-Id", input.credentials.accountId),
+      HttpClientRequest.setHeaders(CODEX_TRANSCRIPTION_HEADERS),
+      httpClient.execute,
+      Effect.flatMap(HttpClientResponse.filterStatusOk),
+      Effect.flatMap(HttpClientResponse.schemaBodyJson(CodexTranscriptionResponse)),
+      Effect.mapError(
+        () => new EnvironmentHttpInternalServerError({ message: "Voice transcription failed." }),
+      ),
+    );
+  },
+);
+
+export const transcribeVoice = Effect.fn("VoiceTranscription.transcribe")(function* (input: {
+  readonly providerInstanceId: ProviderInstanceId;
+  readonly audio: Uint8Array;
+  readonly mimeType: string;
+}) {
+  const settingsService = yield* ServerSettingsService;
+  const settings = yield* settingsService.getSettings.pipe(
+    Effect.mapError(
+      () => new EnvironmentHttpInternalServerError({ message: "Voice transcription failed." }),
+    ),
+  );
+  const instance = deriveProviderInstanceConfigMap(settings)[input.providerInstanceId];
+  if (!instance || instance.driver !== "codex") {
+    return yield* new EnvironmentHttpForbiddenError({
+      message: "Voice input requires a Codex provider instance.",
+    });
+  }
+  const config = yield* decodeCodexSettings(instance.config ?? {}).pipe(
+    Effect.mapError(
+      () =>
+        new EnvironmentHttpForbiddenError({
+          message: "The selected Codex provider configuration is invalid.",
+        }),
+    ),
+  );
+  const credentials = yield* readCodexVoiceCredentials(config);
+  return yield* transcribeCodexAudio({ ...input, credentials });
+});

apps/server/src/voice/http.ts

@@ -0,0 +1,17 @@
+import { AuthOrchestrationOperateScope, EnvironmentHttpApi } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
+
+import { annotateEnvironmentRequest, requireEnvironmentScope } from "../auth/http.ts";
+import { transcribeVoice } from "./VoiceTranscription.ts";
+
+export const voiceHttpApiLayer = HttpApiBuilder.group(EnvironmentHttpApi, "voice", (handlers) =>
+  handlers.handle(
+    "transcribe",
+    Effect.fn("environment.voice.transcribe")(function* (args) {
+      yield* annotateEnvironmentRequest(args.endpoint.name);
+      yield* requireEnvironmentScope(AuthOrchestrationOperateScope);
+      return yield* transcribeVoice(args.payload);
+    }),
+  ),
+);

apps/web/src/components/chat/ChatComposer.tsx

@@ -22,6 +22,7 @@ import {
   connectionStatusText,
   type EnvironmentConnectionPresentation,
 } from "@t3tools/client-runtime/connection";
+import { scopedThreadKey } from "@t3tools/client-runtime/environment";
 import { serializeComposerFileLink } from "@t3tools/shared/composerTrigger";
 import { createModelSelection, normalizeModelSlug } from "@t3tools/shared/model";
 import {
@@ -73,6 +74,7 @@ import { type ComposerCommandItem, ComposerCommandMenu } from "./ComposerCommand
 import { ComposerPendingApprovalActions } from "./ComposerPendingApprovalActions";
 import { CompactComposerControlsMenu } from "./CompactComposerControlsMenu";
 import { ComposerPrimaryActions } from "./ComposerPrimaryActions";
+import { ComposerVoiceInput } from "./ComposerVoiceInput";
 import { ComposerPendingApprovalPanel } from "./ComposerPendingApprovalPanel";
 import { ComposerPendingUserInputPanel } from "./ComposerPendingUserInputPanel";
 import { ComposerPlanFollowUpBanner } from "./ComposerPlanFollowUpBanner";
@@ -1554,6 +1556,23 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
     };
   }, [composerCursor, composerTerminalContexts, promptRef]);
 
+  const insertVoiceTranscription = useCallback(
+    (transcription: string) => {
+      const snapshot = readComposerSnapshot();
+      const insertionPoint = snapshot.expandedCursor;
+      const needsLeadingSpace =
+        insertionPoint > 0 && !/\s/u.test(snapshot.value[insertionPoint - 1] ?? "");
+      const needsTrailingSpace =
+        insertionPoint < snapshot.value.length && !/\s/u.test(snapshot.value[insertionPoint] ?? "");
+      applyPromptReplacement(
+        insertionPoint,
+        insertionPoint,
+        `${needsLeadingSpace ? " " : ""}${transcription}${needsTrailingSpace ? " " : ""}`,
+      );
+    },
+    [applyPromptReplacement, readComposerSnapshot],
+  );
+
   const resolveActiveComposerTrigger = useCallback((): {
     snapshot: { value: string; cursor: number; expandedCursor: number };
     trigger: ComposerTrigger | null;
@@ -2575,6 +2594,27 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
                 }
                 className="flex shrink-0 flex-nowrap items-center justify-end gap-2"
               >
+                <ComposerVoiceInput
+                  key={
+                    typeof composerDraftTarget === "string"
+                      ? composerDraftTarget
+                      : scopedThreadKey(composerDraftTarget)
+                  }
+                  providerInstanceId={selectedInstanceId}
+                  hasCodexOauth={
+                    selectedProviderEntry?.driverKind === "codex" &&
+                    selectedProviderStatus?.auth.type === "chatgpt" &&
+                    selectedProviderStatus.auth.status === "authenticated"
+                  }
+                  disabled={
+                    isConnecting ||
+                    isComposerApprovalState ||
+                    pendingUserInputs.length > 0 ||
+                    projectSelectionRequired ||
+                    environmentUnavailable !== null
+                  }
+                  onTranscribed={insertVoiceTranscription}
+                />
                 <ComposerFooterPrimaryActions
                   compact={isComposerPrimaryActionsCompact}
                   activeContextWindow={activeContextWindow}