From 90bb7eaa659a9025bb5cca182b1708dd0daf8d57 Mon Sep 17 00:00:00 2001 From: AssemblyAI Date: Thu, 5 Mar 2026 09:44:29 -0500 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: e8d8085c215c2290078d83824da5b75650298877 --- package.json | 2 +- src/services/streaming/service.ts | 13 +++++++ src/types/streaming/index.ts | 22 ++++++++++++ tests/integration/streaming.test.ts | 1 + tests/unit/streaming.test.ts | 56 +++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5b772b0..c9f139e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.26.0", + "version": "4.26.1", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.", "engines": { "node": ">=18" diff --git a/src/services/streaming/service.ts b/src/services/streaming/service.ts index e332186..f12a111 100644 --- a/src/services/streaming/service.ts +++ b/src/services/streaming/service.ts @@ -13,6 +13,7 @@ import { BeginEvent, StreamingEventMessage, TurnEvent, + LLMGatewayResponseEvent, StreamingUpdateConfiguration, StreamingForceEndpoint, } from "../.."; @@ -171,6 +172,10 @@ export class StreamingTranscriber { searchParams.set("max_speakers", this.params.maxSpeakers.toString()); } + if (this.params.llmGateway !== undefined) { + searchParams.set("llm_gateway", JSON.stringify(this.params.llmGateway)); + } + url.search = searchParams.toString(); return url; @@ -178,6 +183,10 @@ export class StreamingTranscriber { on(event: "open", listener: (event: BeginEvent) => void): void; on(event: "turn", listener: (event: TurnEvent) => void): void; + on( + event: "llmGatewayResponse", + listener: (event: LLMGatewayResponseEvent) => void, + ): void; on(event: "error", listener: (error: Error) => void): void; on(event: "close", listener: (code: number, reason: string) => void): void; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -248,6 +257,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.listeners.speechStarted?.(message); break; } + case "LLMGatewayResponse": { + this.listeners.llmGatewayResponse?.(message); + break; + } case "Termination": { this.sessionTerminatedResolve?.(); break; diff --git a/src/types/streaming/index.ts b/src/types/streaming/index.ts index 31a13de..16f5d88 100644 --- a/src/types/streaming/index.ts +++ b/src/types/streaming/index.ts @@ -1,5 +1,16 @@ import { AudioEncoding } from ".."; +export type LLMGatewayMessage = { + role: string; + content: string; +}; + +export type LLMGatewayConfig = { + model: string; + messages: LLMGatewayMessage[]; + max_tokens: number; +}; + export type StreamingTranscriberParams = { websocketBaseUrl?: string; apiKey?: string; @@ -24,6 +35,7 @@ export type StreamingTranscriberParams = { inactivityTimeout?: number; speakerLabels?: boolean; maxSpeakers?: number; + llmGateway?: LLMGatewayConfig; }; export type StreamingEvents = @@ -31,6 +43,7 @@ export type StreamingEvents = | "close" | "turn" | "speechStarted" + | "llmGatewayResponse" | "error"; export type StreamingListeners = { @@ -38,6 +51,7 @@ export type StreamingListeners = { close?: (code: number, reason: string) => void; turn?: (event: TurnEvent) => void; speechStarted?: (event: SpeechStartedEvent) => void; + llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void; error?: (error: Error) => void; }; @@ -125,11 +139,19 @@ export type ErrorEvent = { error: string; }; +export type LLMGatewayResponseEvent = { + type: "LLMGatewayResponse"; + turn_order: number; + transcript: string; + data: unknown; +}; + export type StreamingEventMessage = | BeginEvent | TurnEvent | SpeechStartedEvent | TerminationEvent + | LLMGatewayResponseEvent | ErrorEvent; export type StreamingOperationMessage = diff --git a/tests/integration/streaming.test.ts b/tests/integration/streaming.test.ts index 4926f88..5d6d863 100644 --- a/tests/integration/streaming.test.ts +++ b/tests/integration/streaming.test.ts @@ -73,6 +73,7 @@ async function createStreamingTranscriber(useToken: boolean) { const serviceParams: StreamingTranscriberParams = { websocketBaseUrl: process.env.ASSEMBLYAI_STREAMING_WS_API_HOST, sampleRate: 16_000, + speechModel: "universal-streaming-english", apiKey: useToken ? undefined : process.env.ASSEMBLYAI_API_KEY, token: useToken ? await client.streaming.createTemporaryToken({ diff --git a/tests/unit/streaming.test.ts b/tests/unit/streaming.test.ts index f0a2885..e01b99c 100644 --- a/tests/unit/streaming.test.ts +++ b/tests/unit/streaming.test.ts @@ -134,4 +134,60 @@ describe("streaming", () => { const turn = await turnPromise; expect(turn.speaker_label).toBe("A"); }); + + it("should include llm_gateway in connection URL", async () => { + await cleanup(); + WS.clean(); + + const llmGatewayConfig = { + model: "claude-3-5-sonnet", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello" }, + ], + max_tokens: 100, + }; + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&speech_model=universal-streaming-english&llm_gateway=${encodeURIComponent(JSON.stringify(llmGatewayConfig))}`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + speechModel: "universal-streaming-english", + llmGateway: llmGatewayConfig, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + }); + + it("should parse LLMGatewayResponse event", async () => { + const llmResponsePromise = new Promise<{ + turn_order: number; + transcript: string; + data: unknown; + }>((resolve) => { + rt.on("llmGatewayResponse", (event) => resolve(event)); + }); + + const llmResponseData = { + type: "LLMGatewayResponse", + turn_order: 1, + transcript: "hello world", + data: { + response: "This is an LLM response", + model: "claude-3-5-sonnet", + }, + }; + + server.send(JSON.stringify(llmResponseData)); + const response = await llmResponsePromise; + expect(response.turn_order).toBe(1); + expect(response.transcript).toBe("hello world"); + expect(response.data).toEqual({ + response: "This is an LLM response", + model: "claude-3-5-sonnet", + }); + }); });