From 6f303a3c13344b2e43fe634017483db648fd9568 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 14:53:35 +0200 Subject: [PATCH 01/67] add realtime text to speech client, WIP --- orchestration/pom.xml | 5 + .../orchestration/OrchestrationClient.java | 165 ++++++++++++++- .../orchestration/RealtimeClientFactory.java | 200 ++++++++++++++++++ .../TextToSpeechRealtimeClient.java | 53 +++++ .../WSSOpenAIRealtimeClient.java | 157 ++++++++++++++ .../orchestration/WebsocketListenerGlue.java | 31 +++ .../java/com/sap/ai/sdk/app/Application.java | 25 ++- .../app/services/OrchestrationService.java | 9 + 8 files changed, 640 insertions(+), 5 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java diff --git a/orchestration/pom.xml b/orchestration/pom.xml index f8027fe4a..a1ef764f1 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,6 +122,11 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml + + com.openai + openai-java + 4.41.0 + org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 73c7161e0..4e901e966 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,6 +7,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.models.beta.realtime.sessions.Session; +import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; @@ -17,16 +21,27 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import io.vavr.collection.Array; import io.vavr.control.Try; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -34,9 +49,12 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private final OrchestrationHttpExecutor executor; private final List
customHeaders = new ArrayList<>(); @@ -103,6 +121,147 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream speech(String input) { + + var deploymentID = ""; + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer .......") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(REALTIME_API_PATH + deploymentID), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ +{ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } +} +"""; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java new file mode 100644 index 000000000..d48dd7b9f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java @@ -0,0 +1,200 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.responses.Response; +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Stream; + +@Slf4j +@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) +public class RealtimeClientFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + private final Supplier deploymentIdSupplier; + + /** + * Voices input text into output audio + * + * @param input text to voice + * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + */ + public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var deploymentId = deploymentIdSupplier.get(); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + sb.append(destination.getUri().getHost()); + // set port explicitly if defined in the uri + if (destination.getUri().getPort() != -1) { + sb.append(":").append(destination.getUri().getPort()); + } + sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); + var endpoint = sb.toString(); + + + final List result = new ArrayList<>(); + try { + HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable + WebSocket ws = + client + .newWebSocketBuilder() + .header( + "Authorization", + "Bearer ....") + .header("AI-Resource-Group", "default") + .buildAsync( + URI.create(endpoint), + new WebSocket.Listener() { + + @Override + public void onOpen(WebSocket webSocket) { + log.warn("Websocket connection opened"); + String sessionUpdateJson = + """ + { + "type": "session.update", + "session": { + "type": "realtime", + "output_modalities": ["audio"], + "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", + "audio": { + "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, + "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } + } + } + } + """; + + // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": + // {\"modalities\": [\"audio\", \"text\"]}}"; + webSocket.sendText(sessionUpdateJson, true); + webSocket.request(1); + + String textMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "system", + "content": [ + { + "type": "input_text", + "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" + } + ] + } + } + """; + //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input + log.warn("Sending message to websocket: {}", textMessage); + webSocket.sendText(textMessage, true); + webSocket.request(1); + + String voiceMessage = + """ + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Eat more of these fresh french baguettes" + } + ] + } + } + """; + + webSocket.sendText(voiceMessage, true); + webSocket.request(1); + + + + + String askForResponse = "{ \"type\": \"response.create\" }"; + webSocket.sendText(askForResponse, true); + + webSocket.request(80); + } + + @Override + public CompletionStage onText( + WebSocket webSocket, CharSequence data, boolean last) { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + log.warn("text received: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + Response r = null; + + String type = event.get("type").asText(); + if ("response.output_audio.delta".equals(type)) { + String base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + result.add(audio); + } else if ("response.audio_transcript.delta".equals(type)) { + log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); + } else if ("response.output_audio.done".equals(type)) { + // TODO: we should not wait last ack, we should return stream immediately, + // this requires + // TODO: thread management, factory and thread safety of all structs + webSocket.sendClose(1000, "done"); + return null; + } else { + log.warn("Unhandled event type: {}", data); + } + return CompletableFuture.completedStage(null); + } + }) + .join(); + Thread.sleep(25000); + return result.stream(); + } catch (Exception e) { + //log.error(e.getMessage(), e); + throw new OrchestrationClientException("Failed to call realtime API", e); + } + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..7df1881da --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -0,0 +1,53 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.RealtimeSessionCreateRequest; +import io.vavr.collection.Array; + +import java.util.*; +import java.util.function.BiConsumer; + +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final BiConsumer outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java new file mode 100644 index 000000000..588e5853b --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -0,0 +1,157 @@ +package com.sap.ai.sdk.orchestration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; + +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + +@Slf4j +@AllArgsConstructor(access = AccessLevel.PACKAGE) +public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { + + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider + * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, + * this is normally not configurable and would leave realtime conversation unusable on a pause. + *

+ * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds + * before it gets terminated + *

+ * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required + * to prevent unintended closing of connections while conversation is still used + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); + } + this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + } + + public synchronized void askForResponse() { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); + ws.request(1); + } + + @Override + public synchronized void close() { + WebSocket ws = this.ws.join(); + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + this.client.close(); + } + + protected abstract String getSystemPrompt(); + + protected abstract List getOutputModalities(); + + protected abstract void onResponse(String eventType, JsonNode event); + + protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + WebSocket ws = this.ws.join(); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException( + "Error parsing JSON response from speech API", e); + } + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } + + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; + } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } + + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = SessionUpdateEvent.builder() + .session(ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(getOutputModalities()) + .audio(RealtimeAudioConfig.builder() + .input(RealtimeAudioConfigInput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .build()) + .output(RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + ws.sendText(JACKSON.valueToTree(sue), true); + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var message = new RealtimeConversationItemSystemMessage.Builder(). + addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); + ws.sendText(JACKSON.valueToTree(message), true); + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = new TimerTask(){ + @Override + public void run() { + sendPing(ws); + } + + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java new file mode 100644 index 000000000..28bb102ec --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java @@ -0,0 +1,31 @@ +package com.sap.ai.sdk.orchestration; + +import java.net.http.WebSocket; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class WebsocketListenerGlue implements WebSocket.Listener { + + private final Consumer onOpen; + private final BiConsumer onText; + + public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + } + + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + this.onText.accept(webSocket, data); + webSocket.request(1); + return CompletableFuture.completedStage(null); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 0f4ed8348..01269478d 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.app.services.OrchestrationService; +import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -13,6 +16,10 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; +import java.io.*; +import java.nio.file.Files; +import java.util.stream.Collectors; + /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -37,7 +44,21 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) { - SpringApplication.run(Application.class, args); + public static void main(final String[] args) throws IOException { + var ctx = SpringApplication.run(Application.class, args); + var svc = ctx.getBean(OrchestrationService.class); + var res = svc.speech("Eat more of these fresh french baguettes").toList(); + String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + File file = new File(path); + try (FileOutputStream in = new FileOutputStream(file)) { + var ttl = 0; + for (byte[] chunk: res) { + System.out.println("Writing bytes: " + chunk.length); + in.write(chunk); + ttl += chunk.length; + } + System.out.println("Written bytes: " + ttl); + } + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index bf8d82b9d..4ef188372 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -82,6 +82,15 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } + /** + * TODO: document + * @param input + * @return + */ + public Stream speech(@Nonnull final String input) { + return client.speech(input); + } + /** * Chat request to OpenAI through the Orchestration service with an image. * From ecd953e472948e64b9ccd96176ea7ede8f7142a5 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 19:42:08 +0200 Subject: [PATCH 02/67] polish minimap text-to-speech realtime poc --- .../foundationmodels/openai/OpenAiModel.java | 2 +- .../orchestration/OrchestrationAiModel.java | 3 + .../orchestration/OrchestrationClient.java | 152 ++----------- .../orchestration/RealtimeChannelFactory.java | 58 +++++ .../orchestration/RealtimeClientFactory.java | 200 ------------------ .../sdk/orchestration/TextInputChannel.java | 5 + .../TextToSpeechRealtimeClient.java | 22 +- .../WSSOpenAIRealtimeClient.java | 121 +++++++---- .../java/com/sap/ai/sdk/app/Application.java | 28 ++- .../app/services/OrchestrationService.java | 34 +-- 10 files changed, 225 insertions(+), 400 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java index 1926f56ce..62c435bcc 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java @@ -121,7 +121,7 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem /** Azure OpenAI GPT-5-nano model */ public static final OpenAiModel GPT_5_NANO = new OpenAiModel("gpt-5-nano", null); - /** Azure OpenAI GPT-5-nano model */ + /** Azure OpenAI GPT-realtime model */ public static final OpenAiModel GPT_REALTIME = new OpenAiModel("gpt-realtime", null); /** Azure OpenAI GPT-5.2 model */ diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java index 06b6beb08..1c6cdf7b3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java @@ -330,6 +330,9 @@ public class OrchestrationAiModel { /** Azure OpenAI GPT-5-mini model */ public static final OrchestrationAiModel GPT_5_MINI = new OrchestrationAiModel("gpt-5-mini"); + /** Azure OpenAI GPT-realtime model */ + public static final OrchestrationAiModel GPT_REALTIME = new OrchestrationAiModel("gpt-realtime"); + /** Azure OpenAI GPT-5-nano model */ public static final OrchestrationAiModel GPT_5_NANO = new OrchestrationAiModel("gpt-5-nano"); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 4e901e966..ad082065d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -12,6 +12,7 @@ import com.openai.models.beta.realtime.sessions.Session; import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; +import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -32,6 +33,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; @@ -42,6 +44,8 @@ import lombok.val; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -124,142 +128,26 @@ public OrchestrationChatResponse chatCompletion( /** * Voices input text into output audio * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(String input) { - - var deploymentID = ""; - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer .......") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(REALTIME_API_PATH + deploymentID), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ -{ - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } -} -"""; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - webSocket.request(80); - } + var factory = new RealtimeChannelFactory(() -> destination); - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } + return factory.textToSpeech(output); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java new file mode 100644 index 000000000..cffba0e1f --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -0,0 +1,58 @@ +package com.sap.ai.sdk.orchestration; + +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; +import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.Nonnull; +import java.util.*; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +public class RealtimeChannelFactory { + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + + private static final String REALTIME_API_PATH = + "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and + * Boolean - marker if current audio reply has been finished (current response ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java deleted file mode 100644 index d48dd7b9f..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeClientFactory.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.responses.Response; -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Flow; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Stream; - -@Slf4j -@RequiredArgsConstructor(access = lombok.AccessLevel.PRIVATE) -public class RealtimeClientFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - - private final Supplier destinationSupplier; - private final Supplier deploymentIdSupplier; - - /** - * Voices input text into output audio - * - * @param input text to voice - * @return byte chunks output stream in PCM16 format, 16-bit, mono, 24000 Hz, little-endian - */ - public Stream textToSpeech(@Nonnull final Consumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var deploymentId = deploymentIdSupplier.get(); - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - sb.append(destination.getUri().getHost()); - // set port explicitly if defined in the uri - if (destination.getUri().getPort() != -1) { - sb.append(":").append(destination.getUri().getPort()); - } - sb.append(String.format(REALTIME_API_PATH_SUFFIX, deploymentId)); - var endpoint = sb.toString(); - - - final List result = new ArrayList<>(); - try { - HttpClient client = HttpClient.newHttpClient(); // TODO: fix local setup and use autocloseable - WebSocket ws = - client - .newWebSocketBuilder() - .header( - "Authorization", - "Bearer ....") - .header("AI-Resource-Group", "default") - .buildAsync( - URI.create(endpoint), - new WebSocket.Listener() { - - @Override - public void onOpen(WebSocket webSocket) { - log.warn("Websocket connection opened"); - String sessionUpdateJson = - """ - { - "type": "session.update", - "session": { - "type": "realtime", - "output_modalities": ["audio"], - "instructions": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input", - "audio": { - "input": { "format": { "type": "audio/pcm", "rate": 24000 } }, - "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "voice": "alloy" } - } - } - } - """; - - // String sessionUpdateJson = "{\"type\": \"session.update\", \"session\": - // {\"modalities\": [\"audio\", \"text\"]}}"; - webSocket.sendText(sessionUpdateJson, true); - webSocket.request(1); - - String textMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "system", - "content": [ - { - "type": "input_text", - "text": "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input" - } - ] - } - } - """; - //.replaceFirst("%VALUE%", input); // TODO: sanitize or/and escape input - log.warn("Sending message to websocket: {}", textMessage); - webSocket.sendText(textMessage, true); - webSocket.request(1); - - String voiceMessage = - """ - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Eat more of these fresh french baguettes" - } - ] - } - } - """; - - webSocket.sendText(voiceMessage, true); - webSocket.request(1); - - - - - String askForResponse = "{ \"type\": \"response.create\" }"; - webSocket.sendText(askForResponse, true); - - webSocket.request(80); - } - - @Override - public CompletionStage onText( - WebSocket webSocket, CharSequence data, boolean last) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - log.warn("text received: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - Response r = null; - - String type = event.get("type").asText(); - if ("response.output_audio.delta".equals(type)) { - String base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - result.add(audio); - } else if ("response.audio_transcript.delta".equals(type)) { - log.warn("RECEIVED TRANSCRIPT: {}", event.get("delta").asText()); - } else if ("response.output_audio.done".equals(type)) { - // TODO: we should not wait last ack, we should return stream immediately, - // this requires - // TODO: thread management, factory and thread safety of all structs - webSocket.sendClose(1000, "done"); - return null; - } else { - log.warn("Unhandled event type: {}", data); - } - return CompletableFuture.completedStage(null); - } - }) - .join(); - Thread.sleep(25000); - return result.stream(); - } catch (Exception e) { - //log.error(e.getMessage(), e); - throw new OrchestrationClientException("Failed to call realtime API", e); - } - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java new file mode 100644 index 000000000..20e6fcc66 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -0,0 +1,5 @@ +package com.sap.ai.sdk.orchestration; + +public interface TextInputChannel extends AutoCloseable { + void sendText(String text); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 7df1881da..45ea9335a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -1,13 +1,14 @@ package com.sap.ai.sdk.orchestration; import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.RealtimeSessionCreateRequest; -import io.vavr.collection.Array; +import com.openai.models.realtime.*; +import lombok.extern.slf4j.Slf4j; import java.util.*; import java.util.function.BiConsumer; -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient { +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -30,6 +31,18 @@ public TextToSpeechRealtimeClient(String url, Map httpHeaders, B this.outputConsumer = outputConsumer; } + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + } + @Override protected String getSystemPrompt() { return TASK; @@ -46,8 +59,11 @@ protected void onResponse(String eventType, JsonNode event) { var base64Audio = event.get("delta").asText(); byte[] audio = Base64.getDecoder().decode(base64Audio); this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); } else if ("response.output_audio.done".equals(eventType)) { this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 588e5853b..8770baf5f 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,12 +1,16 @@ package com.sap.ai.sdk.orchestration; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.conversations.Message; +import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import java.net.URI; @@ -19,11 +23,12 @@ import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; @Slf4j -@AllArgsConstructor(access = AccessLevel.PACKAGE) +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); + private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider @@ -34,7 +39,7 @@ public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { * before it gets terminated *

* In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still used + * to prevent unintended closing of connections while conversation is still expected to continue */ private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; @@ -55,22 +60,30 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } - public synchronized void askForResponse() { + public void askForResponse() { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(ResponseCreateEvent.builder().build()), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); + } } @Override - public synchronized void close() { + public void close() { WebSocket ws = this.ws.join(); - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } - this.client.close(); } protected abstract String getSystemPrompt(); @@ -79,10 +92,19 @@ public synchronized void close() { protected abstract void onResponse(String eventType, JsonNode event); - protected synchronized void sendMessage(RealtimeConversationItemUserMessage message) { + protected void sendMessage(ConversationItemCreateEvent message) { WebSocket ws = this.ws.join(); - ws.sendText(JACKSON.valueToTree(message), true); - ws.request(1); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } + + } } private synchronized void onSocketOpen(WebSocket ws) { @@ -92,6 +114,7 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); @@ -116,31 +139,57 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = SessionUpdateEvent.builder() - .session(ClientSecretCreateParams.Session.ofRealtime( + SessionUpdateEvent sue = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio(RealtimeAudioConfig.builder() - .input(RealtimeAudioConfigInput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .build()) - .output(RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice(RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - ws.sendText(JACKSON.valueToTree(sue), true); + .outputModalities(getOutputModalities()) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm.builder().build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); + } + ws.request(1); } private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); - var message = new RealtimeConversationItemSystemMessage.Builder(). - addContent(RealtimeConversationItemSystemMessage.Content.builder().text(systemPrompt).build()); - ws.sendText(JACKSON.valueToTree(message), true); + var systemConversationItem = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent(RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } ws.request(1); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 01269478d..aced5c87e 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -18,6 +18,8 @@ import java.io.*; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; /** Main class to start the Spring Boot application. */ @@ -44,15 +46,35 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws IOException { + public static void main(final String[] args) throws Exception { var ctx = SpringApplication.run(Application.class, args); var svc = ctx.getBean(OrchestrationService.class); - var res = svc.speech("Eat more of these fresh french baguettes").toList(); + var outputBytes = new ArrayList(); + var done = new AtomicBoolean(false); + try (var channel = svc.textToSpeech((bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { + System.err.println("BEFORE SENDING TEXT"); + channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); + System.err.println("AFTER SENDING TEXT"); + long timeout = System.currentTimeMillis() + 20000; + while (!done.get() && System.currentTimeMillis() < timeout) { + Thread.yield(); + System.err.println("Waiting for messages..."); + Thread.sleep(500); + } + } + + //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: res) { + for (byte[] chunk: outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 4ef188372..49fbaa594 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -10,26 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.orchestration.AzureContentFilter; -import com.sap.ai.sdk.orchestration.AzureFilterThreshold; -import com.sap.ai.sdk.orchestration.DpiMasking; -import com.sap.ai.sdk.orchestration.Grounding; -import com.sap.ai.sdk.orchestration.ImageItem; -import com.sap.ai.sdk.orchestration.LlamaGuardFilter; -import com.sap.ai.sdk.orchestration.Message; -import com.sap.ai.sdk.orchestration.OrchestrationAiModel; -import com.sap.ai.sdk.orchestration.OrchestrationChatResponse; -import com.sap.ai.sdk.orchestration.OrchestrationClient; -import com.sap.ai.sdk.orchestration.OrchestrationClientException; -import com.sap.ai.sdk.orchestration.OrchestrationConfigReference; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingRequest; -import com.sap.ai.sdk.orchestration.OrchestrationEmbeddingResponse; -import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; -import com.sap.ai.sdk.orchestration.OrchestrationPrompt; -import com.sap.ai.sdk.orchestration.ResponseJsonSchema; -import com.sap.ai.sdk.orchestration.SystemMessage; -import com.sap.ai.sdk.orchestration.TemplateConfig; -import com.sap.ai.sdk.orchestration.TranslationConfig; +import com.sap.ai.sdk.orchestration.*; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.DataRepositoryType; import com.sap.ai.sdk.orchestration.model.DocumentGroundingFilter; @@ -52,6 +33,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -83,12 +65,14 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) } /** - * TODO: document - * @param input - * @return + * Voices input text into output audio + * + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, + * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore */ - public Stream speech(@Nonnull final String input) { - return client.speech(input); + public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { + return client.textToSpeech(outputConsumer); } /** From 71d4a4cc2b5cd9db00edaf14546030dbdd0e7938 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:03:22 +0200 Subject: [PATCH 03/67] add incomplete interim packets buffering and glueing --- ...{WebsocketListenerGlue.java => BufferedWebSocketListener.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename orchestration/src/main/java/com/sap/ai/sdk/orchestration/{WebsocketListenerGlue.java => BufferedWebSocketListener.java} (100%) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java similarity index 100% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WebsocketListenerGlue.java rename to orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java From 232725d79a04f8b1b162b41b97a01cbca4da1eb0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 7 Jul 2026 20:06:44 +0200 Subject: [PATCH 04/67] remove obsolete debug constants --- .../orchestration/BufferedWebSocketListener.java | 14 +++++++++++--- .../ai/sdk/orchestration/OrchestrationClient.java | 2 -- .../sdk/orchestration/RealtimeChannelFactory.java | 5 ----- .../sdk/orchestration/WSSOpenAIRealtimeClient.java | 5 ++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 28bb102ec..118b2d5ce 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -6,14 +6,16 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -public class WebsocketListenerGlue implements WebSocket.Listener { +public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; private final BiConsumer onText; + private final StringBuilder buffer; - public WebsocketListenerGlue(Consumer onOpen, BiConsumer onText) { + public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); } @Override @@ -24,8 +26,14 @@ public void onOpen(WebSocket webSocket) { @Override public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - this.onText.accept(webSocket, data); + buffer.append(data); webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); + } + return CompletableFuture.completedStage(null); } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index ad082065d..7c52d772b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -53,8 +53,6 @@ public class OrchestrationClient { private static final String DEFAULT_SCENARIO = "orchestration"; private static final String COMPLETION_ENDPOINT = "/v2/completion"; - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index cffba0e1f..28edd05c4 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -16,13 +16,8 @@ public class RealtimeChannelFactory { private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final String REALTIME_API_PATH_SUFFIX = "/v2/inference/deployments/%s/v1/realtime"; private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private static final String REALTIME_API_PATH = - "wss://realtime.ai.intprod-eu12.eu-central-1.aws.ml.hana.ondemand.com/v2/inference/deployments/ddb5a959fdb51ffe/v1/realtime"; - private final Supplier destinationSupplier; /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 8770baf5f..678bff93a 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -5,8 +5,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.conversations.Message; -import com.openai.models.conversations.TextContent; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import lombok.AccessLevel; @@ -55,7 +53,7 @@ public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set< for (Map.Entry entry : httpHeaders.entrySet()) { wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - this.ws = wsBuilder.buildAsync(URI.create(url), new WebsocketListenerGlue(this::onSocketOpen, this::onText)); + this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); this.handleMessageTypes = handleMessageTypes; this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); } @@ -149,6 +147,7 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) From 6b872d4b76a02976b4f4ad98ef0606bc7681704f Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Tue, 7 Jul 2026 18:08:02 +0000 Subject: [PATCH 05/67] Formatting --- .../BufferedWebSocketListener.java | 49 +-- .../orchestration/OrchestrationClient.java | 65 ++-- .../orchestration/RealtimeChannelFactory.java | 77 ++--- .../sdk/orchestration/TextInputChannel.java | 2 +- .../TextToSpeechRealtimeClient.java | 99 +++--- .../WSSOpenAIRealtimeClient.java | 297 +++++++++--------- .../java/com/sap/ai/sdk/app/Application.java | 42 +-- .../app/services/OrchestrationService.java | 5 +- 8 files changed, 321 insertions(+), 315 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java index 118b2d5ce..6116baeea 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java @@ -8,32 +8,33 @@ public class BufferedWebSocketListener implements WebSocket.Listener { - private final Consumer onOpen; - private final BiConsumer onText; - private final StringBuilder buffer; + private final Consumer onOpen; + private final BiConsumer onText; + private final StringBuilder buffer; - public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { - this.onOpen = onOpen; - this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); - } - - @Override - public void onOpen(WebSocket webSocket) { - this.onOpen.accept(webSocket); - webSocket.request(1); - } + public BufferedWebSocketListener( + Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); + } - @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - buffer.append(data); - webSocket.request(1); - if (last) { - var completeMessage = buffer.toString(); - buffer.setLength(0); - this.onText.accept(webSocket, completeMessage); - } + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } - return CompletableFuture.completedStage(null); + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + buffer.append(data); + webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); } + + return CompletableFuture.completedStage(null); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..b03548e49 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,10 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; @@ -22,28 +18,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; - -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; @@ -56,7 +39,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,30 +105,35 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } + /** + * Voices input text into output audio + * + * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) + * @return text input channel to supply text to voice, should be closed when not needed anymore + */ + public TextInputChannel textToSpeech(BiConsumer output) { + var destination = + new AiCoreService() + .getInferenceDestination() + .forModel( + new AiModel() { + @Override + public @NonNull String name() { + return OrchestrationAiModel.GPT_REALTIME.getName(); + } - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); + @Override + public @Nullable String version() { + return OrchestrationAiModel.GPT_REALTIME.getVersion(); + } + }); - var factory = new RealtimeChannelFactory(() -> destination); + var factory = new RealtimeChannelFactory(() -> destination); - return factory.textToSpeech(output); - } + return factory.textToSpeech(output); + } /** * Generate a completion for the given prompt. diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java index 28edd05c4..6eb076f12 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java @@ -2,52 +2,53 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; import java.util.*; import java.util.function.BiConsumer; import java.util.function.Supplier; +import javax.annotation.Nonnull; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); + private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; + private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + private final Supplier destinationSupplier; + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 + * Hz output and Boolean - marker if current audio reply has been finished (current response + * ended) + * @return input channel, allowing for text input + */ + public TextInputChannel textToSpeech( + @Nonnull final BiConsumer audioOutputConsumer) { + var destination = destinationSupplier.get(); + + var extraHeaders = destination.getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + var endpoint = sb.toString(); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java index 20e6fcc66..eff6a15df 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java @@ -1,5 +1,5 @@ package com.sap.ai.sdk.orchestration; public interface TextInputChannel extends AutoCloseable { - void sendText(String text); + void sendText(String text); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java index 45ea9335a..74552bb68 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java @@ -2,68 +2,71 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - import java.util.*; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; @Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient + implements TextInputChannel { - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - private final BiConsumer outputConsumer; + private final BiConsumer outputConsumer; - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } + public TextToSpeechRealtimeClient( + String url, Map httpHeaders, BiConsumer outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); + public void sendText(String text) { + var message = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent( + RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - super.sendMessage(message); - } + super.sendMessage(message); + } - @Override - protected String getSystemPrompt() { - return TASK; - } + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } + @Override + protected List getOutputModalities() { + return OUTPUT_MODALITIES; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.accept(audio, Boolean.FALSE); + log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java index 678bff93a..9415f1840 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java @@ -1,5 +1,7 @@ package com.sap.ai.sdk.orchestration; +import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; + import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,136 +9,138 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.CompletableFuture; - -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider - * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, - * this is normally not configurable and would leave realtime conversation unusable on a pause. - *

- * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds - * before it gets terminated - *

- * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + getOrchestrationObjectMapper() + .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } + } + + @Override + public void close() { + WebSocket ws = this.ws.join(); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } + } - protected abstract String getSystemPrompt(); - - protected abstract List getOutputModalities(); + protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract List getOutputModalities(); - protected void sendMessage(ConversationItemCreateEvent message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } + protected abstract void onResponse(String eventType, JsonNode event); - } + protected void sendMessage(ConversationItemCreateEvent message) { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); + askForResponse(); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); + } } - - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + log.warn("WSS received message: {}", data); + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Error parsing JSON response from speech API", e); } - - private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } - - webSocket.request(1); + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); } - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } - private void configureSession(WebSocket ws) { + private void configureSession(WebSocket ws) { SessionUpdateEvent sue = SessionUpdateEvent.builder() .session( @@ -147,10 +151,12 @@ private void configureSession(WebSocket ws) { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection(Optional.empty()) .format( RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .build()) .build()) .output( @@ -163,43 +169,48 @@ private void configureSession(WebSocket ws) { .build()) .asRealtime()) .build(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); - } - - ws.request(1); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize session request", e); } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - var systemConversationItem = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent(RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); - } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); - } - ws.request(1); - } - - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = new TimerTask(){ - @Override - public void run() { - sendPing(ws); - } + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + log.warn("WSS conversation config sent: {}", json); + } catch (JsonProcessingException e) { + throw new OrchestrationClientException("Failed to serialize message", e); } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..61b5be494 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -5,10 +5,11 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; -import javax.annotation.Nonnull; - import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; +import java.io.*; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nonnull; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +17,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -51,13 +46,15 @@ public static void main(final String[] args) throws Exception { var svc = ctx.getBean(OrchestrationService.class); var outputBytes = new ArrayList(); var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { + try (var channel = + svc.textToSpeech( + (bytes, last) -> { + System.err.println("appended output bytes: " + bytes.length); + outputBytes.add(bytes); + if (last) { + done.set(true); + } + })) { System.err.println("BEFORE SENDING TEXT"); channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); System.err.println("AFTER SENDING TEXT"); @@ -69,18 +66,23 @@ public static void main(final String[] args) throws Exception { } } - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; + // var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); + String path = + System.getProperty("user.home") + + File.separator + + "Documents" + + File.separator + + System.currentTimeMillis() + + ".pcm"; File file = new File(path); try (FileOutputStream in = new FileOutputStream(file)) { var ttl = 0; - for (byte[] chunk: outputBytes) { + for (byte[] chunk : outputBytes) { System.out.println("Writing bytes: " + chunk.length); in.write(chunk); ttl += chunk.length; } System.out.println("Written bytes: " + ttl); } - } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..8a26d382b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -67,8 +67,9 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) /** * Voices input text into output audio * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) + * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, + * little-endian, boolean flag signifies end of the current reply (only true for last chunk of + * current response) * @return text input channel to supply text to voice, should be closed when not needed anymore */ public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { From 54b06a55a28df501253efb0050e5608e91c3e3a2 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 14:56:26 +0200 Subject: [PATCH 06/67] add speech to speech realtime support and text to speech example --- foundation-models/openai/pom.xml | 4 + .../openai/AudioInputChannel.java | 17 +++ .../openai/AudioOutputChannel.java | 22 ++++ .../openai}/BufferedWebSocketListener.java | 17 ++- .../foundationmodels/openai/OpenAiClient.java | 73 ++++++++++++ .../openai/SpeechToSpeechRealtimeClient.java | 103 +++++++++++++++++ .../openai/SpeechToTextRealtimeClient.java | 93 +++++++++++++++ .../openai}/TextInputChannel.java | 2 +- .../openai/TextOutputChannel.java | 20 ++++ .../openai/TextToSpeechRealtimeClient.java | 100 ++++++++++++++++ .../openai}/WSSOpenAIRealtimeClient.java | 58 +++------- orchestration/pom.xml | 5 - .../orchestration/OrchestrationClient.java | 46 -------- .../orchestration/RealtimeChannelFactory.java | 53 --------- .../TextToSpeechRealtimeClient.java | 69 ----------- pom.xml | 6 + sample-code/spring-app/pom.xml | 4 + .../java/com/sap/ai/sdk/app/Application.java | 46 +------- .../com/sap/ai/sdk/app/WebsocketConfig.java | 23 ++++ .../TextToSpeechWebsocketHandler.java | 61 ++++++++++ .../ai/sdk/app/services/OpenAiService.java | 48 ++++++-- .../app/services/OrchestrationService.java | 12 -- .../src/main/resources/static/index.html | 107 ++++++++++++++++++ 23 files changed, 707 insertions(+), 282 deletions(-) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/BufferedWebSocketListener.java (69%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/TextInputChannel.java (65%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java rename {orchestration/src/main/java/com/sap/ai/sdk/orchestration => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/WSSOpenAIRealtimeClient.java (70%) delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java delete mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 4e47adcd3..81ea4024f 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -66,6 +66,10 @@ com.sap.ai.sdk core + + com.openai + openai-java + com.google.code.findbugs jsr305 diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java new file mode 100644 index 000000000..9b4862802 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -0,0 +1,17 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing audio input channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioInputChannel extends AutoCloseable { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + */ + void inputAudio(byte[] rawBytesChunk); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java new file mode 100644 index 000000000..e62e3c4cd --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -0,0 +1,22 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import java.io.IOException; + +/** + * Functional interface representing audio output channel (audio data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface AudioOutputChannel { + + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context + * + * @param rawBytesChunk binary data in the depending on the use case format + * @param isLast true if this call logically concludes previous and this passed bytes data into a + * single logical entity (e.g. gets called at the end when all byte parts of a single + * message get passed) + */ + void outputAudio(byte[] rawBytesChunk, boolean isLast); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java similarity index 69% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 118b2d5ce..8377a2889 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -1,11 +1,15 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; + +import lombok.extern.slf4j.Slf4j; import java.net.http.WebSocket; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; +@Slf4j public class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; @@ -36,4 +40,15 @@ public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean return CompletableFuture.completedStage(null); } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("Websocket error occurred during realtime communication", error); + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + log.warn("Received unexpected binary bytes for WebSocket connection"); + return CompletableFuture.completedStage(null); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index ce981fa2a..72e162e6a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -33,6 +33,7 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -50,6 +51,8 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; + private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -260,6 +263,76 @@ public Stream streamChatCompletion(@Nonnull final String prompt) .map(OpenAiChatCompletionDelta::getDeltaContent); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint() + "?intent=transcription"; + + return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel); + } + + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } + private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) { final String finishReason = delta.getFinishReason(); if (finishReason != null && finishReason.equals("content_filter")) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java new file mode 100644 index 000000000..3addfabfe --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -0,0 +1,103 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import java.util.*; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = ""; + + + private final AudioOutputChannel outputConsumer; + + public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java new file mode 100644 index 000000000..6e56866fb --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -0,0 +1,93 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import java.util.*; + +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { + + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio_transcript.done" + ); + + private static final String TASK = + "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + + + private final TextOutputChannel outputConsumer; + + public SpeechToTextRealtimeClient(String url, Map httpHeaders, TextOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void inputAudio(byte[] rawAudioChunk){ + if (rawAudioChunk.length == 0) { + return; + } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)).build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + } + + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio_transcript.done".equals(eventType)) { + this.outputConsumer + .outputText(event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); + } else { + log.trace("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofTranscription( + RealtimeTranscriptionSessionCreateRequest.builder() + .audio( + RealtimeTranscriptionSessionAudio.builder() + .input( + RealtimeTranscriptionSessionAudioInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .transcription( + AudioTranscription.builder() + .model(AudioTranscription.Model.GPT_4O_TRANSCRIBE) + .build()) + .build()) + .build()) + .build()) + .asTranscription()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java similarity index 65% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index 20e6fcc66..8e2372024 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; public interface TextInputChannel extends AutoCloseable { void sendText(String text); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java new file mode 100644 index 000000000..1c0425bb4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -0,0 +1,20 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +/** + * Functional interface representing test output channel (text data consumer) + *

+ * Should be closed by application (try-with-resources) when not needed anymore + */ +public interface TextOutputChannel { + + /** + * This method is sequentially invoked by test data provider to supply implementer (consumer) + * with the test data. + * + * @param textChunk chunk of the text (possibly partial content) + * @param isLast true if this call logically concludes previous and this passed text data into a + * single logical entity (e.g. gets called at the end when all parts of a single + * message get passed) + */ + void outputText(CharSequence textChunk, boolean isLast); +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java new file mode 100644 index 000000000..5e42e5ac4 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -0,0 +1,100 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.*; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; + +@Slf4j +public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = Set.of( + "response.output_audio.delta", "response.output_audio.done" + ); + + private static final List OUTPUT_MODALITIES = List.of( + RealtimeSessionCreateRequest.OutputModality.AUDIO + ); + + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + + + private final AudioOutputChannel outputConsumer; + + public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } + + public void sendText(String text) { + var message = ConversationItemCreateEvent.builder() + .item(ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent(RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() + )).build(); + + super.sendMessage(message); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build()) + .output( + RealtimeAudioConfigOutput.builder() + .format(RealtimeAudioFormats.AudioPcm + .builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice( + RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); + return sessionUpdateEvent; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java similarity index 70% rename from orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 678bff93a..77e2d881b 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.orchestration; +package com.sap.ai.sdk.foundationmodels.openai; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.common.ClientException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -18,14 +19,12 @@ import java.util.*; import java.util.concurrent.CompletableFuture; -import static com.sap.ai.sdk.orchestration.OrchestrationJacksonConfiguration.getOrchestrationObjectMapper; - @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = getOrchestrationObjectMapper() + private static final ObjectMapper JACKSON = new ObjectMapper() .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); /** @@ -86,20 +85,18 @@ public void close() { protected abstract String getSystemPrompt(); - protected abstract List getOutputModalities(); - protected abstract void onResponse(String eventType, JsonNode event); - protected void sendMessage(ConversationItemCreateEvent message) { + protected abstract SessionUpdateEvent sessionConfiguration(); + + protected void sendMessage(Object message) { WebSocket ws = this.ws.join(); synchronized (this) { try { ws.sendText(JACKSON.writeValueAsString(message), true); ws.request(1); - log.warn("WSS sent message: {}", JACKSON.writeValueAsString(message)); - askForResponse(); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } } @@ -112,17 +109,17 @@ private synchronized void onSocketOpen(WebSocket ws) { } private void onText(WebSocket webSocket, CharSequence data) { - log.warn("WSS received message: {}", data); final JsonNode event; try { event = JACKSON.readTree(data.toString()); } catch (JsonProcessingException e) { - throw new OrchestrationClientException( - "Error parsing JSON response from speech API", e); + throw new ClientException("Error parsing JSON response from speech API", e); } var eventType = event.get("type").asText(); if (handleMessageTypes.contains(eventType)) { onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); } webSocket.request(1); @@ -137,36 +134,11 @@ private synchronized void sendPing(WebSocket ws) { } private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(getOutputModalities()) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm.builder().build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); + SessionUpdateEvent sue = sessionConfiguration(); try { ws.sendText(JACKSON.writeValueAsString(sue), true); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize session request", e); + throw new ClientException("Failed to serialize session request", e); } ws.request(1); @@ -174,6 +146,9 @@ private void configureSession(WebSocket ws) { private void configureConversation(WebSocket ws) { var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; + } var systemConversationItem = ConversationItemCreateEvent.builder() .item(ConversationItem.ofRealtimeConversationItemSystemMessage( RealtimeConversationItemSystemMessage.builder() @@ -185,9 +160,8 @@ private void configureConversation(WebSocket ws) { try { var json = JACKSON.writeValueAsString(systemConversationItem); ws.sendText(json, true); - log.warn("WSS conversation config sent: {}", json); } catch (JsonProcessingException e) { - throw new OrchestrationClientException("Failed to serialize message", e); + throw new ClientException("Failed to serialize message", e); } ws.request(1); } diff --git a/orchestration/pom.xml b/orchestration/pom.xml index a1ef764f1..f8027fe4a 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -122,11 +122,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - com.openai - openai-java - 4.41.0 - org.projectlombok diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 7c52d772b..78f347b56 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -7,12 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.Beta; -import com.openai.client.OpenAIClient; -import com.openai.client.okhttp.OpenAIOkHttpClient; -import com.openai.models.beta.realtime.sessions.Session; -import com.openai.models.realtime.SessionUpdateEvent; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.core.AiModel; import com.sap.ai.sdk.orchestration.model.CompletionPostRequest; import com.sap.ai.sdk.orchestration.model.CompletionPostResponse; import com.sap.ai.sdk.orchestration.model.CompletionRequestConfiguration; @@ -22,30 +17,15 @@ import com.sap.ai.sdk.orchestration.model.OrchestrationConfig; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import io.vavr.collection.Array; import io.vavr.control.Try; -import java.io.ByteArrayOutputStream; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.function.BiConsumer; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; /** Client to execute requests to the orchestration service. */ @Slf4j @@ -56,7 +36,6 @@ public class OrchestrationClient { static final ObjectMapper JACKSON = getOrchestrationObjectMapper(); - private final OrchestrationHttpExecutor executor; private final List

customHeaders = new ArrayList<>(); @@ -123,31 +102,6 @@ public OrchestrationChatResponse chatCompletion( return new OrchestrationChatResponse(response); } - /** - * Voices input text into output audio - * - * @param output byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(BiConsumer output) { - var destination = new AiCoreService().getInferenceDestination().forModel(new AiModel() { - @Override - public @NonNull String name() { - return OrchestrationAiModel.GPT_REALTIME.getName(); - } - - @Override - public @Nullable String version() { - return OrchestrationAiModel.GPT_REALTIME.getVersion(); - } - }); - - var factory = new RealtimeChannelFactory(() -> destination); - - return factory.textToSpeech(output); - } - /** * Generate a completion for the given prompt. * diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java deleted file mode 100644 index 28edd05c4..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/RealtimeChannelFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.sap.cloud.sdk.cloudplatform.connectivity.Header; -import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; -import java.util.*; -import java.util.function.BiConsumer; -import java.util.function.Supplier; - -@Slf4j -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) -public class RealtimeChannelFactory { - private static final String AI_RESOURCE_GROUP_HEADER_NAME = "AI-Resource-Group"; - private static final String AI_RESOURCE_GROUP_HEADER_VALUE = "default"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Supplier destinationSupplier; - - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - * @param audioOutputConsumer - audio bytes consumer, where byte[] - chunks of raw PCM mono 24000 Hz output and - * Boolean - marker if current audio reply has been finished (current response ended) - * @return input channel, allowing for text input - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer audioOutputConsumer) { - var destination = destinationSupplier.get(); - - var extraHeaders = destination.getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - headers.put(AI_RESOURCE_GROUP_HEADER_NAME, AI_RESOURCE_GROUP_HEADER_VALUE); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - var endpoint = sb.toString(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); - } - -} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java deleted file mode 100644 index 45ea9335a..000000000 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextToSpeechRealtimeClient.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.sap.ai.sdk.orchestration; - -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; -import java.util.function.BiConsumer; - -@Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); - - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); - - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; - - - private final BiConsumer outputConsumer; - - public TextToSpeechRealtimeClient(String url, Map httpHeaders, BiConsumer outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } - - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - super.sendMessage(message); - } - - @Override - protected String getSystemPrompt() { - return TASK; - } - - @Override - protected List getOutputModalities() { - return OUTPUT_MODALITIES; - } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.accept(audio, Boolean.FALSE); - log.warn("Called consumer with base64 converted to bytes: {} (base64)", base64Audio); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.accept(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } - } -} diff --git a/pom.xml b/pom.xml index 6cf8692d1..03781bc31 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 2.1.3 3.5.6 1.1.8 + 4.41.0 3.8.6 3.2.0 5.23.0 @@ -117,6 +118,11 @@ pom import + + com.openai + openai-java + ${com.openai.openai-java.version} + io.projectreactor diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index 41f069c14..c5ccd9c1f 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -130,6 +130,10 @@ org.springframework.ai spring-ai-commons + + org.springframework.boot + spring-boot-starter-websocket + org.springframework.ai spring-ai-model diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index aced5c87e..0f6b0e26c 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; -import com.sap.ai.sdk.app.services.OrchestrationService; -import com.sap.ai.sdk.core.AiCoreService; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; @@ -16,12 +14,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Primary; -import java.io.*; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** Main class to start the Spring Boot application. */ @SpringBootApplication @ComponentScan({"com.sap.cloud.sdk", "com.sap.ai.sdk.app"}) @@ -46,41 +38,7 @@ public ObjectMapper objectMapper() { * * @param args Command line arguments. */ - public static void main(final String[] args) throws Exception { - var ctx = SpringApplication.run(Application.class, args); - var svc = ctx.getBean(OrchestrationService.class); - var outputBytes = new ArrayList(); - var done = new AtomicBoolean(false); - try (var channel = svc.textToSpeech((bytes, last) -> { - System.err.println("appended output bytes: " + bytes.length); - outputBytes.add(bytes); - if (last) { - done.set(true); - } - })) { - System.err.println("BEFORE SENDING TEXT"); - channel.sendText("Eat more of these fresh french baguettes and drink the tea!"); - System.err.println("AFTER SENDING TEXT"); - long timeout = System.currentTimeMillis() + 20000; - while (!done.get() && System.currentTimeMillis() < timeout) { - Thread.yield(); - System.err.println("Waiting for messages..."); - Thread.sleep(500); - } - } - - //var res = svc.textToSpeech("Eat more of these fresh french baguettes").toList(); - String path = System.getProperty("user.home") + File.separator + "Documents" + File.separator + System.currentTimeMillis() + ".pcm"; - File file = new File(path); - try (FileOutputStream in = new FileOutputStream(file)) { - var ttl = 0; - for (byte[] chunk: outputBytes) { - System.out.println("Writing bytes: " + chunk.length); - in.write(chunk); - ttl += chunk.length; - } - System.out.println("Written bytes: " + ttl); - } - + public static void main(final String[] args) { + SpringApplication.run(Application.class, args); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java new file mode 100644 index 000000000..606b3bc4a --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -0,0 +1,23 @@ +package com.sap.ai.sdk.app; + +import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + +@Configuration +@EnableWebSocket +public class WebsocketConfig implements WebSocketConfigurer { + + private final TextToSpeechWebsocketHandler textToSpeech; + + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + this.textToSpeech = textToSpeech; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + } +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java new file mode 100644 index 000000000..27a667d3c --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +@Slf4j +public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public TextToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] textBytes = payload.array(); + TextInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.textToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send text message to realtime api", e); + } + })); + channel.sendText(new String(textBytes)); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index 1b84e56f6..efb884097 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,15 +5,8 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingRequest; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingResponse; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; +import com.sap.ai.sdk.foundationmodels.openai.*; + import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -74,6 +67,43 @@ public Stream streamChatCompletionDeltas( return OpenAiClient.forModel(GPT_5_MINI).streamChatCompletionDeltas(request); } + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

The input channel should be used with a try-with-resources block to ensure that the underlying + * connection is closed. + * + *

Example: + * + * 

{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
+ * + * This API implements full duplex (input + output) communication channels. Application should logically + * synchronize their state and close input channel when it is appropriate (e.g. last part of the response + * has been received via output channel and application does not need to send any other input). When input + * channel is closed, output channel will be closed automatically and output consumer will not be called + * anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output + * @return input channel, allowing for text input + */ + @Nonnull + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); + } + + public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToText(textOutputConsumer); + } + + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); + } + /** * Asynchronous stream of an OpenAI chat request * diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java index 49fbaa594..f6fc24853 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OrchestrationService.java @@ -33,7 +33,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -64,17 +63,6 @@ public OrchestrationChatResponse completion(@Nonnull final String famousPhrase) return client.chatCompletion(prompt, config); } - /** - * Voices input text into output audio - * - * @param outputConsumer byte chunks output consumer in PCM16 format, 16-bit, mono, 24000 Hz, little-endian, - * boolean flag signifies end of the current reply (only true for last chunk of current response) - * @return text input channel to supply text to voice, should be closed when not needed anymore - */ - public TextInputChannel textToSpeech(@Nonnull final BiConsumer outputConsumer) { - return client.textToSpeech(outputConsumer); - } - /** * Chat request to OpenAI through the Orchestration service with an image. * diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index cd874bc61..be0528612 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1355,6 +1355,113 @@

📂 Batch API

+ +
+
+
+
+

⏰ Realtime API

+
+ Realtime API allows for various real time interactions TBD Documentation +
+
+
    +
  • +
    +

    Text to speech

    + disabled +
    + + + +
    +
    + +
    • +
+
+
+
From 80a5d3eecd598f90d110ce2abcfbac1e716a9b10 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 10 Jul 2026 19:07:08 +0200 Subject: [PATCH 07/67] add speech to speech demo scenario --- .../openai/SpeechToSpeechRealtimeClient.java | 34 ++--- .../com/sap/ai/sdk/app/WebsocketConfig.java | 6 +- .../SpeechToSpeechWebsocketHandler.java | 61 +++++++++ .../src/main/resources/static/index.html | 91 ++------------ .../main/resources/static/speech-to-speech.js | 117 ++++++++++++++++++ .../main/resources/static/text-to-speech.js | 77 ++++++++++++ 6 files changed, 292 insertions(+), 94 deletions(-) create mode 100644 sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java create mode 100644 sample-code/spring-app/src/main/resources/static/speech-to-speech.js create mode 100644 sample-code/spring-app/src/main/resources/static/text-to-speech.js diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3addfabfe..afbefea13 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -44,9 +44,10 @@ public void inputAudio(byte[] rawAudioChunk) { cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + // TODO: make turn detection configurable and uncomment for explicit turn control + // var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + // super.sendMessage(commitAudioMessage); + // askForResponse(); } @Override @@ -67,7 +68,8 @@ protected void onResponse(String eventType, JsonNode event) { @Override protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() @@ -76,21 +78,25 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) + .turnDetection( + RealtimeAudioInputTurnDetection.ofSemanticVad(RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) .format( - RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) .build()) .output( RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + .build()) .voice( RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) .build()) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 606b3bc4a..ac7b07758 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app; +import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; @@ -11,13 +12,16 @@ public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; + private final SpeechToSpeechWebsocketHandler speechToSpeech; - public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech) { + public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { this.textToSpeech = textToSpeech; + this.speechToSpeech = speechToSpeech; } @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java new file mode 100644 index 000000000..72f27fab0 --- /dev/null +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -0,0 +1,61 @@ +package com.sap.ai.sdk.app.realtime; + +import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.jspecify.annotations.NonNull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; + +@Component +@Slf4j +public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { + + private final OpenAiService service; + private final Map channels; + + @Autowired + public SpeechToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] chunkBytes = payload.array(); + AudioInputChannel channel = channels.computeIfAbsent( + session.getId(), + sessionId -> service.speechToSpeech((rawBytesChunk, isLast) -> { + try { + session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); + } catch (IOException e) { + log.error("failed to send audio data to realtime api", e); + } + })); + channel.inputAudio(chunkBytes); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { + channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; + }); + super.afterConnectionClosed(session, status); + } + +} diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index be0528612..a30067be7 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1378,85 +1378,18 @@

Text to speech

- + + +
  • +
    +

    Speech to speech

    +
    disconnected
    +
    disabled
    +
    + +
    +
    +
    • diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js new file mode 100644 index 000000000..e6cd3f9be --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -0,0 +1,117 @@ +const SPEECH_TO_SPEECH_URL = 'ws://localhost:8080/speech-to-speech'; +const SPEECH_TO_SPEECH_SAMPLE_RATE = 24000; + +const wsStatusEl = document.getElementById('speech-to-speech-websocket-status') +const micStatusEl = document.getElementById('speech-to-speech-mic-status') +const speechBtn = document.getElementById('speech-to-speech-btn') + +let sts_ws = null; +let sts_audioCtx = null; +let sts_mediaStream = null; +let sts_processor = null; +let sts_nextStartTime = 0; +let sts_started = false; + +async function startSession() { + sts_ws = new WebSocket(SPEECH_TO_SPEECH_URL); + sts_ws.binaryType = 'arraybuffer'; + + sts_ws.onopen = async () => { + wsStatusEl.textContent = `connected to ${SPEECH_TO_SPEECH_URL}`; + wsStatusEl.style.color = 'green'; + + await startMicrophone(); + speechBtn.innerText = 'Stop'; + sts_started = true; + } + + sts_ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + sts_playPcmAudio(event.data) + } + } + + sts_ws.onclose = () => stopSession(); +} + +async function startMicrophone() { + try { + sts_mediaStream = await navigator.mediaDevices.getUserMedia({audio: true}); + console.log("mediaStream is: ", sts_mediaStream) + micStatusEl.textContent = 'active'; + micStatusEl.style.color = 'green'; + + sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); + + const source = sts_audioCtx.createMediaStreamSource(sts_mediaStream); + sts_processor = sts_audioCtx.createScriptProcessor(2048, 1, 1); + sts_processor.onaudioprocess = (e) => { + if (sts_ws && sts_ws.readyState === WebSocket.OPEN) { + const float32Input = e.inputBuffer.getChannelData(0); + const int16Buffer = new Int16Array(float32Input.length); + + for (let i = 0; i < float32Input.length; i++) { + let s = Math.max(-1, Math.min(1, float32Input[i])); + int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + sts_ws.send(int16Buffer.buffer); + } + }; + + const gainNode = sts_audioCtx.createGain(); + gainNode.gain.value = 0; + + source.connect(sts_processor); + sts_processor.connect(gainNode); + gainNode.connect(sts_audioCtx.destination) + + } catch (err) { + console.error('failed to find or bind microphone: ', err); + micStatusEl.innerText = 'mic binding error'; + micStatusEl.style.color = 'red'; + } +} + +function sts_playPcmAudio(arrayBuffer) { + if (!sts_audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = sts_audioCtx.createBuffer(1, float32Data.length, SPEECH_TO_SPEECH_SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = sts_audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(sts_audioCtx.destination); + + if (sts_nextStartTime < sts_audioCtx.currentTime) { + sts_nextStartTime = sts_audioCtx.currentTime; + } + + source.start(sts_nextStartTime); + sts_nextStartTime += audioBuffer.duration; +} + +function stopSession() { + if (sts_ws) sts_ws.close(); + if (sts_mediaStream) sts_mediaStream.getTracks().forEach(track => track.stop()); + if (sts_processor) sts_processor.disconnect(); + if (sts_audioCtx) { + sts_audioCtx.close(); + sts_audioCtx = null; + } + + wsStatusEl.textContent = 'disconnected'; + wsStatusEl.style.color = 'red'; + micStatusEl.textContent = 'disabled'; + micStatusEl.style.color = 'red'; + speechBtn.innerText = 'Start'; + sts_started = false; +} + +speechBtn.addEventListener('click', () => { + sts_started ? stopSession() : startSession(); +}) \ No newline at end of file diff --git a/sample-code/spring-app/src/main/resources/static/text-to-speech.js b/sample-code/spring-app/src/main/resources/static/text-to-speech.js new file mode 100644 index 000000000..80bd75841 --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/text-to-speech.js @@ -0,0 +1,77 @@ +const WS_URL = 'ws://localhost:8080/text-to-speech'; +const SAMPLE_RATE = 24000; + +const statusEl = document.getElementById('text-to-speech-status') +const textInput = document.getElementById('text-to-speech-input') +const sendBtn = document.getElementById('text-to-speech-send-btn') + +let audioCtx; +let nextStartTime = 0; + +let ws = new WebSocket(WS_URL); +ws.binaryType = 'arraybuffer'; + +ws.onopen = () => { + statusEl.textContent = `connected to ${WS_URL}`; + statusEl.style.color = 'green'; + textInput.disabled = false; + sendBtn.disabled = false; +}; + +ws.onclose = () => { + statusEl.textContent = 'disconnected'; + statusEl.style.color = 'red'; + textInput.disabled = true; + sendBtn.disabled = true; +} + +ws.onerror = (error) => { + console.error('text-to-speech WebSocket error:', error); + statusEl.textContent = 'connection error' + statusEl.style.color = 'red'; +} + +ws.onmessage = (event) => { + if (event.data instanceof ArrayBuffer) { + playPcmAudio(event.data); + } else { + console.log('text data received:', event.data) + } +} + +sendBtn.addEventListener('click', () => { + const text = textInput.value.trim(); + if (text && ws.readyState === WebSocket.OPEN) { + if (!audioCtx) { + audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE}); + } + + const encoder = new TextEncoder(); + const binaryData = encoder.encode(text); + ws.send(binaryData); + textInput.value = ''; + } +}) + +function playPcmAudio(arrayBuffer) { + if (!audioCtx) return; + + const int16Data = new Int16Array(arrayBuffer); + const float32Data = new Float32Array(int16Data.length); + for (let i = 0; i < int16Data.length; i++) { + float32Data[i] = int16Data[i] / 32768.0; + } + + const audioBuffer = audioCtx.createBuffer(1, float32Data.length, SAMPLE_RATE); + audioBuffer.copyToChannel(float32Data, 0); + const source = audioCtx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioCtx.destination); + + if (nextStartTime < audioCtx.currentTime) { + nextStartTime = audioCtx.currentTime; + } + + source.start(nextStartTime); + nextStartTime += audioBuffer.duration; +} \ No newline at end of file From d6ad031d61916b33d828979636a9f7705c50898b Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:34:57 +0200 Subject: [PATCH 08/67] add smoke test for realtime api and configurability --- .../ai/sdk/core/model/SpeechOutputParam.java | 14 +++ .../model/SpeechOutputParamTurnDetection.java | 35 ++++++ .../core/model/SpeechOutputParamVoice.java | 35 ++++++ .../foundationmodels/openai/OpenAiClient.java | 14 ++- .../openai/SpeechToSpeechRealtimeClient.java | 118 ++++++++++++------ .../openai/TextToSpeechRealtimeClient.java | 37 +++++- .../SpeechToSpeechWebsocketHandler.java | 4 +- .../ai/sdk/app/services/OpenAiService.java | 4 - .../sdk/app/controllers/RealtimeApiTest.java | 47 +++++++ 9 files changed, 253 insertions(+), 55 deletions(-) create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java create mode 100644 sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java new file mode 100644 index 000000000..55a7d08c1 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java @@ -0,0 +1,14 @@ +package com.sap.ai.sdk.core.model; + +import javax.annotation.Nonnull; + +public interface SpeechOutputParam { + enum SpeechOutputParamName { + VOICE, + TURN_DETECTION, + } + + @Nonnull + SpeechOutputParamName getParamName(); + +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java new file mode 100644 index 000000000..aea82e0d8 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java @@ -0,0 +1,35 @@ +package com.sap.ai.sdk.core.model; + +import org.jspecify.annotations.NonNull; + +import java.util.Objects; + +public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { + + public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); + public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = + new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String value; + + private SpeechOutputParamTurnDetection(String value) { + this.value = value; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java new file mode 100644 index 000000000..8f8c12473 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java @@ -0,0 +1,35 @@ +package com.sap.ai.sdk.core.model; + +import org.jspecify.annotations.NonNull; + +import javax.annotation.Nonnull; +import java.util.Objects; + +public final class SpeechOutputParamVoice implements SpeechOutputParam { + + public static final SpeechOutputParamVoice DEFAULT_MAN = new SpeechOutputParamVoice("DEFAULT_MAN"); + public static final SpeechOutputParamVoice DEFAULT_WOMAN = new SpeechOutputParamVoice("DEFAULT_WOMAN"); + + private final String voice; + + private SpeechOutputParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 72e162e6a..de19dbd71 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -15,6 +15,7 @@ import com.sap.ai.sdk.core.common.ClientStreamingHandler; import com.sap.ai.sdk.core.common.RequestLogContext; import com.sap.ai.sdk.core.common.StreamedDelta; +import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.foundationmodels.openai.generated.model.ChatCompletionStreamOptions; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionResponse; @@ -288,7 +289,8 @@ public Stream streamChatCompletion(@Nonnull final String prompt) * @return input channel, allowing for text input */ @Nonnull - public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { + public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer, + SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -296,10 +298,11 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp } var endpoint = getRealtimeEndpoint(); - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); } - public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { + // unstable, requires further debugging and testing + AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -310,7 +313,8 @@ public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutpu return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel) { + public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel, + SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -318,7 +322,7 @@ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioO } var endpoint = getRealtimeEndpoint(); - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel); + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel, params); } private String getRealtimeEndpoint() { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index afbefea13..3efbd96ad 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -4,6 +4,10 @@ import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import java.util.*; + +import com.sap.ai.sdk.core.model.SpeechOutputParam; +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -22,12 +26,37 @@ public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implem private static final String TASK = ""; - private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; + + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = false; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; } public void inputAudio(byte[] rawAudioChunk) { @@ -44,10 +73,11 @@ public void inputAudio(byte[] rawAudioChunk) { cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - // TODO: make turn detection configurable and uncomment for explicit turn control - // var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - // super.sendMessage(commitAudioMessage); - // askForResponse(); + if (eagerTurnDetection) { + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } } @Override @@ -68,42 +98,50 @@ protected void onResponse(String eventType, JsonNode event) { @Override protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build() + ); + } + + return SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() + RealtimeSessionCreateRequest + .builder() .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection( - RealtimeAudioInputTurnDetection.ofSemanticVad(RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - return sessionUpdateEvent; + .audio(RealtimeAudioConfig.builder() + .input( + RealtimeAudioConfigInput.builder() + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build() + ) + .build() + ) + .output( + RealtimeAudioConfigOutput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build() + ) + .voice(voice) + .build() + ) + .build() + ) + .build() + ) + .asRealtime() + ) + .build(); } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 5e42e5ac4..d4e812cf0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,6 +3,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.model.SpeechOutputParam; +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; import lombok.extern.slf4j.Slf4j; import java.util.*; @@ -25,10 +28,35 @@ public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implemen private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer) { + public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = true; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; } public void sendText(String text) { @@ -41,7 +69,9 @@ public void sendText(String text) { )).build(); super.sendMessage(message); - askForResponse(); + if (eagerTurnDetection) { + askForResponse(); + } } @Override @@ -88,8 +118,7 @@ protected SessionUpdateEvent sessionConfiguration() { .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) .build()) - .voice( - RealtimeAudioConfigOutput.Voice.UnionMember1.ALLOY) + .voice(voice) .build()) .build()) .build()) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 72f27fab0..e593fb651 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -2,7 +2,6 @@ import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; @@ -41,7 +40,8 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } catch (IOException e) { log.error("failed to send audio data to realtime api", e); } - })); + }) + ); channel.inputAudio(chunkBytes); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index efb884097..ad25c15a5 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -96,10 +96,6 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); } - public AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToText(textOutputConsumer); - } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); } diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java new file mode 100644 index 000000000..988ae7031 --- /dev/null +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -0,0 +1,47 @@ +package com.sap.ai.sdk.app.controllers; + +import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.ByteArrayOutputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +@Slf4j +public class RealtimeApiTest { + + @Test + @Timeout(value = 30, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + + try (TextInputChannel input = client.textToSpeech((byteChunk, last) -> { + outputBuffer.writeBytes(byteChunk); + if (last) { + monitor.countDown(); + } + }, SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.sendText("Ordnung muss sein!"); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is reached + return; + } + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + } + +} \ No newline at end of file From 2ed77e0435e2c2c889004798403cb9235058e15e Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:48:08 +0200 Subject: [PATCH 09/67] rework sample application so it no longer use deprecated browser microphone api --- .../main/resources/static/speech-to-speech.js | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js index e6cd3f9be..71d224a9c 100644 --- a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -8,7 +8,7 @@ const speechBtn = document.getElementById('speech-to-speech-btn') let sts_ws = null; let sts_audioCtx = null; let sts_mediaStream = null; -let sts_processor = null; +let sts_workletNode = null; let sts_nextStartTime = 0; let sts_started = false; @@ -43,27 +43,38 @@ async function startMicrophone() { sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); + const workletCode = ` + class MicProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0]) { + const float32Input = input[0]; + const int16Buffer = new Int16Array(float32Input.length); + for (let i = 0; i < float32Input.length; i++) { + const s = Math.max(-1, Math.min(1, float32Input[i])); + int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; + } + this.port.postMessage(int16Buffer.buffer, [int16Buffer.buffer]); + } + return true; + } + } + registerProcessor('mic-processor', MicProcessor); + `; + const blob = new Blob([workletCode], {type: 'application/javascript'}); + const workletUrl = URL.createObjectURL(blob); + await sts_audioCtx.audioWorklet.addModule(workletUrl); + URL.revokeObjectURL(workletUrl); + const source = sts_audioCtx.createMediaStreamSource(sts_mediaStream); - sts_processor = sts_audioCtx.createScriptProcessor(2048, 1, 1); - sts_processor.onaudioprocess = (e) => { + sts_workletNode = new AudioWorkletNode(sts_audioCtx, 'mic-processor'); + sts_workletNode.port.onmessage = (e) => { if (sts_ws && sts_ws.readyState === WebSocket.OPEN) { - const float32Input = e.inputBuffer.getChannelData(0); - const int16Buffer = new Int16Array(float32Input.length); - - for (let i = 0; i < float32Input.length; i++) { - let s = Math.max(-1, Math.min(1, float32Input[i])); - int16Buffer[i] = s < 0 ? s * 0x8000 : s * 0x7FFF; - } - sts_ws.send(int16Buffer.buffer); + sts_ws.send(e.data); } }; - const gainNode = sts_audioCtx.createGain(); - gainNode.gain.value = 0; - - source.connect(sts_processor); - sts_processor.connect(gainNode); - gainNode.connect(sts_audioCtx.destination) + source.connect(sts_workletNode); } catch (err) { console.error('failed to find or bind microphone: ', err); @@ -98,7 +109,7 @@ function sts_playPcmAudio(arrayBuffer) { function stopSession() { if (sts_ws) sts_ws.close(); if (sts_mediaStream) sts_mediaStream.getTracks().forEach(track => track.stop()); - if (sts_processor) sts_processor.disconnect(); + if (sts_workletNode) sts_workletNode.disconnect(); if (sts_audioCtx) { sts_audioCtx.close(); sts_audioCtx = null; From c212c1e854c5c8dd06b1e3c0dc1b84260bd2c09f Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 13 Jul 2026 17:56:23 +0200 Subject: [PATCH 10/67] make realtime implementation package local --- .../foundationmodels/openai/SpeechToSpeechRealtimeClient.java | 2 +- .../sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java | 2 +- .../sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java | 2 +- .../ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3efbd96ad..8ee5280d6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -11,7 +11,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index 6e56866fb..aacdf2a0a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -8,7 +8,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = Set.of( diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d4e812cf0..b42731278 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -11,7 +11,7 @@ import java.util.*; @Slf4j -public class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 77e2d881b..5d8f571e6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -21,7 +21,7 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) -public abstract class WSSOpenAIRealtimeClient implements AutoCloseable { +abstract class WSSOpenAIRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; private static final ObjectMapper JACKSON = new ObjectMapper() From 03d1e70a75fc0f04233ee53bdf73d3ec9c69eaad Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 13 Jul 2026 16:02:30 +0000 Subject: [PATCH 11/67] Formatting --- .../ai/sdk/core/model/SpeechOutputParam.java | 13 +- .../model/SpeechOutputParamTurnDetection.java | 56 ++-- .../core/model/SpeechOutputParamVoice.java | 57 ++-- .../openai/AudioInputChannel.java | 19 +- .../openai/AudioOutputChannel.java | 27 +- .../openai/BufferedWebSocketListener.java | 74 ++--- .../foundationmodels/openai/OpenAiClient.java | 29 +- .../openai/SpeechToSpeechRealtimeClient.java | 217 +++++++------ .../openai/SpeechToTextRealtimeClient.java | 97 +++--- .../openai/TextOutputChannel.java | 24 +- .../openai/TextToSpeechRealtimeClient.java | 170 ++++++----- .../openai/WSSOpenAIRealtimeClient.java | 288 +++++++++--------- .../java/com/sap/ai/sdk/app/Application.java | 1 - .../com/sap/ai/sdk/app/WebsocketConfig.java | 23 +- .../SpeechToSpeechWebsocketHandler.java | 70 +++-- .../TextToSpeechWebsocketHandler.java | 78 ++--- .../ai/sdk/app/services/OpenAiService.java | 15 +- .../sdk/app/controllers/RealtimeApiTest.java | 64 ++-- 18 files changed, 672 insertions(+), 650 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java index 55a7d08c1..d9c372cbd 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java @@ -3,12 +3,11 @@ import javax.annotation.Nonnull; public interface SpeechOutputParam { - enum SpeechOutputParamName { - VOICE, - TURN_DETECTION, - } - - @Nonnull - SpeechOutputParamName getParamName(); + enum SpeechOutputParamName { + VOICE, + TURN_DETECTION, + } + @Nonnull + SpeechOutputParamName getParamName(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java index aea82e0d8..1a0e60d27 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java @@ -1,35 +1,35 @@ package com.sap.ai.sdk.core.model; -import org.jspecify.annotations.NonNull; - import java.util.Objects; +import org.jspecify.annotations.NonNull; public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { - public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); - public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = - new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); - - private final String value; - - private SpeechOutputParamTurnDetection(String value) { - this.value = value; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.TURN_DETECTION; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; - return Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } + public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = + new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); + public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = + new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String value; + + private SpeechOutputParamTurnDetection(String value) { + this.value = value; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; + return Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } } diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java index 8f8c12473..35f4f5eb2 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java @@ -1,35 +1,36 @@ package com.sap.ai.sdk.core.model; -import org.jspecify.annotations.NonNull; - -import javax.annotation.Nonnull; import java.util.Objects; +import javax.annotation.Nonnull; +import org.jspecify.annotations.NonNull; public final class SpeechOutputParamVoice implements SpeechOutputParam { - public static final SpeechOutputParamVoice DEFAULT_MAN = new SpeechOutputParamVoice("DEFAULT_MAN"); - public static final SpeechOutputParamVoice DEFAULT_WOMAN = new SpeechOutputParamVoice("DEFAULT_WOMAN"); - - private final String voice; - - private SpeechOutputParamVoice(@Nonnull String voice) { - this.voice = voice; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.VOICE; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; - return Objects.equals(voice, that.voice); - } - - @Override - public int hashCode() { - return Objects.hashCode(voice); - } + public static final SpeechOutputParamVoice DEFAULT_MAN = + new SpeechOutputParamVoice("DEFAULT_MAN"); + public static final SpeechOutputParamVoice DEFAULT_WOMAN = + new SpeechOutputParamVoice("DEFAULT_WOMAN"); + + private final String voice; + + private SpeechOutputParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java index 9b4862802..50bea1504 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -2,16 +2,17 @@ /** * Functional interface representing audio input channel (audio data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioInputChannel extends AutoCloseable { - /** - * This method is sequentially invoked by audio data provider to supply implementer (consumer) - * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context - * - * @param rawBytesChunk binary data in the depending on the use case format - */ - void inputAudio(byte[] rawBytesChunk); + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage + * context + * + * @param rawBytesChunk binary data in the depending on the use case format + */ + void inputAudio(byte[] rawBytesChunk); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index e62e3c4cd..b190fbda5 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -1,22 +1,21 @@ package com.sap.ai.sdk.foundationmodels.openai; -import java.io.IOException; - /** * Functional interface representing audio output channel (audio data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioOutputChannel { - /** - * This method is sequentially invoked by audio data provider to supply implementer (consumer) - * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage context - * - * @param rawBytesChunk binary data in the depending on the use case format - * @param isLast true if this call logically concludes previous and this passed bytes data into a - * single logical entity (e.g. gets called at the end when all byte parts of a single - * message get passed) - */ - void outputAudio(byte[] rawBytesChunk, boolean isLast); + /** + * This method is sequentially invoked by audio data provider to supply implementer (consumer) + * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage + * context + * + * @param rawBytesChunk binary data in the depending on the use case format + * @param isLast true if this call logically concludes previous and this passed bytes data into a + * single logical entity (e.g. gets called at the end when all byte parts of a single message + * get passed) + */ + void outputAudio(byte[] rawBytesChunk, boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 8377a2889..8d29cb358 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -1,54 +1,54 @@ package com.sap.ai.sdk.foundationmodels.openai; -import lombok.extern.slf4j.Slf4j; - import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; @Slf4j public class BufferedWebSocketListener implements WebSocket.Listener { - private final Consumer onOpen; - private final BiConsumer onText; - private final StringBuilder buffer; - - public BufferedWebSocketListener(Consumer onOpen, BiConsumer onText) { - this.onOpen = onOpen; - this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); + private final Consumer onOpen; + private final BiConsumer onText; + private final StringBuilder buffer; + + public BufferedWebSocketListener( + Consumer onOpen, BiConsumer onText) { + this.onOpen = onOpen; + this.onText = onText; + this.buffer = new StringBuilder(1024 * 1024); + } + + @Override + public void onOpen(WebSocket webSocket) { + this.onOpen.accept(webSocket); + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + buffer.append(data); + webSocket.request(1); + if (last) { + var completeMessage = buffer.toString(); + buffer.setLength(0); + this.onText.accept(webSocket, completeMessage); } - @Override - public void onOpen(WebSocket webSocket) { - this.onOpen.accept(webSocket); - webSocket.request(1); - } + return CompletableFuture.completedStage(null); + } - @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { - buffer.append(data); - webSocket.request(1); - if (last) { - var completeMessage = buffer.toString(); - buffer.setLength(0); - this.onText.accept(webSocket, completeMessage); - } - - return CompletableFuture.completedStage(null); - } - - @Override - public void onError(WebSocket webSocket, Throwable error) { - log.error("Websocket error occurred during realtime communication", error); - } + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("Websocket error occurred during realtime communication", error); + } - @Override - public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { - log.warn("Received unexpected binary bytes for WebSocket connection"); - return CompletableFuture.completedStage(null); - } + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + log.warn("Received unexpected binary bytes for WebSocket connection"); + return CompletableFuture.completedStage(null); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index de19dbd71..00f803d3d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -52,8 +52,8 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; - private static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -267,8 +267,8 @@ public Stream streamChatCompletion(@Nonnull final String prompt) /** * Creates realtime channel allowing to input text and voice it (receive audio output) * - *

    The input channel should be used with a try-with-resources block to ensure that the underlying - * connection is closed. + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. * *

    Example: * @@ -279,18 +279,18 @@ public Stream streamChatCompletion(@Nonnull final String prompt) * } * } * - * This API implements full duplex (input + output) communication channels. Application should logically - * synchronize their state and close input channel when it is appropriate (e.g. last part of the response - * has been received via output channel and application does not need to send any other input). When input - * channel is closed, output channel will be closed automatically and output consumer will not be called - * anymore. + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output * @return input channel, allowing for text input */ @Nonnull - public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer, - SpeechOutputParam... params) { + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -313,8 +313,8 @@ AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsum return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); } - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputChannel, - SpeechOutputParam... params) { + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputChannel, SpeechOutputParam... params) { var extraHeaders = destination.asHttp().getHeaders(); var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -330,7 +330,8 @@ private String getRealtimeEndpoint() { sb.append("wss://"); var pathParts = destination.asHttp().getUri().toString().split("//"); if (pathParts.length != 2) { - throw new IllegalArgumentException("Invalid destination URI: " + destination.asHttp().getUri()); + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); } sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); sb.append("v1/realtime"); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 8ee5280d6..3202f1c1f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -3,117 +3,119 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import java.util.*; - import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); - - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); - - private static final String TASK = ""; - - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - private final boolean eagerTurnDetection; - - public SpeechToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - var turnDetectionEager = false; - for (SpeechOutputParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - } - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; - } - - public void inputAudio(byte[] rawAudioChunk) { - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)).build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + + private static final String TASK = ""; + + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; + + public SpeechToSpeechRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = false; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } } - - if (eagerTurnDetection) { - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } + } } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + } - @Override - protected String getSystemPrompt() { - return TASK; + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = + InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)) + .build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - @Override - protected SessionUpdateEvent sessionConfiguration() { - RealtimeAudioInputTurnDetection turnDetection; - if (eagerTurnDetection) { - turnDetection = null; - } else { - turnDetection = RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build() - ); - } + if (eagerTurnDetection) { + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + } + + @Override + protected String getSystemPrompt() { + return TASK; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + } - return SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest - .builder() - .outputModalities(OUTPUT_MODALITIES) - .audio(RealtimeAudioConfig.builder() + return SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() .input( RealtimeAudioConfigInput.builder() .turnDetection(turnDetection) @@ -121,27 +123,20 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioFormats.AudioPcm.builder() .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build() - ) - .build() - ) + .build()) + .build()) .output( RealtimeAudioConfigOutput.builder() .format( RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build() - ) + .build()) .voice(voice) - .build() - ) - .build() - ) - .build() - ) - .asRealtime() - ) + .build()) + .build()) + .build()) + .asRealtime()) .build(); - } + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index aacdf2a0a..a6a606da3 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -2,66 +2,66 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import java.util.*; - import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio_transcript.done" - ); + private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio_transcript.done"); - private static final String TASK = - "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + private static final String TASK = + "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; + private final TextOutputChannel outputConsumer; - private final TextOutputChannel outputConsumer; + public SpeechToTextRealtimeClient( + String url, Map httpHeaders, TextOutputChannel outputConsumer) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + } - public SpeechToTextRealtimeClient(String url, Map httpHeaders, TextOutputChannel outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; + public void inputAudio(byte[] rawAudioChunk) { + if (rawAudioChunk.length == 0) { + return; } - - public void inputAudio(byte[] rawAudioChunk){ - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)).build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; - } - - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); + var cursorLeft = 0; + while (cursorLeft < rawAudioChunk.length) { + var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + var audioInputMessage = + InputAudioBufferAppendEvent.builder() + .audio(Base64.getEncoder().encodeToString(part)) + .build(); + super.sendMessage(audioInputMessage); + cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; } - @Override - protected String getSystemPrompt() { - return TASK; - } + var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + super.sendMessage(commitAudioMessage); + askForResponse(); + } + + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio_transcript.done".equals(eventType)) { - this.outputConsumer - .outputText(event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); - } else { - log.trace("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio_transcript.done".equals(eventType)) { + this.outputConsumer.outputText( + event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); + } else { + log.trace("skipping message type: {}", eventType); } + } - @Override - protected SessionUpdateEvent sessionConfiguration() { + @Override + protected SessionUpdateEvent sessionConfiguration() { SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() .session( @@ -81,13 +81,14 @@ protected SessionUpdateEvent sessionConfiguration() { .build()) .transcription( AudioTranscription.builder() - .model(AudioTranscription.Model.GPT_4O_TRANSCRIBE) - .build()) + .model( + AudioTranscription.Model.GPT_4O_TRANSCRIBE) + .build()) .build()) .build()) .build()) .asTranscription()) .build(); - return sessionUpdateEvent; - } + return sessionUpdateEvent; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 1c0425bb4..ff482d553 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -2,19 +2,19 @@ /** * Functional interface representing test output channel (text data consumer) - *

    - * Should be closed by application (try-with-resources) when not needed anymore + * + *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { - /** - * This method is sequentially invoked by test data provider to supply implementer (consumer) - * with the test data. - * - * @param textChunk chunk of the text (possibly partial content) - * @param isLast true if this call logically concludes previous and this passed text data into a - * single logical entity (e.g. gets called at the end when all parts of a single - * message get passed) - */ - void outputText(CharSequence textChunk, boolean isLast); + /** + * This method is sequentially invoked by test data provider to supply implementer (consumer) with + * the test data. + * + * @param textChunk chunk of the text (possibly partial content) + * @param isLast true if this call logically concludes previous and this passed text data into a + * single logical entity (e.g. gets called at the end when all parts of a single message get + * passed) + */ + void outputText(CharSequence textChunk, boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index b42731278..a94b957c5 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -6,95 +6,100 @@ import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; -import lombok.extern.slf4j.Slf4j; - import java.util.*; +import lombok.extern.slf4j.Slf4j; @Slf4j class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private static final Set HANDLED_RESPONSE_TYPES = Set.of( - "response.output_audio.delta", "response.output_audio.done" - ); + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final List OUTPUT_MODALITIES = List.of( - RealtimeSessionCreateRequest.OutputModality.AUDIO - ); + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); - private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private static final String TASK = + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + private final boolean eagerTurnDetection; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - private final boolean eagerTurnDetection; - - public TextToSpeechRealtimeClient(String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - var turnDetectionEager = true; - for (SpeechOutputParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } + public TextToSpeechRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + SpeechOutputParam... params) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + this.outputConsumer = outputConsumer; + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + var turnDetectionEager = true; + for (SpeechOutputParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + case TURN_DETECTION -> { + if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; + } } + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + } - public void sendText(String text) { - var message = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemUserMessage( - RealtimeConversationItemUserMessage.builder() - .addContent(RealtimeConversationItemUserMessage.Content.builder() - .text(text) - .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); + public void sendText(String text) { + var message = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemUserMessage( + RealtimeConversationItemUserMessage.builder() + .addContent( + RealtimeConversationItemUserMessage.Content.builder() + .text(text) + .type(RealtimeConversationItemUserMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); - super.sendMessage(message); - if (eagerTurnDetection) { - askForResponse(); - } + super.sendMessage(message); + if (eagerTurnDetection) { + askForResponse(); } + } - @Override - protected String getSystemPrompt() { - return TASK; - } + @Override + protected String getSystemPrompt() { + return TASK; + } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } + @Override + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); } + } - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = SessionUpdateEvent.builder() + @Override + protected SessionUpdateEvent sessionConfiguration() { + SessionUpdateEvent sessionUpdateEvent = + SessionUpdateEvent.builder() .session( ClientSecretCreateParams.Session.ofRealtime( RealtimeSessionCreateRequest.builder() @@ -105,25 +110,28 @@ protected SessionUpdateEvent sessionConfiguration() { RealtimeAudioConfigInput.builder() .turnDetection(Optional.empty()) .format( - RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) .build()) .output( RealtimeAudioConfigOutput.builder() - .format(RealtimeAudioFormats.AudioPcm - .builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) + .build()) .voice(voice) .build()) .build()) .build()) .asRealtime()) .build(); - return sessionUpdateEvent; - } + return sessionUpdateEvent; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java index 5d8f571e6..a78f66d83 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java @@ -6,174 +6,182 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.common.ClientException; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.CompletableFuture; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = new ObjectMapper() - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider - * in cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of inactivity, - * this is normally not configurable and would leave realtime conversation unusable on a pause. - *

    - * In mobile networks this period is even shorter and typical CGNAT session can only idle for 10-15 seconds - * before it gets terminated - *

    - * In realtime client we rely on explicit lifetime management (create/close), heartbeat mechanism is required - * to prevent unintended closing of connections while conversation is still expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient(String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = wsBuilder.buildAsync(URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME,true); + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAIRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } + } + + @Override + public void close() { + WebSocket ws = this.ws.join(); + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 } + } - protected abstract String getSystemPrompt(); + protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract void onResponse(String eventType, JsonNode event); - protected abstract SessionUpdateEvent sessionConfiguration(); - - protected void sendMessage(Object message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - - } - } + protected abstract SessionUpdateEvent sessionConfiguration(); - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); + protected void sendMessage(Object message) { + WebSocket ws = this.ws.join(); + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } } - - private void onText(WebSocket webSocket, CharSequence data) { - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new ClientException("Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } else { - log.trace("Unhandled event type: {}", eventType); - } - - webSocket.request(1); + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new ClientException("Error parsing JSON response from speech API", e); } - - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); } - private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = sessionConfiguration(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize session request", e); - } + webSocket.request(1); + } - ws.request(1); + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - if (systemPrompt == null || systemPrompt.isEmpty()) { - return; - } - var systemConversationItem = ConversationItemCreateEvent.builder() - .item(ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent(RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT).build()).build() - )).build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - ws.request(1); + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = sessionConfiguration(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize session request", e); } - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = new TimerTask(){ - @Override - public void run() { - sendPing(ws); - } + ws.request(1); + } - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; } + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java index 0f6b0e26c..0f4ed8348 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/Application.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import javax.annotation.Nonnull; - import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.web.servlet.ServletComponentScan; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index ac7b07758..cff9b17c9 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -11,17 +11,18 @@ @EnableWebSocket public class WebsocketConfig implements WebSocketConfigurer { - private final TextToSpeechWebsocketHandler textToSpeech; - private final SpeechToSpeechWebsocketHandler speechToSpeech; + private final TextToSpeechWebsocketHandler textToSpeech; + private final SpeechToSpeechWebsocketHandler speechToSpeech; - public WebsocketConfig(TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { - this.textToSpeech = textToSpeech; - this.speechToSpeech = speechToSpeech; - } + public WebsocketConfig( + TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { + this.textToSpeech = textToSpeech; + this.speechToSpeech = speechToSpeech; + } - @Override - public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); - registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); - } + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); + registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index e593fb651..696d10fab 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -19,43 +19,47 @@ @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { - private final OpenAiService service; - private final Map channels; + private final OpenAiService service; + private final Map channels; - @Autowired - public SpeechToSpeechWebsocketHandler(OpenAiService service) { - this.service = service; - channels = new ConcurrentHashMap<>(); - } + @Autowired + public SpeechToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } - @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] chunkBytes = payload.array(); - AudioInputChannel channel = channels.computeIfAbsent( - session.getId(), - sessionId -> service.speechToSpeech((rawBytesChunk, isLast) -> { - try { + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] chunkBytes = payload.array(); + AudioInputChannel channel = + channels.computeIfAbsent( + session.getId(), + sessionId -> + service.speechToSpeech( + (rawBytesChunk, isLast) -> { + try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (IOException e) { log.error("failed to send audio data to realtime api", e); - } - }) - ); - channel.inputAudio(chunkBytes); - } + } + })); + channel.inputAudio(chunkBytes); + } - @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { - channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { - try { - inputChannel.close(); - } catch (Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); - } - return null; + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + throws Exception { + channels.computeIfPresent( + session.getId(), + (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; }); - super.afterConnectionClosed(session, status); - } - + super.afterConnectionClosed(session, status); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 27a667d3c..34469513f 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -2,6 +2,10 @@ import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; @@ -11,51 +15,51 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { - private final OpenAiService service; - private final Map channels; + private final OpenAiService service; + private final Map channels; - @Autowired - public TextToSpeechWebsocketHandler(OpenAiService service) { - this.service = service; - channels = new ConcurrentHashMap<>(); - } + @Autowired + public TextToSpeechWebsocketHandler(OpenAiService service) { + this.service = service; + channels = new ConcurrentHashMap<>(); + } - @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] textBytes = payload.array(); - TextInputChannel channel = channels.computeIfAbsent( - session.getId(), - sessionId -> service.textToSpeech((rawBytesChunk, isLast) -> { - try { + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + ByteBuffer payload = message.getPayload(); + byte[] textBytes = payload.array(); + TextInputChannel channel = + channels.computeIfAbsent( + session.getId(), + sessionId -> + service.textToSpeech( + (rawBytesChunk, isLast) -> { + try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (IOException e) { log.error("failed to send text message to realtime api", e); - } - })); - channel.sendText(new String(textBytes)); - } + } + })); + channel.sendText(new String(textBytes)); + } - @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) throws Exception { - channels.computeIfPresent(session.getId(), (sessionId, inputChannel) -> { - try { - inputChannel.close(); - } catch (Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); - } - return null; + @Override + public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + throws Exception { + channels.computeIfPresent( + session.getId(), + (sessionId, inputChannel) -> { + try { + inputChannel.close(); + } catch (Exception e) { + log.warn("failed to close input channel for session {}", sessionId, e); + } + return null; }); - super.afterConnectionClosed(session, status); - } - + super.afterConnectionClosed(session, status); + } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index ad25c15a5..4bc64655b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -6,7 +6,6 @@ import com.sap.ai.sdk.core.AiCoreService; import com.sap.ai.sdk.foundationmodels.openai.*; - import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -70,8 +69,8 @@ public Stream streamChatCompletionDeltas( /** * Creates realtime channel allowing to input text and voice it (receive audio output) * - *

    The input channel should be used with a try-with-resources block to ensure that the underlying - * connection is closed. + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. * *

    Example: * @@ -82,11 +81,11 @@ public Stream streamChatCompletionDeltas( * } * } * - * This API implements full duplex (input + output) communication channels. Application should logically - * synchronize their state and close input channel when it is appropriate (e.g. last part of the response - * has been received via output channel and application does not need to send any other input). When input - * channel is closed, output channel will be closed automatically and output consumer will not be called - * anymore. + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output * @return input channel, allowing for text input diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 988ae7031..b9098335b 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -1,47 +1,49 @@ package com.sap.ai.sdk.app.controllers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - import java.io.ByteArrayOutputStream; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; @Slf4j public class RealtimeApiTest { - @Test - @Timeout(value = 30, unit = TimeUnit.SECONDS) - void testSpeechToSpeech() { - var outputBuffer = new ByteArrayOutputStream(300000); - var monitor = new CountDownLatch(1); - var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + @Test + @Timeout(value = 30, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); - try (TextInputChannel input = client.textToSpeech((byteChunk, last) -> { - outputBuffer.writeBytes(byteChunk); - if (last) { + try (TextInputChannel input = + client.textToSpeech( + (byteChunk, last) -> { + outputBuffer.writeBytes(byteChunk); + if (last) { monitor.countDown(); - } - }, SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { - input.sendText("Ordnung muss sein!"); - monitor.await(); - } catch (Exception e) { - if (!(e instanceof InterruptedException)) { - fail(e); - } - // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is reached - return; - } - assertThat(monitor.getCount()).isEqualTo(0); - assertThat(outputBuffer.size()).isGreaterThan(0); + } + }, + SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.sendText("Ordnung muss sein!"); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is + // reached + return; } - -} \ No newline at end of file + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + } +} From 5415795a6c1292a147ffaf31ed5d6725cfd816b7 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:32:42 +0200 Subject: [PATCH 12/67] address pr feedback, add tests --- ...echOutputParam.java => RealtimeParam.java} | 7 +- .../sdk/core/RealtimeParamTurnDetection.java | 40 ++++ .../sap/ai/sdk/core/RealtimeParamVoice.java | 45 ++++ .../model/SpeechOutputParamTurnDetection.java | 35 --- .../core/model/SpeechOutputParamVoice.java | 36 ---- .../openai/BufferedWebSocketListener.java | 10 +- .../foundationmodels/openai/OpenAiClient.java | 88 +------- .../openai/OpenAiRealtimeClient.java | 113 ++++++++++ .../openai/SpeechToSpeechRealtimeClient.java | 20 +- .../openai/SpeechToTextRealtimeClient.java | 2 +- .../openai/TextOutputChannel.java | 6 +- .../openai/TextToSpeechRealtimeClient.java | 25 ++- .../openai/ToAudioRealtimeClient.java | 33 +++ .../openai/WSSOpenAiRealtimeClient.java | 204 ++++++++++++++++++ .../orchestration/OrchestrationAiModel.java | 3 - .../com/sap/ai/sdk/app/WebsocketConfig.java | 4 +- .../ai/sdk/app/services/OpenAiService.java | 41 +++- .../src/main/resources/static/index.html | 2 +- .../sdk/app/controllers/RealtimeApiTest.java | 115 +++++++++- .../src/test/resources/fixtures/question.pcm | Bin 0 -> 97200 bytes 20 files changed, 633 insertions(+), 196 deletions(-) rename core/src/main/java/com/sap/ai/sdk/core/{model/SpeechOutputParam.java => RealtimeParam.java} (60%) create mode 100644 core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java create mode 100644 core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java delete mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java delete mode 100644 core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java create mode 100644 sample-code/spring-app/src/test/resources/fixtures/question.pcm diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java similarity index 60% rename from core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java rename to core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index d9c372cbd..bb4cd79c4 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -1,8 +1,8 @@ -package com.sap.ai.sdk.core.model; +package com.sap.ai.sdk.core; import javax.annotation.Nonnull; -public interface SpeechOutputParam { +public interface RealtimeParam { enum SpeechOutputParamName { VOICE, TURN_DETECTION, @@ -10,4 +10,7 @@ enum SpeechOutputParamName { @Nonnull SpeechOutputParamName getParamName(); + + @Nonnull + String getValueAsString(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java new file mode 100644 index 000000000..60f736fd9 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -0,0 +1,40 @@ +package com.sap.ai.sdk.core; + +import java.util.Objects; +import org.jspecify.annotations.NonNull; + +public final class RealtimeParamTurnDetection implements RealtimeParam { + + public static final RealtimeParamTurnDetection BY_MODEL_AUTO = + new RealtimeParamTurnDetection("BY_MODEL_AUTO"); + public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = + new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); + + private final String turnDetectionKind; + + private RealtimeParamTurnDetection(String turnDetectionKind) { + this.turnDetectionKind = turnDetectionKind; + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.TURN_DETECTION; + } + + @Override + public @NonNull String getValueAsString() { + return turnDetectionKind; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RealtimeParamTurnDetection that = (RealtimeParamTurnDetection) o; + return Objects.equals(turnDetectionKind, that.turnDetectionKind); + } + + @Override + public int hashCode() { + return Objects.hashCode(turnDetectionKind); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java new file mode 100644 index 000000000..10fc818a8 --- /dev/null +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -0,0 +1,45 @@ +package com.sap.ai.sdk.core; + +import java.util.Objects; +import javax.annotation.Nonnull; +import org.jspecify.annotations.NonNull; + +public final class RealtimeParamVoice implements RealtimeParam { + + public static final RealtimeParamVoice DEFAULT_1 = + new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_2 = + new RealtimeParamVoice("DEFAULT_1"); + + private final String voice; + + private RealtimeParamVoice(@Nonnull String voice) { + this.voice = voice; + } + + public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { + return new RealtimeParamVoice(voiceName); + } + + @Override + public @NonNull SpeechOutputParamName getParamName() { + return SpeechOutputParamName.VOICE; + } + + @Override + public @NonNull String getValueAsString() { + return voice; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RealtimeParamVoice that = (RealtimeParamVoice) o; + return Objects.equals(voice, that.voice); + } + + @Override + public int hashCode() { + return Objects.hashCode(voice); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java deleted file mode 100644 index 1a0e60d27..000000000 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamTurnDetection.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.sap.ai.sdk.core.model; - -import java.util.Objects; -import org.jspecify.annotations.NonNull; - -public final class SpeechOutputParamTurnDetection implements SpeechOutputParam { - - public static final SpeechOutputParamTurnDetection BY_MODEL_AUTO = - new SpeechOutputParamTurnDetection("BY_MODEL_AUTO"); - public static final SpeechOutputParamTurnDetection EACH_CALL_IS_A_TURN = - new SpeechOutputParamTurnDetection("EACH_CALL_IS_A_TURN"); - - private final String value; - - private SpeechOutputParamTurnDetection(String value) { - this.value = value; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.TURN_DETECTION; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamTurnDetection that = (SpeechOutputParamTurnDetection) o; - return Objects.equals(value, that.value); - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } -} diff --git a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java deleted file mode 100644 index 35f4f5eb2..000000000 --- a/core/src/main/java/com/sap/ai/sdk/core/model/SpeechOutputParamVoice.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.sap.ai.sdk.core.model; - -import java.util.Objects; -import javax.annotation.Nonnull; -import org.jspecify.annotations.NonNull; - -public final class SpeechOutputParamVoice implements SpeechOutputParam { - - public static final SpeechOutputParamVoice DEFAULT_MAN = - new SpeechOutputParamVoice("DEFAULT_MAN"); - public static final SpeechOutputParamVoice DEFAULT_WOMAN = - new SpeechOutputParamVoice("DEFAULT_WOMAN"); - - private final String voice; - - private SpeechOutputParamVoice(@Nonnull String voice) { - this.voice = voice; - } - - @Override - public @NonNull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.VOICE; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - SpeechOutputParamVoice that = (SpeechOutputParamVoice) o; - return Objects.equals(voice, that.voice); - } - - @Override - public int hashCode() { - return Objects.hashCode(voice); - } -} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 8d29cb358..1eb449229 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -9,13 +9,13 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class BufferedWebSocketListener implements WebSocket.Listener { +class BufferedWebSocketListener implements WebSocket.Listener { private final Consumer onOpen; private final BiConsumer onText; private final StringBuilder buffer; - public BufferedWebSocketListener( + BufferedWebSocketListener( Consumer onOpen, BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; @@ -29,10 +29,10 @@ public void onOpen(WebSocket webSocket) { } @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean isLast) { buffer.append(data); webSocket.request(1); - if (last) { + if (isLast) { var completeMessage = buffer.toString(); buffer.setLength(0); this.onText.accept(webSocket, completeMessage); @@ -47,7 +47,7 @@ public void onError(WebSocket webSocket, Throwable error) { } @Override - public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean isLast) { log.warn("Received unexpected binary bytes for WebSocket connection"); return CompletableFuture.completedStage(null); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 00f803d3d..3402e39d8 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -15,7 +15,6 @@ import com.sap.ai.sdk.core.common.ClientStreamingHandler; import com.sap.ai.sdk.core.common.RequestLogContext; import com.sap.ai.sdk.core.common.StreamedDelta; -import com.sap.ai.sdk.core.model.SpeechOutputParam; import com.sap.ai.sdk.foundationmodels.openai.generated.model.ChatCompletionStreamOptions; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CreateChatCompletionResponse; @@ -52,8 +51,6 @@ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class OpenAiClient { private static final String DEFAULT_API_VERSION = "2024-02-01"; - private static final int PATH_BUFFER_SIZE = - 400; // existing URLs are ~120 symbols long, 400 has reasonable margin private static final ObjectMapper JACKSON = getOpenAiObjectMapper(); @@ -79,6 +76,17 @@ public static OpenAiClient forModel(@Nonnull final OpenAiModel foundationModel) return client.withApiVersion(DEFAULT_API_VERSION); } + /** + * Creates and configures OpenAI Realtime API client + * + * @return created client + */ + @Beta + public static OpenAiRealtimeClient realtimeClient() { + var withResolvedDestination = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + return new OpenAiRealtimeClient(withResolvedDestination.destination); + } + /** * Create a new OpenAI client targeting the specified API version. * @@ -264,80 +272,6 @@ public Stream streamChatCompletion(@Nonnull final String prompt) .map(OpenAiChatCompletionDelta::getDeltaContent); } - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
-   *       textInputChannel.sendText("...");
-   *       ....
-   * }
-   * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output - * @return input channel, allowing for text input - */ - @Nonnull - public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, SpeechOutputParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); - } - - // unstable, requires further debugging and testing - AudioInputChannel speechToText(@Nonnull final TextOutputChannel textOutputConsumer) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint() + "?intent=transcription"; - - return new SpeechToTextRealtimeClient(endpoint, headers, textOutputConsumer); - } - - public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputChannel, SpeechOutputParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputChannel, params); - } - - private String getRealtimeEndpoint() { - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.asHttp().getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException( - "Invalid destination URI: " + destination.asHttp().getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - return sb.toString(); - } - private static void throwOnContentFilter(@Nonnull final OpenAiChatCompletionDelta delta) { final String finishReason = delta.getFinishReason(); if (finishReason != null && finishReason.equals("content_filter")) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java new file mode 100644 index 000000000..f1923347e --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -0,0 +1,113 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.google.common.annotations.Beta; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; +import com.sap.cloud.sdk.cloudplatform.connectivity.Header; + +import javax.annotation.Nonnull; +import java.util.HashMap; + +public class OpenAiRealtimeClient { + + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin + + private final Destination destination; + + OpenAiRealtimeClient(Destination destination) { + this.destination = destination; + } + + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+     * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+     *       textInputChannel.sendText("...");
+     *       ....
+     * }
+     * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @return input channel, allowing for text input + */ + @Nonnull + @Beta + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + + + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+     * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+     *       audioInputChannel.inputAudio(audioBytesData);
+     *       ....
+     * }
+     * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @param params - optional configuration params + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + */ + @Nonnull + @Beta + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); + } + var endpoint = getRealtimeEndpoint(); + + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } + +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3202f1c1f..59fab1a02 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.model.SpeechOutputParam; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; -import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -31,25 +31,25 @@ public SpeechToSpeechRealtimeClient( String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = false; - for (SpeechOutputParam param : params) { + for (RealtimeParam param : params) { switch (param.getParamName()) { case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; } } case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { turnDetectionEager = false; } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index a6a606da3..377e43625 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToTextRealtimeClient extends WSSOpenAIRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index ff482d553..5896b85ab 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,15 +1,15 @@ package com.sap.ai.sdk.foundationmodels.openai; /** - * Functional interface representing test output channel (text data consumer) + * Functional interface representing text output channel (text data consumer) * *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { /** - * This method is sequentially invoked by test data provider to supply implementer (consumer) with - * the test data. + * This method is sequentially invoked by text data provider to supply implementer (consumer) with + * the text data. * * @param textChunk chunk of the text (possibly partial content) * @param isLast true if this call logically concludes previous and this passed text data into a diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index a94b957c5..4ad9f6918 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.model.SpeechOutputParam; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; -import com.sap.ai.sdk.core.model.SpeechOutputParamVoice; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -21,7 +21,8 @@ class TextToSpeechRealtimeClient extends WSSOpenAIRealtimeClient implements Text List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input"; + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + + "do not answer questions, just read them"; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; @@ -31,24 +32,26 @@ public TextToSpeechRealtimeClient( String url, Map httpHeaders, AudioOutputChannel outputConsumer, - SpeechOutputParam... params) { + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); this.outputConsumer = outputConsumer; var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = true; - for (SpeechOutputParam param : params) { + for (RealtimeParam param : params) { switch (param.getParamName()) { case VOICE -> { - if (SpeechOutputParamVoice.DEFAULT_WOMAN.equals(param)) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (SpeechOutputParamVoice.DEFAULT_MAN.equals(param)) { + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } else { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.of(param.getValueAsString().toLowerCase()); } } case TURN_DETECTION -> { - if (SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; - } else if (SpeechOutputParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { turnDetectionEager = false; } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java new file mode 100644 index 000000000..785414a0f --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -0,0 +1,33 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.SessionUpdateEvent; + +import java.util.Map; +import java.util.Set; + +class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { + + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + private static final String DEFAULT_SYSTEM_PROMPT = ""; + + public ToAudioRealtimeClient(String url, Map httpHeaders) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + } + + @Override + protected String getSystemPrompt() { + return ""; + } + + @Override + protected void onResponse(String eventType, JsonNode event) { + + } + + @Override + protected SessionUpdateEvent sessionConfiguration() { + return null; + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java new file mode 100644 index 000000000..213725452 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -0,0 +1,204 @@ +package com.sap.ai.sdk.foundationmodels.openai; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.*; +import com.sap.ai.sdk.core.common.ClientException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +abstract class WSSOpenAiRealtimeClient implements AutoCloseable { + + private static final int SUCCESS_FINISH_WSS_CODE = 1000; + private static final ObjectMapper JACKSON = + new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + + /** + * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in + * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of + * inactivity, this is normally not configurable and would leave realtime conversation unusable on + * a pause. + * + *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for + * 10-15 seconds before it gets terminated + * + *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat + * mechanism is required to prevent unintended closing of connections while conversation is still + * expected to continue + */ + private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; + + private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; + + private final HttpClient client; + private final CompletableFuture ws; + private final Timer heartbeatTimer; + private final Set handleMessageTypes; + + public WSSOpenAiRealtimeClient( + String url, Map httpHeaders, Set handleMessageTypes) { + this.client = HttpClient.newHttpClient(); + var wsBuilder = this.client.newWebSocketBuilder(); + for (Map.Entry entry : httpHeaders.entrySet()) { + wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); + } + this.ws = + wsBuilder.buildAsync( + URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); + this.handleMessageTypes = handleMessageTypes; + this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); + } + + public void askForResponse() { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize ask for response", e); + } + ws.request(1); + } + } + + @Override + public void close() { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + heartbeatTimer.cancel(); + try { + ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); + } catch (Exception e) { + log.error("Error while closing WebSocket", e); + } + // this.client.close(); // exists only since java 21 + } + } + + protected abstract String getSystemPrompt(); + + protected abstract void onResponse(String eventType, JsonNode event); + + protected abstract SessionUpdateEvent sessionConfiguration(); + + protected void sendMessage(Object message) { + WebSocket ws; + try { + ws = this.ws.join(); + } catch (CompletionException e) { + throw new ClientException("failed to establish web socket connection", e); + } + synchronized (this) { + try { + ws.sendText(JACKSON.writeValueAsString(message), true); + ws.request(1); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + } + } + + private synchronized void onSocketOpen(WebSocket ws) { + configureSession(ws); + configureConversation(ws); + scheduleHeartbeat(ws); + } + + private void onText(WebSocket webSocket, CharSequence data) { + final JsonNode event; + try { + event = JACKSON.readTree(data.toString()); + } catch (JsonProcessingException e) { + throw new ClientException("Error parsing JSON response from speech API", e); + } + var eventType = event.get("type").asText(); + if (handleMessageTypes.contains(eventType)) { + onResponse(eventType, event); + } else { + log.trace("Unhandled event type: {}", eventType); + } + + webSocket.request(1); + } + + private synchronized void sendPing(WebSocket ws) { + if (ws.isInputClosed()) { + return; + } + ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.request(1); + } + + private void configureSession(WebSocket ws) { + SessionUpdateEvent sue = sessionConfiguration(); + try { + ws.sendText(JACKSON.writeValueAsString(sue), true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize session request", e); + } + + ws.request(1); + } + + private void configureConversation(WebSocket ws) { + var systemPrompt = getSystemPrompt(); + if (systemPrompt == null || systemPrompt.isEmpty()) { + return; + } + var systemConversationItem = + ConversationItemCreateEvent.builder() + .item( + ConversationItem.ofRealtimeConversationItemSystemMessage( + RealtimeConversationItemSystemMessage.builder() + .addContent( + RealtimeConversationItemSystemMessage.Content.builder() + .text(systemPrompt) + .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) + .build()) + .build())) + .build(); + + try { + var json = JACKSON.writeValueAsString(systemConversationItem); + ws.sendText(json, true); + } catch (JsonProcessingException e) { + throw new ClientException("Failed to serialize message", e); + } + ws.request(1); + } + + private void scheduleHeartbeat(WebSocket ws) { + TimerTask task = + new TimerTask() { + @Override + public void run() { + sendPing(ws); + } + }; + heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java index 15058cc5e..1345deaba 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationAiModel.java @@ -346,9 +346,6 @@ public class OrchestrationAiModel { /** Azure OpenAI GPT-5-mini model */ public static final OrchestrationAiModel GPT_5_MINI = new OrchestrationAiModel("gpt-5-mini"); - /** Azure OpenAI GPT-realtime model */ - public static final OrchestrationAiModel GPT_REALTIME = new OrchestrationAiModel("gpt-realtime"); - /** Azure OpenAI GPT-5-nano model */ public static final OrchestrationAiModel GPT_5_NANO = new OrchestrationAiModel("gpt-5-nano"); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index cff9b17c9..9273342c8 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -22,7 +22,7 @@ public WebsocketConfig( @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { - registry.addHandler(textToSpeech, "/text-to-speech").setAllowedOrigins("*"); - registry.addHandler(speechToSpeech, "/speech-to-speech").setAllowedOrigins("*"); + registry.addHandler(textToSpeech, "/text-to-speech"); + registry.addHandler(speechToSpeech, "/speech-to-speech"); } } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index 4bc64655b..ad6c92c2a 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,7 +5,18 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; -import com.sap.ai.sdk.foundationmodels.openai.*; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingRequest; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiEmbeddingResponse; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -92,11 +103,35 @@ public Stream streamChatCompletionDeltas( */ @Nonnull public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).textToSpeech(audioOutputConsumer); + return OpenAiClient.realtimeClient().textToSpeech(audioOutputConsumer); } + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+   *       audioInputChannel.inputAudio(audioBytesData);
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + */ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { - return OpenAiClient.forModel(OpenAiModel.GPT_REALTIME).speechToSpeech(audioOutputConsumer); + return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); } /** diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 70a5ffe2e..014fddcf1 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1453,7 +1453,7 @@

    📂 Batch API

    ⏰ Realtime API

    Realtime API allows for various real time interactions TBD Documentation
    diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index b9098335b..8b73f7c77 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -3,36 +3,54 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import com.sap.ai.sdk.core.model.SpeechOutputParamTurnDetection; +import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiModel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @Slf4j public class RealtimeApiTest { + private static final double LOG_2 = Math.log(2.0); + + private static byte[] QUESTION_FIXTURE_PCM; + + @BeforeAll + public static void setUp() { + try (var fis = new FileInputStream( "src/test/resources/fixtures/question.pcm")) { + QUESTION_FIXTURE_PCM = fis.readAllBytes(); + } catch (IOException e) { + fail(e.getMessage()); + } + } + @Test @Timeout(value = 30, unit = TimeUnit.SECONDS) - void testSpeechToSpeech() { + void testTextToSpeech() { var outputBuffer = new ByteArrayOutputStream(300000); var monitor = new CountDownLatch(1); - var client = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + var client = OpenAiClient.realtimeClient(); try (TextInputChannel input = client.textToSpeech( - (byteChunk, last) -> { + (byteChunk, isLast) -> { outputBuffer.writeBytes(byteChunk); - if (last) { + if (isLast) { monitor.countDown(); } }, - SpeechOutputParamTurnDetection.EACH_CALL_IS_A_TURN)) { + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN)) { input.sendText("Ordnung muss sein!"); monitor.await(); } catch (Exception e) { @@ -45,5 +63,88 @@ void testSpeechToSpeech() { } assertThat(monitor.getCount()).isEqualTo(0); assertThat(outputBuffer.size()).isGreaterThan(0); + + var metrics = pcm16AudioMetrics(outputBuffer.toByteArray()); + + // root mean squire (measures the deviation from an average value) + assertThat(metrics.rms).isGreaterThan(500d); + // asserts that variety of deviation is sufficient (not a trivial repeating pattern) + assertThat(metrics.entropy).isGreaterThan(4); + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + void testSpeechToSpeech() { + var outputBuffer = new ByteArrayOutputStream(300000); + var monitor = new CountDownLatch(1); + var client = OpenAiClient.realtimeClient(); + + try (AudioInputChannel input = + client.speechToSpeech( + (byteChunk, isLast) -> { + outputBuffer.writeBytes(byteChunk); + if (isLast) { + monitor.countDown(); + } + }, + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN)) { + input.inputAudio(QUESTION_FIXTURE_PCM); + monitor.await(); + } catch (Exception e) { + if (!(e instanceof InterruptedException)) { + fail(e); + } + // do nothing, test has either been interrupted by user (intended) or by jupiter if timeout is + // reached + return; + } + + try (var fis = new FileOutputStream("/tmp/response.pcm")) { + fis.write(outputBuffer.toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private record AudioMetrics(double rms, double entropy){}; + + /** + * Audio quality metrics for signed 16-bit little-endian PCM, computed on the mean-subtracted + * (DC-offset-removed) signal so they reflect only the varying, audible part of the audio. + * @param pcm - The raw PCM audio buffer. + * @return The RMS amplitude and Shannon entropy (bits) of the zero-mean samples. + */ + AudioMetrics pcm16AudioMetrics(byte[] pcm) { + if (pcm.length < 1) { + return new AudioMetrics(0, 0); + } + var sum = 0L; + for (var i = 0; i < pcm.length; i+=2) { + sum += (pcm[i]) | ((pcm[i+1]) << 8); + } + var mean = sum / (pcm.length / 2); + + var sumOfSquares = 0d; + var counts = new char[Character.MAX_VALUE]; + for (var i = 0; i < pcm.length; i+= 2) { + var residual = ((pcm[i]) | ((pcm[i+1]) << 8)) - mean; + sumOfSquares += residual * residual; + var key = (int) (residual + Short.MAX_VALUE + 1); + counts[key]++; + } + + var rms = Math.sqrt(sumOfSquares / (double) ( pcm.length / 2 )); + var entropy = 0d; + for (var i = 0; i < counts.length; i++) { + var p = counts[i] / counts.length; + entropy -= p * log2(p); + } + + return new AudioMetrics(rms, entropy); + } + + private static double log2(double logNumber) { + return Math.log(logNumber) / LOG_2; } } diff --git a/sample-code/spring-app/src/test/resources/fixtures/question.pcm b/sample-code/spring-app/src/test/resources/fixtures/question.pcm new file mode 100644 index 0000000000000000000000000000000000000000..d0707295bf8098bee8e97e78b77d3ac79a22629b GIT binary patch literal 97200 zcmYhj1$Z1eur8|RVIH$%JC2h$%*@OT8|G}7nVE6J%&=kRWW&tNhL}A}&FZH~&bjA} zzHY(vNL8g$Nv*#$0N^M9pePu?Pz=!hPpl}i0+sORPyY9pzi$#M3;!bgZ8#3!|9t+W z|F0zc%Rk?SQ_}BWgzw>+!&mqo{v}i-l#)>TPuZW8jLO1PgntpPC){qhEZpv&(s1g3 zYx`5Li1Oju!uS6_UEx;#Q1ho8L-mFE28eqMwMqj5r73jB0EuD@l@sXtQxg8=pDQe# zaC!I}3f=!a_otpeyrO3RpFG0l;ryRZn42(%;nHv{|CVgH_3$@}BA_@dr7(8@{E-j; zn-@v>-#EE6n*HIl=$QhvgNf;s06%zd8LwYncAwkZvM^96zRQ3Cp#Ba*NwDAgyI1egb#r-?vD>+K8lBHxS+33nZ;ivLL zNk`!aN~fVXL-~c`U+9?}#i`^gF7!Lk|J(zpCjK{9f4Cq}n;2?Y^_vPcqD_m^24F&Y z6EFe`y3D|a(q>>qIUBm0P_UtNIB!QG0_DRw!|!K|=$A&6HlmsgD6Rd?vj){lA-;8p zUlnREtTzP8<0$^qH{5SI3krvNS5iWVN=^x&E2Q|41pFu+zPr&C4j~lS-yt0HD951? z9w%Yl`ZE?OB-d~sgo6sT748A%|K-DchkH2OUJ%tDZpVY9>_8zO$@UkL<9Fq=@=$j0KayOt2WN0BgV|upR6JN5Bbi0bB&v!CmkeJOKAm zxQ)V1RCXL3M9=&KmVy~zH0X`$sRK%&Uci7``KsJe&M14770L`{l+sISt29>1DG5rf zVp2#Y1Ou=T=D-a24gNsaXOzl>SuhtCKrakJ2{MXViBgi3@=6`0xza`XTbZUTQ8p+? zl^clHOvDid;z32w1oQ&q!9uVB(Q*kq1@Ay6a03Y#FdLSDrC`Oe(pWjHGFBd4<*+hX z1uPkbL@X9FV1?8}^ zNm;CnR|Y9Plx9i|B?Zx_RyaiU7x)z3gs0#^xE*eQtKlLz7tV$=;4C;FE{4nCdbkb# z3(vy4@ICwmS!h7K)m6GGVocII#`FK{sd$I4pf!$yg zSb%zWFzAdVRsnHkL0n}ct-6C+-;Q`1hG?m$#3_v8gfHMlMC}4N7WRhCU=^4Gqo5X< zM@e?expKCgA^(zp%HQNP6z8DelKpZ}hDhacXoX3z46F`Y!hvuq+z8LXSI`Z$sPDTg zGnIqN3#8d`pcT@Xqu?zdu&US)Y%SvG7pB20<2~?M_%8ek{t@7z4^!Js)PiL8(Vjij^6W@Vl-P-&u+P;^Qme1J4~8T=bI zhY1jadGd4ljJ#J~EKib0$X(?Ya(%guTnk;zP-r7}k^9QS<>~S=c^!J{uzXg&fx>P1 zKl#3VSH33SLh)@BUzhKpvb*vV`KA0`PM1AMTjF3tB&Ac(32P!6vXshTH5!Y}u|1fK zwZ*sLId}rbm{4J9$YN~-<3LwX1!#H>faZL z%k%O9d852k9xZp5>&iuBwOk-Qkq$@;rIAuQsg6`eij|BKC5fV6grZi8mdZAO?|$*=-E3`?MK*$G*)>6nB~!v%ae@sLO$r;^u6jx0-cq9#+TseRNj>NItYI!^7O z{-I`5BT;!xDw<-+m*iG*3|WuVlAnqF!~~)?K@iXJEl4XW;34cWwi4@x6~%JFRb(~C zfo5nfU_jn)3wIXT*PTcz^OO-vU!^^=*|n8qWa0AQeWa;Nkwogj25Zh3C^AVry^??}}tmkvu{QWKC*3 zbs0^pQFL{>DGCGVfpkkcg=VQQ)M07@Rf8%Zk0QNEMjCR67(x^y{P+ue2R<3EkE`$> z*m-1m2B8^20BPVWxCf3QJGmWM-+5>T7zPG`4rqoX!87HMGGFPfR8SzAtFFNva3LHA z`ysIBumnJ>8f;0Iw);H91WBjOGPDC z%n&b$OU0pLC6N@L2s?y@LN}qIP*Er=lo6^49fX;}R^gfu6v~N{#M@#;>5NnY+1rb7 zyRr}r#`@z8h$PZQuBD38`{^poM&<7oW2Ci`8{8B z0Bt}`5Dh%YTAV~P@NlJ*(gtaTMbRj^@E*K`tm6zMkCteJBtafojEl%#%t6}GTqflw z(lcqFv>tI^7x8Y8tP+ski5JC{;t;W>s1`p8`-GuFtdPxL;kWQ>_CO7xm07+$JM{oMom-A zD2-WjLETG@tBb^wzc5?)OMv_vel_2pPvtYX z3tU?+g8R&#Vb8Hk*rV)jc02o;EzRxW(z)*Z13p4{B6JXAB#(4XJ_5HYm%w$*OdKUQ zQtxP%`Ju{E@6i0Em9=Aa%XH;*RdfluOWM)e|1>|;)zxcNX^ewzOi!hzlaXW>Vja%l zqp=(`Lv{qWm2OH?^V+1XM`>o(~I=puC; zw7Hs_>g6gwvxiL z;*ceFz_w_Gl&MSyKF|;IVnfl&q%Cn8jlfx?j_OG*qkd8hJ(ylbPo;a%3KdUBAUVG$ zGm(~^M9S59Cyfw$35WT7t{(TC?ZWy(+e1A=rqIRUlHk~2yI`fDHTWVh zD=;-MI?yW6E>I>A9hex{9yk#w3``6D6^dkO?iM$aKOtm_ljMrZGcX)~MNFqQGyBvQ zU5;LD%rZ_f-83Jt+_E~XO)dYJ=bC)R6UGhtm)fgpH#3FaMAjr?ux?6gXpv>{s-PF< z^AWt8JIy8XrT9pGHh+iTAdC}hN&iVNqzUpuxLg?kqOf;Zd%ObCn5<4k)7R<7jFTCs zx~J-|ey!%!d1|#LNBxibih7{>u_{H?oVh@|sT1Ttq7-h!oT&G;a%J&7|B>TZnw=3e z1-kkjzW2U^z6L(Rm*uVLJLT)-|LHFh7!=qLs1$q@936TRS{phSl0x;_nQUve3|k+) zV9nt6aS6PYf6X@*&q%U-UO9;k!%veVn1gDYzJsxmDZ(7I6t~^7KaX_UKij_9Zdh|I z5#}62aa}ddXjL;Rnpg~W$Rnlu!W8ZclKc0-Q-7L&4|<8a6nGYh2_6mdp~GBBp@Fzj z%#)5lQ9-M2yfT54PpPgDcGk;F#yw;dtm6;9TkY?r!Tn?R(*W z8psN2*x}p_o)xBx)6o1dK>8((kS8Jwnk)}S^JqJ{47@3?fd+6_!ST(+C$cn?rmC)u z(+@L`vL3WV+h$mw*v3Uxj*PZdG2Jyh(Jj-SRF5K;gZc7aVKNsVdgcG&v3YK}S?44d z^Vu{N+SwbZu!GCwh0GuGC(*63ATsA2dqNQ&Lq zasJ~T(z!kVL(b%^LBIB7dNL|!PR;!MOPA9suSj7Hr|1s(e8D*0E)I}a!`8|>;Kn38 z8?TMm!JeSE`GLw;_#d(hb40!PO{guL6LyI$kd1F6Tb0&$KHXn+NPS*g&R8sJN!-`y zZ+3%ybHu5La#0qmQ(c*|V62qHp9y?%!@@Ip)~xB7s_anCmfV^7&kH8F5`!!F?$S{u zl^UU)Xnba>Yb;^rY&9c$MRl{UF;6okn_HT97=P&7>uzgV%|`WO^)HR6P1H5iw$v2W ztkX2nY*1%0|4}aq^gbecxCx=Hfrs8HuJr{{?%-S|@1MK}c}w%_7QS?@^i~db=F-Ft zFc)kg&Qexp0JDTPQ~5+!yaV_mZxp7o3Bkv{GoJJAsxHbo+BpX;nLB!BdQ`r@gXQ^t zat*u#5y{ll-O)d>!`RIDj)~Wc42kX)u_>-(T%Cwg=JRB0_)Yfks{*Ax0}CEyS+d(_ zoz6U&{Vk6y$afU?{Nu0Ay^}sE(}{~rSmQH(- zzU9}=oN)z}Jd*-{aUJDb$}HSP4WVz-ewwGt(X+{XObr)HL&Z@-jIe}#61e4i?928q z38sY}2lobgvhh5|+oh)P7CcTaVG1aRW|m>Laed^a=(e`Q=5EF`=3lZeZ&bH5BaEG`NfCAJea!7l@60bu_w_lt zmHM7W-n7WjPCwPu+}1H_WbDA$#<6o_Z^UemdK?+FS2ADLmLW;m#SZb`alJ3No!vCE zdFGQ$LpI23TCm9Z)ZdTKln!F)z=B~{-jg6N9%hA+8^p|w7n=ajdtXM~&d zI2|$03~x36bKktc3ZK#4(BIondK-k61Vqm`wm_^Ro2W&aPuhji)r)7P3`^xpI1=|2 zJ5eH`$Rb-^ZBye9>fj4EokI!G>Jm8+i?gb+>qH(#gn9*)s z9^tfJ)|F(MszzxX+ArDz>fVMsw(^m8<7cN#D}AJ*wo>nM8_V`9U$Jce;v?g_8zr)m zG}<#JEA_kjjstvy#DCWFaI20JNBZ`Te>e77a9j|20jHN{M9|U`|m<)L9X+H>z(JM;E`7o zZ%s8L=12Y&w=b5BQ>Sc99u}Qw-b8K{nh5U$Kk|;JKmSzat?To-uOGkk{t46X6_n*B zz-3r1X0-mMrCAIfJ1L@}y{2WKv7z~hVWUQ)+hn+Bd}#IBheypyD3(&XxGOcX)UxE> zNq1rwoBC;*5SfAc`76`jzK(p^>-CA}(VyraFTUK&p6l|2?rF+Ic8D#XxHdtE-V%E> zk&Lfz&thtGz5V&V&#p}ct8zbO-OhdFIO1C6E5{!bi^z|$1Y(UcRm=!>4{T=7aa3S~ zdy;FUe;#blRMwp`wY7}0{T*XSIGx~%>tX$m=*=DU>hnuxFrUi4Z20W9xj2_EiacO6FCVTccgA zA=SrM%t>_@T@X#`zr#1qnD5`8*ZQya<%lb-uYbRH_|3jgru1gc``kEv#nhsehLo>b zPF<{C5o^lzxO{UvZI9r*+}+uavkUXDxSF`9J9n_#iIemf)pw>cIe{6k>M5stlk+NM zq-2iF*^@Ugk0^ZWO%b)mjR|7$ywYt-uT33Q+Fa^PkLQf?vA^& z?cR@PjX%B0Fu9hpV?e5&wtceE*0Yu+wxjl&rW|G-wio}vOfw&kZV_WG5>;$q>ZGc5 z>({SUyK1|#FO#;!EHK+ZbLY7X`K5B(bv@(UhzpkgG9MKA(6X?P&m>Pa`;yz%nb)p- z(+)K*SJ_d%WQ9v9_tXYP=DbNXDNIB?T>K59mw%mxjx--g= z;9f5ktQ{=ro#^=P7#!*jzUc4RC)*C#cSH=aiiX|HE^(EoTUPF;d9T~NTJU1Pi&yVA ze4m^5I^Y(*!}WA6;~v|68>7__?Iki#Q+k8-AgGCDm zr$6~#>_gt0pCspX}1C6BZARKJx~-VLq=&OwDY z-HDzpzGlLI@D6cVV>DeiO*Pcjo*{ZEW2G2&eW1YC!F|&`(tl9cMNcuAqTa->N?4Xu zyI4We-|-Lat<{l2rNVkYroLM8r0avpw|m`=zkB`8oco23n|@s7DvIqOdx1&NB&Cyk zv=5kE^(1wada&`G%^1Ix(uhwV~pO>ffu~DPJTp!q}7eE;eH8g)VTP1ET_`#cCL>ZD47uKdoPG_?Ov2 zG*uFVG2Xg9pJTFv^BwXG;~eA)=CtKv-LIy195JP`qGQ z+L;fl?+v{&>Uz@UA(wvMxptp=tID2IFw@goIz{c${cW>HHnHD~YnpO1b!4>;)jL<( zP}Nj!U&@Ds52n*{f9LnKRj*n+aXq~7VB(t!KMrIbcHb8-F$bcSC+;r#EU7_4*=V!< zm8qFdukJ$Mr0dfesz%yt`Xn=HYig@w`lvZZpTj$WmCBURg1`ZP7ynS94j&5nq^j~6 z;MDiGbdOjZv&?qTUL=-`Efv$vuF}jW=7*@P3!f{!8TMk#;|Y&@J$mx^&D&C6p8R}U zkXbm#QOvctV7IG5@Q1jHaO$b3hY3+-n$~DocVN9ywQ|aADst8^L7G!gJ^imQ!{2@V zSo?F)ACCMkp-y0@KEhnqSjDnXzge|SeT{60FIN{;uV)BDl%bR5Yy6w|gVC2GzZmc8 z8tdce8uEO}$}bG+*$&=h_gUX0Uq62h@g-4Cb6?NtIvXoRSB|L`{UFL1)z?lKD&YD+ zUf#v*B0pPxtoe4(%bU+TK7aSV!-zjEh_ z?MpJnk4MGn7UM5MD(9oZ!v$;e{EmUHuAU!3Nw^}ORf;R~MT1xhtb|{&H0?xn7rQE9 zUexs16*0GM*P`o2p%sFzi-x3g!AC$TgNd>X$*uF=cTUJo%PZ?H>F5-4$fc!RVz92I z>8G_+#1J!XHtQNdPk{86$@`jfEcawuY+Cb-L+LxSXBFW2Gjg6}reti%+2`!$4+zWX zSNcOy^Aa8>?<(E6)XOqkN)(MvvGB}TsV5J4jI(k1!E-?$TTcAKXA3hz69eg?D7I*@ z3ZIL&qsQt-TCdp(W7{W8j2IVjJTlu}FY=jjt2U^qXYgv*;N$3n_(Nd;f5TZnzmcb( zBJ3CO3n(o0>?wW zd`aHjz89WJ{vWV|B9dcF0}PL?10%ke7h5LUmRK6vsz!+R?h#Y$r7ZW%y-Yz>QSz%$ z)mOf-Aa8}EmLtMHncd8nBOtLKZ%ZfRZ$KybEtKZ(;Gf{DRTwNt&99qfOW*hH`9-p|^|xKZC6 zUn)>yW&a}AS8p@ND5ouOKj7gm;L-GUHDw&5e`EM!Jz^{WiSHR|5X$7f2a5z>b2@KDUtwU8OY80K);i*IVshJM z-2SXjFZ{CSL*|c1pRx2e!5y%VVVCWONo6tF%g5}FRK+cemaTJQMn|8weMRf-W+u$M zUApS9Q3XY{iTK|@Ph)kxXaDTa!%#!ihJ z5mPc^is_TK3fUNIE#KzOvGJTPcreh)KgIpfUBy?*>2tR840d#Pe{&BgOm*tqW%8AXn4N*=7qk9Z!RQ>1$QE?a~R#Qch`n_M?}chan+%(zdn z)uSueE1J}*t@1{$h;O~Shp&p)9S~TJ;Fa!54dhMmmE2Rw0y6$wou*%FZyr?;)iquh zebQdZI#An<8VLG}=^=*wH&o#76#VM_U>wQEO&DLzu9AQ$oy8h!MquHDY@C% z=Y9n;#-x8j@35Ql-@5k%J`im*`y(499w~Az(GxSk-pqC^`g2^5WJjtuX-ndS=n-}? zs)OaZ@ip;H6uF(=;=yM@Q|OxblK;iuM>`)mTy6dz-YRUDc=(eTt39R9FrT(5X3jF) z@Jg4Y3o#~U1$G`*g3X0X-09#2x65&>ph!-woO?MXa}2r5v!imAoRs{id{<77+=T2h z*@^l43$8mx`-X>B@#U$V`o{Kc2_@rVA{*M(hJJ=3Q3Ioz#up@Pig^%QE_zS&@CdEt zvf&JsOCHAE$|md|w4;3)U`lI*VBM7@wDYiwo5J1XKZ_UTPgqH!HhzGZue_1VK@p}R z9N-ohs0>F)Weny*D8O^Mr?{2x$`uKT>TClF12F0)1T7RJ}}N*Ys6ks@wDwdJnaN+(`_?UMdq|UAe6kC)E=< zw1d}%{S_LLKNi-a-Jsjz9cie%L~e@^q$g5Sxd=kv z9>ZD63ZTQ@AfJT(i0+~Y#cYjL$1ftJFpg9sN=H$tRAp*3MbdWKNH?T8Dwldk4X1We zwW-eJabh0612bUT(Z2UFgn4HpJmauZ1{8oUSX-5yF^)JQyx0286qp0(XE78-R2zk91Iz=?(N8I)RzWBrqkINAy>^J2Qv5Nq3?D zqHd5WWIKd;13VIo2R9MU{2A($^9YG(jJ#TS1ygj8kh73HQ{@J7Q-sJimxoE0B%L%> z_{3*(YJNGtiZ=`Q`67HdVLZ|Yvou5QgAloQ@@1F?8=$@ay|5he6j_D5AF6^|ia}|G z_9-4BSG#(+XRS+h94x#g#VKIc2&qaPEB=9Rmz*#_J zO|hQHKVmY%a`Hhewg>b@`8L=DEE{<{oB$f^C-wt>fG;MyQ&;H=%ujkN!_n)=+0+Q4 z6&^u_2os|16}W|OBxfS5tgG@CdWp5*4zwXZgnUJYHh7tzDBqFha7+1xVzg-CdkZ?D zoFGX-xvk=oN(z<5?&4!fEBBZF5yy%jq=^WtD~4*AC-0Vm(hR6VUNuJSJo2*9fGgM$ zgg?DTb)Uy4;g`s0sti#Ye+&wdoq3F{!UZfIzl(Lp^N>_YVk~|NZ-BSMdSQpLRK!Oi zS%FL>HsaR-hONf8W5bmRa-{T2w2Fs?MZ#RZBR7=&J0x?PgcPv}LJkuV@-!Iv7$nN$ zrDWu3@HaxSCWBn?1|%V^+JhCt$ANr=el9|^wFPSAhx9k{Gr5QG!HvWU!c8nfDB>G( zKXD47$a}GgL}z*sRutTkV`M$4!6;A-`3Kxnm!Kh;z$_6DN*h35Wk1#sycPS)6;Xd} zkrm;!uwVWFb(mi1Ej|(t%Pol3u%wVJ-4nWV@6ir#6Cnv+0Lj=rG~%*A6uF+%;Emuz zj3HW4b~=Z26EBJNBe7K6VBM%-%t<2}h=R6S-d<6~TOAvK@s zMR|!@WHaI|b%vQl)67%q7TJ&LLCIteehX_&rXxG`0k4l$P>v&9v>CE5-K5^K37(es z$wL(f*aPaw{|OduO{g<_jQ8{RSxe|dsDPcoCbAW{Wqe~+W@qs0rC)Ff+znpJC4{2# zZsbeU0$jr=as`!2)}Z$>u}oL>Jyk_!5wnl!p|NRisTML;dLucPUd2qKZqYbhj2=Z! z0!>H~VW36uPsqPP07Jnp`~kKFp^7Txixdk({Bdy}!Y$RxQE4Sti*3iog|dR%f^&mw zf;$3-{5$;zgNbYjc4DX$J2*5%xGxnV6kjLxM_wc(yoR*hk6ovZkw2+*^a*;gX0+z6 zdbz$py-_<#r%|ufw9zH#Pv}o*(0;zYv2L*T3tfrk$iDbDoJUTYR^+!a1`I`X%#t69 zK)A=Iq4~AF*i_088}ru#CAn_GZ2k$io(qKbvA@`;AQ@~GY8E;g{ORxPU%(~^m4vQx z7x9Wv8+jW z>;KmKwEt+t7i2Z6>LP8x zSAZ0GyzG)5^3OSpO$xF6a;_ssgzorf1?T%}_@@Ml1&#+ysMi*9Z`pIflm6>&ue+7! zn1}S&@YV>lW4nfw;CwE`_m(=t8e&U%r7|18M$X0BBTtjF6r+8l9<8Eut&M9<+ssSM zRV^CpRO@N0*|Nhn-Tu{Tv7{SX7)I#x45jriRXfO?_&wqP_C?t%mE_9`ix8GSosDA? zxF+Z=YNGF*XT5K4pt!e0;2-}9@AW{rV1?jA?{sfv=cL^6&e5Lb?&#n!e};ElC_m&3 zw&q&$Npcaa9_Gh*hSVOxcc1_lTB_^)}|dLR29g--iZygfX%-CMo){ZDr?0fD5q)xwe5W zq5AwlXVnI^PxnSQ%kbKeqVI0)ZqrANjE=R>v<x%O1b~W-$ z^nP}(b*Fg#^2E9}IoG&WI|?183f|>B$?Nal9D2aL=4+u<`ZpHis=-RiSNtn=i2g;d zWM*iKY2ytV;|Jp`Q*Fy3yE~$5bnj@+-YsH_ov~H7uZf%$K}M~#-7`fQC+mmlW3>%b z?TLzD0sbCtQkqI5c_%+7u*LtA7?362>OLxp@G3AVziV{Q6^}cmkL`+O_q|2_e)r$~p zvC&(jszelvI$^0~DrKsrFQZFU)lxO59Qb$gDHw$L(EHpTP71yYz~B?s8#wBl@0;k2 z_FQzIbp3F5@s{&Qjy{e=XQjeT`4NsQSvztDJH{85_00FpX2%5D`Q4$$p4Dt|a8`~- zYq`^8Kl&Wgmr0{L>uwm{nu?m5nm^e!QK`{wqvzSJk+mY@qol|ok-MVvqXyVyv%xe* z@6k@u?PT&vnsh50@Qql!+(J6a7Z1e;L1=dn52Ocn1$KB6ytUjP-N!r`D7CndceHa{ z&g<*=J9mHH#5`^OSm#T}O8=u^ivS38^>p$(`B}mod5SU-w4#2Q6LdJc1+{7mj8o#2!b zB{vlgvsZ)p!9_tcw=LMuzs+C8Tf=?PEqgzEQamNxO&yC0Ul&fytDILSzf4Z^9B1C$ z{J&iR-)#Rj&pFS>K$LGus0aU%|AZZ&XzX8tVdhf1R3Fg1THUC(ZnRyCm~E#c`$Zj$ z#bT5QA+m1dfVdj*r6Yw1w{?nTrg^gNF&Yia)VG*<>BMJSx_hcd_n8H_IWRIUuSpDH9KlKU%TG%&0p$8x|U|GS!XF>pA&T|dQ8;%sF9J=Bj3i>j9ncQ zh`b-wIl8%Jg1L>+Xy|FWr|rlH)OqGPr6Z2udE!ZFkhqR7%0CHS<0i6=L$3l|ywm;l z5%Rdxy~S13ZE&hw9UbNJ`{XswZ<3dm9hV>J%rAWE9_c>raJa8}qk?LF58s`8&grH1 z2nqDl{n6Xj9r~-Pi~6j_X8zkU$8z3ovqjn}MW@BAjh-DfBGMc&E~-`}Zf_8I)Yit* z$kbQ!T|HXYU01+#Bxd1%BmeAD$~MU=ZV{>qbA!gvlb||Ojw=jg_-?uzIV-pqyXv?W zIF1xjjv%^kDcDe$=&0&ySy;Yck7K4Y*{cg33ziB+2(qNR zOXyEzNvuEC7~6#Wm!65c_}#)T;aTWmC^^(Ka5nG?&2!b=n>dJI&bhq(84L0UVp#1^@`P|i)ugI^!3t}&tfme~u8G_!gD%6E^ zCv@|*Jw4?IwvOh}hQ7uG-2|;mvr_$wUV)dx+faWI zYvBv|rtA>!@{ib`Y;lg~E{DRWWwA zOt#*!PB%3)?Y2%g&o^ALd@+yEM_XQ-cWYnkM;H%lTB&N%$#i9MIdPGCM6?7Ol?V7R zcw87DPZG7rJ97iyhMUPYM{Dn!Au+JpzcWzM@AiH5clGD{5Bq}N65ge*zuft5*^PM& zz81j=$g5~eupiriYrxLsONxD^{!nmB zpslZ0a3gz@`-@u?>LjGdcZAN8O<9cmYMbJXiEC6Xq7peq(?nZJwU_FzxofznyMw%L za;@j}PIWKid98t7po=jbQ*Tr~Q{7`OtJaexsoG=+EXEvAfRn&*vfB=&e{O~@E>vJ1J% zp?zrQa35bAof6@tA#xC%C~2Z>2KBHaI8P3zTM#$s6zyzP8LdwCn%A1j%trM_x))uaY(}f7&*=QhM|Aq7qx4nmB1elc>*li1zKT6g%NU7tQjmT zb;G-0$N2BULbT&iA9-fJ!CR9TvDV-8W}}e<7cfFPUnp6QG8&hd4xT7Uzq1pjrA$^z%Q2?m}^PN^plTgBudK>O17~ zh6V)3`5UtXg4cpyxznMK>?*XnKPj%317a3`UeOZGF*DlX97dD_la}6i- zrbz*4)hR>|xeI{w9eIpYk(7Z}=*f2^Ieb+pEwoEGgkxNRZwh!S*7jcGEO0%-H|hnR z!^V=%ZD1pn){33G?71eBm^GB^F92_$JMrXV+H&7;1gES3^XJ`lNFEW!HPG#ZE z2?M5w<;ZPx78gmDBj&RhIfCfJFBKT#D_9N32~(sBV3AUkjhFUdCAjlalDq=#iM{8L zMZ(K-J=hU=YiUiWwqlnSa26#=atNPbI=@XwBrda7b{~}{1J7Z)AN?h`iGHip<5Q^h zU@||42@)S+0nq?kPu#$l(l?-%*rD1l9S65nPVxobfYdQYA`$yrQ=NPU-Vl?BI`jm3 zvQz`q)(l5`I1S`&bP$Dvm`%)OG&DLW-+peyrTx+ByImLZzqOwfuu#L8kZ=#gWj zeK4MHD&3&B@kV(gv5RYuMHBz>1MxS=`{^S-mU_ibCCh48@scpvbeb6Gf1nz#o{sf| z-IOlcMUWE%*l6lKgTu-2qZU)DO6#;5X&<>0GYM`@PbObjC)HOKSKam-30n8Vds=E> z_6l3ryF>%kJ?;}culW*eCHU00#0y*}VgtOyzCgQgxt>pO5xF{O#x6)aI}_W5R}f^R z$rR>f^M#T43AjMg@-2vqc($+zZb6>uC14rhAJ|XxOlU10Rj=R!$~gQyeO=5F8|hb| zXBX>g!_I_}D61S%FXo*Li=W08l6mY(rXE%iTSl(uyBbsZZtm&{g3#7`PPbf%myTl9 zz%tq;eS$TZ)7&}buxehY3$UvSgr@Q{v=>%H?1C?a=g~Q>rJ<_uy`gBZaiFznH@qLH zLCuiO(oMEG)d{l&zmW@pMXE(Eg&zbHfhniaIglGfyeJ1g+l#`)JclWTY~)*|-i;UY zrfAO+h5iWDC-r#u4@_-4;XDZkYS#!qz!l9F{*$=L$_eebo4RB2Po!rA@isI^ZJ}EQ zEE*?XOmX0Kgf!|4xn4L&;4)44guTjQ-K*SF$Q!|H&u8g{t|oVa`#?v?YtZ?_r_v_;9SmS^r7E(GJ`1x%2Qd;` zC6r*!1bBSEYD4INdYUqitw+C8t@Fl^v(ytXO$*24Ubc$lCG- zA(Qq=Cxi7EfgIr(L+{YoJ^R&m{9Z5%uZmB^=lC<3E^>YLr>d2>13ydu6&!{E(-|K@ zE+89=Dl#6QB7PN4Y3g8@Cr?$5De6C?ey#Kq>oe7nN9;YKHf9(1ke|>@6^+fnv_KHg z5J&l)x~{>&;xvO>m=gOauM0jaq z|5jo!L-FH)iaEBL){Cl!*$Hs&=wh}N-C7N51&s3`B@=O z(-!aT8;vI#Q+<8mbGoZi4Bq448lDLo_&cga!VYoK097f=e0M$fKsh=CTJgp@Y7+)GR75kf}Zs`alks(}kZ*syvpPsTRZ! zVtcw+@UAvL1V}Tr*w;!sN#WQAnlB2$-chXwSNNescWNQm41YnF3GUTY6-EAxb)IuH znXO&!&88gcjb53KCKmBtbi{qK@C^i2t^6L^j9ur3YK93VILi21{EJ(mrTGkHJicE# z2piA|LM|9a9to|XQ?b*&4AiFJ>#VWysVcx96tC*--jZZK^(J%(AEQZjI_O;8a>sa0 zbEPqytmF`7WjwMe8imNjC=JY}7h|t>b&ToXgQu=CPFKpqV#Vce$n99y^b;Xc=~m ze(m3_(n$BPJcU*UkTW=eKI`wUTCZFTHZV^JCG%hPccn9d!TJXB7JeFaQXYWSR_cKY z=v?Did=VT5o(Vg2Nx{2ttnQ?52=!j>ihaQoxh1N9u|)QOCRzpQ1mA7MnA7v85#al8Weg1Qg7$K1Gsf-;Nl{mv^;H$Hy25ATJ4`^U$HU48bSiooF3VfsBHSd0pcm9p zUP+bI%K~cE3VDFM7oC7Eg;kU)keT2ra>|*9PAKtMS8R%yMNGr=VkFg$i&g)}Uj-k? zkHR#~=3tC^gftwT7wgPD*4zjjCV5aPNN6^5U6eueC24_@Nm%4Ws<>Pqn?+3JuTu?# zY+|%H6W4>|up)j^NT!n{lRTY#4mV?)#D7(r+3z@xNAn^e6b7u4Rg@i3rNdS#OQdw- zIy@v%*eR?$+^oz+C#EkV}ERpcK{`o%b&bvN4r7pI8ao;v0lyY787M1H2#H4pz!0G+u2?4fNhNhB^-p z3Imx_aIhp%F7Y7V1nwf&NL$bw-b`t*>Lb4#;d=9>NPLI%9A5*!07@=Oxwvl)!H%C!|xno9sd7`i=5L9E$4+we%88S9Zw@u^H%; z;&qsS&4qi=|6Pm+gl;;cS~yrvhyS>Q5s05gyqXhHNc$`JST8brEr zPwz&r*G>^zjXJ~=?Yyzr=n8Mje}oO$ctbokE)$RS(qsdymT6!xT!O!O4|&oUP1J zmG@7gFkG~?2p=&Eb@V5CNAh3d6VX$D4%Pw-SEzs!cT?ge_#Rt~Ys4brGtej1jK)wU zeT1f=HRulyhynUPB8gb?gfSGApE*Q(;<-_q$k4|Se;AL66}V5hOJsy`o~Q_o!b4*Y zMr)8>fc(#&$YCCw08>K3{rqN3G3KLke2z@j&w)15j#y*7CsOdLf8be8M64{vYx5Ir zjRf4&6CkuSCyqg#6F~%U{|lf#oMnu_&x$AB;!3+h4-`#&#(io$LB_1PE5kK=EVh)G{ zKM}Q^jQs=yPRlT2yHN}$kQYW-xUIj!ZM+cgqb_m7=!{W#0-}rxr&vLvKXKofibxqu zv<744A)a9^p@wQ>G=OSn6j1*{4KPD4{kI(OT5?pn6*y4R{~F|Qp9<9&bK3KdXOA=FMWx(xFfHi6&iuN^%|P2|42*C zvGbaWQJ9GL1r`$4ote;0><7=G6|`XM@g6$h^XrY9x?wCrjn!!sCJK`8@xNC!CKG9T zdtw(NC!Y8Xk==}ViYqORdw3ZmRtvu3xCeTBiSd9dHl8 z&%la|(R3m@esC37=(-C@; zKZqvATVx?KjPqDIqDUI|bri1tGG@*aeX!95EBOxS(k^J5phIf}YL9MQ)B9mY_CWN< zfmF~KtH=jLh(TtPM?nEfLUc93bMKN-&R7c0PFLbzj9)Fp&wb+-)HtoE=U^NxL}Uci zpT;KQfUy>9&||C>vy4Ny|CK>z$QA1FjbrHKW$qktQRWQSEBGN7+wmxD$vQ$OtHIarVW@kjjeO&b|s78n2 zcY|m_+#s#c^vOga{iQw=Yexl)P#>d#(UWMRKi4yj=XxFJ^zyWY;HZ4jn?W_y+1LYR zRSsU$29fT;Zs)XdS?>!aBWD~3ujmWr#%IL(CR{^X{Dj6>N2*|@eF56Q9;|C$5kIGm z%|;5|Uklhae zM9vAsaYdt}F%7dn)`$c1;3Rf~Rj^*982ynO-)h{}mqO)I7HZcT&=?9tBVrsP??18= zh4Iz#f3_z+KrQxx*o@DX!IdloF(KQqU_=fP4EYFi@)XgYyaKgbJ8)-8l7~qKF8C5q z8jT?nC)-2@vMw6$`~<-=Wa%R z&{;wt_QV?>^v~LC=mvw@S}m+v)v(-1sUeS%M}eWS6^w}WU@PRvK`0)Ngp0_jVM^W) z{o_(aRM%;(jL&*lkAU(pOcaK8@d5kG4#~i6>lNzALW|XMyJMSPF$} z$z|mH@*Bx4^^~B2k#a&srQYFg()D04xF=X9kQ2xXczwzK(*8rfXy0~!Rln-L=wIe* zLi8zLLmlka4)+9uk!ThhhlmM69m5$33zh==wn zQR%T#!qfO3iN_OdiP3R)?2Ap;%vRe6TU&dkx#R%C(paWpU*!)UPy%)Yt+}+DUr_;opBL1$vQ9L zQ9_ZVnnk!oS3FfDsqo@L1B?8dv@s@MOj~;yaS}hpyv1_bHqH8jS!}$~dBdunmQp-v zITsx#9XYuT-4(q%ec8bhffIS3omKKmxl0B+_n7l!UPN9XuQePGH9d}C3;zb`0=OWW zI4;H=-#*{dLg~p#F=vxD$Ip%VQrK1WRgtD8mKF==pIT5Y(XZ(DGZ|)|6B;O9lq|DQqMRF)dV^6@9;I83Ib{5I`mUSvK?38>1U|l{RPD; zpQ483aSX_9kYmjXWy%?*oK4=+Qb=y2eN!uGQ;Z@|ou{*p_#?tL%RxJwIk6vNx5oYx zUD@uo3^jEVcba>ccA3VB-G%a^kDjB8N<=W%*T`Ks?~e1=y!oh!7ja#26w3By*)j^G zmrk#n-Y|p9I_X&FYU&QS>$;qI&b*ww4W8N1vwZOP2ws#rhDT~Wj56AHraV87X>30k zbt-O1VY)!GcrtE2W~G|^un1WwE&u8w1M;^mP(S%~Qd+^pq?m-J5r3GL*j_~Wqk=YB zXvMUp50HWI1K;34LFroXxNlTmZte}w20!O5;JB9gPv*R=tvS_mJ~(FkV?*_%zogfJ zt-(^^6s41P0P48?+(_ZJCE0$)=CfV3yf7^mQiK_TMI2!%V6I~R!^8>I_(se;Vvtr% z)}=n7EdML00^9h%_=dQvHtT0*<*f4A<8voC&gR9q>f~+9>*4qn)%TaK zhrUGrsz83QX^u+$L*Jp<{agD?*J2n}F;$Mdo!BUUWODa-Dq?qZt=Mj{T7hZ>>KEx; zqI|(sNofg%^Zi z=|J15qO4iiE>NP*cMu*6pM)$>DOxbi=nv#LBcB#jcWTu^(%V*{%Ha^QgfmiM?<_Z~7H*wr#U2;&cSKsusOUzvMV1bh;kL%MGv-^S8RoUN z=T=eJP5r>`c@w-A*e~fTK#qHZYF#t=B`7~Pq|>M?o%Qpf%HcWTRH>xY0Dg`?Ll=W< zLmQqL`b~9Ctx**rF zTCVCt?LJdg)R3yH9Z@f};%smYPwM|TfFtyYAYVFgb{b2@fG{?ST1_pXr!g}bk{LzE z!`;#j4u@A5Phj~W0%s1up@f*O;naEybb~D?VQ_MEz zJUtqeyV~?mu=IIibg*PNF`rmXY%Y!$yNE@_okCT? ziC1jnck>JQn*3944>uTEZJpf=s+*r#%p@}V=^8W-7om;nMII+w;uO%^uo}yB2d>1W zj)#w;sQLg;;W$vmj)ALn2nvjepe&)R3O~iGuo5=OdEs~AyWzLtec^3z2{jG-r6ffh8B-NfE!Ke1K0Ra`bViND8de5_y-iU{?D5yE(3 zAfjNe@J6U1t`i@E-St{LEDjPIinYWTQ5QZ6mxXykE#VWi>_z!W+zYl6+n2cx=3yo^ z6=cJq5Q9`N;L z%WI+7@`O{uGs3@zi-vQg^KgOvA_YSGLK8xzLhj(J;CB3q2Z3r#Xc0bp7v!Zz(lqIf z)H?hg-j6GCb>*fq3Ugr?$ZzGqKl>dg;IUFxpR44iZZ3efnBQu5ZF%v-R z+r@5Yx3LGl_C+}I-!E(6IKj3v}!j0ipa@V+rTpDMFui!T4dgH?9LR!sPv*0Dur4~b)@2R5+9!JR0m7EC?k| z*Ay=q2qD7L*ho@j36-0b3mk-O26yEf`cH3nWd4ur9q@RdXn)Bn%D z+zi&1K{mmCd_e`M9`tEip$9VKm_guProd-ckR5}8C|bhaPc7-YnfY(cg>Mn+(tKs(-& zDFFZ6M|unW8!gb+sNs}>y0{-MyBDCtJpdKX3&LlTwnNRYZdZyax3R(o;nM1k`4g7z zO3S5QQhAA%euQp;T!&{BX3gc$jnM5-DwJs^sS4JTzoflbgYDst;dvMfM*dY^EuWGz zF*{q}-1bI^R)1BOVKpxeQsffwLcQ=|_JOy@sn}Q=Y$|IXI8#!zZ*|rrp(=B#_Utpd&u!^O!H`ssJnTW|U>{n(bc3W|9X`P~{)1~OA)O0Ez^@>~scb*A(fhEY% zMI*a<7|*PN##3;2yMll#=$YCbJZT4OwKNW6cST(fHs}borP>CX!C&!bWt{S5C9HTc zYjTw|B=WOgKjHFm95G$kTe%4OO$`X58bEyqpSg0axeIr=auJ5mSO{Sah$qM zy`;L-Sgk&O&OGpFMZFmauPORvf@khv0d~g8DobS^nc-LgyjB@*nN+ zJef(xQ!S~vaP8UY#`NFxB|4XmW12G45l`uG>sDi%vO~aRJqPmWYxW8v=pk02TZp4Q z7@<}8S_`%i>tr5cSJRV;W?s@e=_!b&65tuXLoD~F%2O)l{wmaBKt5j;(PM-C_x znYMvO5-KHgkje6Dr{RXJtYxW()G6vOs!zFqRkx?oNGT73m?1yEk>)P^~W4M z38L)>WY0>0`Z@&8%LlrM)nF=ee+;S;XDsI!uKOWlM>Xs1?Cli-;y2w!ALwKzz;1@Ld18mUI%&&t>d_EVRF z*Lp`y$7uMm@0Bnj@mdi$Y-@oG*%qY8_F7l18=e&-vFn;y|8YqcENU>S9e?${e$M9xemii6HQ2|mt;q)e8ino~oO-Ath_Q1_`EilvKyaa|WY z=Mjjn1$ahHLxjwrC*#lQ*ip^HXkb^3e-DA)u>qY7?Z%bx1j9)A9 zu>z-$rPLy7J~fA$P0gg{frdSn8isYR3ubUl>KCdwWv6uPR2}3;@)>yt&y%fSJdDPe z)kj>#L#>fU+yifT6)_E&*xwN8h2Yxtqr1UV)bMr~D~%bbO8m!vua3;5Xvk0~zXl)q z0y4fU!Frzt7H%uODX0d8|4)n#V3mFb_Vsztfm5)4|Eul5nPQW+9MQ87-%cZ<6k1{Z8ypq4N*^?hikb8JtBcTaed^c z7ZSU1pWpwV8eak)fF9&Pay&Au>p_S=NS-FIk&m%2d=IYpPx2f29o2&jjKELwD?axP ze|t{8A)kWmdJD7p4EW<)$!+-FWw;Ad@dW7$W^{9Kw9DZm7Av{|PWfx%F6xFUsQb-F zJ*+2k!?i)tjYYosBVzvm$o795T`^7-jFQMVV+&#^m>W6ptbRe9-31H(lD-vg)_I7x zF`&&)1~I-rJi~v1Cp-av9|0C^XT1kvvAP})0==%$V3G@n%b%JAih5Ce*FI$1({wlf z&p(i@FAE~?ed9P*_r|C_o+4x6q^^x>%M~#DcJpP})#~bCT-d3QB95 z>_j`GGIqcLXzI_Qx|OWA(l+QORP;*F&7pr8uf$QSm`n6hdIPaT>7-sX=3&qN${3-K z&`KLsP-BRvCs8tLhmS~u8cY|a=8%6<6Ueg0F}Q5olDQxyc7?j8FVylW;QVJB6!Zfh z(2XM3s7TB=BDKdxDe70O1WqCkGi51iw|xvN@m;H`zbAHJS0j+u^`=Ui)*7+9!WfI% zd^&VNUqEA@N0cWo5(~ARid|QU$(Y}5jA2SItrPhPHRTc1H)E}OP`^oZB+FBsjC7@{ z)`wnCAJ+;QgQ!YIo_yFC!p@*esQcAV=mHT#l%=LYsT(jl)!YQo5198ggOWq~NARDrc zZfJ~AN)eA)m9Bz_s;W;Qi;>HTJ9IfJ7_O<_68_-U=noW0y^M8iG1-+?jBDx&R5?ZC zAT6?zzDY?!mxBr68dP8InI5nFPWEJj`avz*IHT7hD^OL4Hrg}21+-HsL_E)s4Z?}~ zPpnA|K?ZIN@BL#k436_Gq7wY;l~4<+jU4tseF#kvoSu$5M?n$Zly;I+^c6VC652_! z0CYgb^n2tJ%+?X8+)gEQZ4Hz%N#q5+1QAa)*BYt>-G$kpuhUkN_o2O*O8WJk+A=uY zdmFPf+8`+wquGsqLQVunaU1c9vas1;Xm=s*s%N!6Oi{Kr>hv3luv&w_*@haSelQ4V zqN4T6Acpov_1{JYlw_likVhNp2yh}M5HINt{ft%A-F)ann>PYWNv~Th7## zH~C9zlUad3Nzi!5WCYL$cUq+3)MZWjBD8m*6)#)e9h&xEq{4RyFN4|D#mI)E(C zRWZ)W-LPh_QxDM#`Bg+W(6JCLTryjUYNVA=*XRed&rCQ!qN18%(bm+D`ioF9@^Kr5^(}tLbUB_OqI&^9awUxe0&X;GZWr($-b9^!ArQ)7xJU#mH~jqImQ!~Wtb)r9Pe{D=-d zNFDu*(T8>s9;ul!hutl1(8v1MYKz!vxXS882KENI>UH`cUsOLJM;XQShQ=l6K#MTF zhj9*24nCB2-pui@-Z<(G+aZ{# za(pW-6dXpBVRXH-u>xHfVw5MLM5QeuP_(HFb2)fescmdgWA*RhD;0d&$4c zd+1NXHlnY#LY=Ef$kW!-x-$_%f1pwT{_WR>nf}8@oy&}pI@1T}E$rXSYHFgkDYQY0XGc?sL|3qnO6j@AX6V+c3kl?7 zZ=CKhrPCecGN871_IrJE*{RV@xIRjRIy$f?u$%p8>B9B~t$e=pi1e}h`91Vn*)P>2 zE>eZa1<-^&U~#%otFRtEN1Yxztr+A>E`=FNy48WQhE5n}qpL8CE2J)$is)a7;pzjj zIvXMeDHTJN$UiJsv;~sN)-Z1Ra+Hwe8<&fY2WyGOOqzNr^qq|}zf)=jDjRRvf>aWd zMK@AsO04>e{zXeQ_lZcxn{q5*JVHlEzAdaMs#|y>2VoO(ldZ(jzucm_|u@5p9CrQm(HJWMk=KOe-Od zdg^`}xXo^5h7%U%4EM=+Eq#$ysD2EiF*Nfn;ZM1xj+4gf7uoYPlvPv{%}|C@1&HRz z$Nym7vW1Bv(i-_LBMGbJXMwWxUGX%1OwUlAS{3y*p)lh#R-R8kw-ixLp#thwbqtkc z=}ZMeUzGc}zKcdk??zrj+-+tYMiIRzR3dfdWuO7(P}`*6iR#Q^occPDCAryb*vK#U z3?3x?I9Y`Q&nFcW3q1qb~G6hBlH?yrH>X~BL5AsC! zY3Z7Ng;s=J=IeB1$QAfAlp&8{ms$VjdP1S`G~6xJR?VSj84=8F+Z=kOXMXTG6dspU zEAf<{$yL!VVI_R(y&H^Y62u$K1TKks;`jSSy^(O959pjSj7+1e8WCz6W1PMQs>}`4 z-_&nRAFWZanBIvgLX4m&a|x;=a+A}IAY(xnge>X;W^Px7mHt#l&~1<}@6TM&M@SLj zWFrRE1BHz>cT37ROJ_HbD|c_ z;oQB6br9nN<3g9AHJ&BxBX5W2s~1s;8ERA~8t8-7WaAcT}$ORJW+za1HOlv^krl`B>IDPRk{cz zY-M_>QHOd?&!v`Y*HkZgm=eh-7S@Vz5p4<6f{NFy^b5A3T1#$1HR1{r54D9vA?g>b zuRb+IwW3}cW60TT72{Z7O=v9jTv%oN1HVjJx*6(ILr5Dpfix*jWe9nISgi~q_Oo6# z!w{7?b*eI5O(zaf1=+q-CeB>t6;YOr0_gk`Nq3=#8U4dUp#LZZjekvYJ-wc(K+Kn# z1m=aEL}hL!|A_2NtkazG6!jIko~f=EC7QD@^(gtIRs&sWXs(ATNG2$RLTRzkKYd|l z3Ejy}sFj@6%w$!1jo{?0DkU`~4O-T`dLgDAcFU1em|TXK=|vBq4$`;jlg2^ys$PxU zZ9E|x3mbTiZbV&G@5rAGJ6Vc2K<9ApvDcfX{ZwgkBDBTRiOEbnaay&b(iN!b!5O8u6M%ezJanrdZ6X%DaHy)A-1cdl#$wYrH#ztZg*xMgZcST->19`EyL_9 zMt`Fh7=>~2V&qz(9r9}IT9V-xYi@j(YiP^W#h@@v)Hg$OKUA#_r$93KS=&KG(c9H5 zc`kV$T&LoCKDsG$gIq^{W{=aEL=k!++mro++Cj<8Gpu_L>05##+@yQZ>)8nA5cz_M z6&nb4E{R=7{RA;AhB-kWq+4(-`yN`W-=OMpkY&-is~JuW44&>~^hx9e>N0X8t&FzX zpXx%jHwaPNafV!|y+RkHOWIVuF*s)J^-;LqhUy)KSBfZhWUophQ*{C?fR+*it;#If z7hVaKPaynC8X5fQALD)CI^=B;yaku!0%R!&c|ec~9t@5PRgvl`rJ+Zh10B^_qMLrq zC`Tu-NvN%?M8BL$pe8*w?XX;cmwuc5x^0u?g!#RBo%xYTHeI!(o70 zO3_`FT%)O{f z5p(Pxtg9`TEE%?>h-LPDHrCQy$j4PrV8wVzq~ubK}Cp!*itrq4A+N!Rf(m zp*rDd;Z>-BWJ=S*!{s5$CH1^k9Gz2sk|(J4%pO)^JMhcI8s>r)+0k7$j5{8uM{%Yn(7SsM32OPaM+DU{5+Byg4KW$v}dys&~0(py#27K)rsT_jk`8 z_aXOnchJ?=#keTfN>`4nv|Df;aGc5w^HUxG4V|d^5sRC;44rI|csAZKmp?zS@5GvkIf|<6B0UKjM}i*C-4u$Ow^}M%lWl{ne_KV%J-FUa zvXkIaJRvj`T62G}McFu}6PVtgl)b1&Zj%Qo1LQgANwZFxBPB{tgYQ6v33`wCYWUOr z{R2Y-j|0DeYjDF?%NOY#?oM!>amG2P=8VZYoAD^UO2+W4#*VU{=fRKab!rsfR?Ohq z(VAXLGh-LrfgL3th{%XvlDw>N)xsAF%u1>i)z3Z+9ZsIuM@O8B-WiveR50JS#A`9h z)@uB9@|;#g=`5|1>PW4^2hb3-uAYRxE)MBi@J8@saA2@P;Dx_Us9$)EnyU{X@<7P@ z!qjG;GgnaiI8Ba3o@pi5SnOyTW+`c1WNmIWOrM2V?kIJi%q7cG>&YfWDzbR3puN1V zT~hLc(%D(=DbwLYp|Qcw!GA)drQ*`*(78}f=%@54yaDz5r0|?j#~>H%4Yf+1uZ=gK zr?Y#mtDSR7&WDT#>2K2OWeC|3j((nB!|_BW>k)qA`>}_}{_x_r)jAn5%w+RX`}Twz z1?gfJi>U?7Nqb}GN41KmW8H5#VmlPoDqyoc6ucz9!QxPO@cyZ=q7wVW#VR?q5}$uPr$1vrGA%?v;dU_JFKh(d4K z^87KOu=$OpsP&oYFfXundK=jtlmeT2TuD{)A-A7vn^?{jsmAj=%w11NH20S-;}s*5izD1Jf+K- zmfLPbO^H91^emxmOq6{Tzk_J6tn^iO{h2r2-7HvNj>7quC)ZL_p;%q2a#Dxjw?Lm@ z&(J8juvQHon0rhouD)RAQ`qtJBjncl5k*L-R*k>l)~W~n=xNk|;us3+$xb3d|0dUk zo}#2esuQ#=TB_av^~51it*_VLsY8|Sa)a<_xrZz&oSLGhX$8@nv7h!1bltC^UjjXR zbv?8z&e_k=H0N2?{H(n{CV#K~?b?@_->?2`kyFF-BfNr~Y?>S~#&*T_R_w|t%oSn@ zS&9_DR@0@gT0cFi)VWhQ(8>5?T;KCeItsF=xcE@pVWu` z-&~s=7aVt8Z@hy-ht$c$dwLnVUFByFQ?m_9`=Vy6ca>VGf)pZxbR<`Wf6r;0lXbF# znYzg7SkneDTzO3YjeW)* z_9ab`BaJ=ka%HP5$bIBRa;CBx^{Tn*UrLv7LMY8&#xvhpC+B+RxS#PqYJY3>CGa`o z>w<65>3y>uPCmF;pU*zC?2jxTofw&FZ6rSAHnMeD^sX}JMGefS7F%5=x?I&Vl}iQ- ztxMQs-^Pt5MxvMF0OAbuhj`E&ja9jou#Ks1%nUvE{F3);UT4oY|4~U%D^VUUSF8#} zT`|teOa`@c9G>d8jWJN;ry;A+ky$|(q$P40928?UtDFb!p36TcP&Du~a3a`SDy^K> zJ|bT?hA5A7{ScxP`inZzDe3GyZLDz+6_>%vTe*F>O*kiX zG1NKK0t6S*`_6UCSuFSWEbCA2H~mZK^PTPv6PS8YRVVgKOvCxIDS-g zEBg%7D82;4(|+!QwPZYie8r>6c*_c9Dir%OUvA_uVKAXcze3SqQmzvH7!$uw)J^Th zuWVi8mGmVb`%e0v_{)ZhDiqnB8!q-WvzFTCOQwcGTQ-g^L(VY1YNu7N;#cZIdE-O% z`ijyH)yE&`IrVQ~v_IQ-!sqeGzN&$Sp(miI^fNZnv)L1DWqvSMhv~%}Azo{9j9hh` zz8|^#-^k*0FQzJppU24J#9c5hvqEej#&^YA&wJjR>AmMY?>X*fTtghz?989m@3+6? zeeU*U-}h75d7dgtkXvED6@Rawz3{w(6Ozr*f~7E@M&*(VnS$a2`>Mn~MXHsmQEoz+ zQN^0%D<3_JziZqM4f34z^!FE&?-A44!Tf5|Aya$d9`jDSBdreJ3>*)33LjT%kR>N1gKk6;v zJMLxuDZ#|>YCV-&!tE4OOs&kdOpQSd`3?Mzy2@kOC;y4Hb0E2luE&mNUeklfm)c!< zROpl+1M3kygFJUUIi8!I>+Yg?g>q|TP5&ACKJxq3A5AjeJ12(v6Z_3wVt*ucEnKDO zi~M7gjz{vsV!DYjMw>z|;d8C?6H*I(DRI2afl|)GbrP%C!;C{2>A{|7QRI| z*gB^DmYLQZa|YjwvTNhQG19M6XStmAl^Do%G{xDbL`;v|9nr@2hdG9SPHoZeNo9R^ zT?3ubj#S4G=Pu_~&waeZ{^2X2`lLxY!REfo-UaTF?lfV8-^0YE6u-PT$+OmxFf$61wuKqbNXRb@~K9MRyFI4=$+Pr*P za{8~hj<^Q8Cwa1irPS8sM6r|oLR41F)u_d`qNdw)f-zej5sC;l3&qKuz>ocl9!L)* zd+Fokmw{QXjX5o{%49jSw}OTF+IK6IB!5xnsVC%jU@N8sOG(pk(%(;xWxtAkn>lh_ zlpb9+x_DIn$hY>^wj=g@5##K9RJSO1Wc8S|D674?t)O`%*1Rgz5U^Xy8GmW#;6y7= zJf<5m%g|4ECUcCb#YJ#E*$?atUKMtNgSyacvDAeJ;u|}PEUQ%V#Gjy=Kv7Z$ zxx_!nWa73KsmxST<*Dj)YAR>3oQ*mfYf5aATsF}Z?}%PyTW#vVC$Yzv_S{FlpsBZI zm~Ex~oB4`4lOKofzeV+ON?W}Js)C8^MsA<**kZD{%{MK(OvrSJAwCM4!xX*;n@;$Z zDE|QmlRo}q-FH)77kvBgr@a|pJ@eEG^aK7cQ(18#TUHPG2Rb9sQ(~5*j&FeSl(}h} z8UH37m9i)uc3u{=|8>Ia#jB4ghilN!G$F(UCwY>CJh zmL|d#aEyMC)j=1YFO;&jwa<&>$zWMXAXC{v4bT|AAaav5< zq}&Mo^uO_(bZ6$Raz;4U?>WclSrLT$NdNFQhDLX-3OeMBY*A?M<3S(-#2NkHihcR)fF$8o0+Sd_VCp~+p0@k z!*jhJd6?cWOtEP89#M(WYopdjHnoS$gTTaV4L0&Bp^&M8WvngHo@xJXduzFDqJ=27 zD!q){Pu8L`sfWx?b_!orI4SPMvsbW9wAQp$uoX6cH}5i4BM(Zm^LBrq^l|e0QmJKM ze|Vq$^+E0~X`nus;`wqItAg?o&whuIRVB-vbK84baWHdizsEF5uqQj?W20)Bn~>#| z^uRmMJ$DCBd0&@MVby}_aXZqcyOk;7)&A|C%6UI?8s#2wOm7?1=`ayjBR%lwFok$ce@rEo+sD{0vytmpp`Itdro z_7^pS-7Sm}AES?QD*Gp$LA@d05(Q}|Q=iXg{@3b&_iAiJwEbWEE6XR-U{jXxLhNIn zZ&_;{X4mYe?aS@|T1%U!3FE+QQpiNggzVQ&GK*f#Z4!E$qRg`Z1M3!fFJh_fgy}15 zH_C)o%$=6EEHBsBUK*kuA)lfie}wF96jrJQfAKtam~s#04tDl+ zkMpxqeQiAX3mY$-7EHoU{u7&z?ZY%hRiq@do~2vV4^L4$~{`Wh76v|2We3Ospf6&p?S^0G!Hg8{cQg(&x+m2}e zP<1rdCqj*Tl~gn7TkHn=Oa3`gK<54ZJ-yw%y&FP1mH+Dd-H=P13O2xwaEkw++v_Op zc;kr3OZM&wZjvV%PP!7_$$H@v*Oz<6G=h%m2$Ze&*cQSX@tP%UO#totu)Vtdv2CGk zoOP77j9s&4*nWvfw5Qo7pxe+obaG{QCv%@Z1Xa)!Xorf^zp`Vv!$JbO3oo)Qvi)g4 zXui*#rKZcZy-_)7pNY5K-dNr|`%w7Dnau0%rNMRTIHQ*y4vz>#xVGk=%{`iX##2nH zW>CV=h(WQH5|+mgiL7W|${1>4Db~NwQ`Kt*8LhoukS@(W1hH=y*-aY|F6Zy%`s^Hq zZkcDi=3r6zux^lDQLXO5Kj7|w{PmXE4K3y>?kV3_sBhY89%uexo^6iDudawii+x$d z&B%>WTGa8VMUe^ijuwaTo&B5cNnV1VKM9`Rb7WB_oz?hJrc35emT_>WG`D;e?=cDb zrm*0vpK~U?>*osZy&ugAx4dVg=M8kwNyxr@a|d#xoG){~8>Q`$xL2 zW;gyh{@dCwXTIM3F)p)&W1y>wXTGn1&+YxwSIvvgwcfkF2cg-W3!jDE^KVe7GT=}cq1+CY9Bx)ZpGe9&5do1iVUQmUZ1 z)JOUY@^_{KzrxhoGSb@Hy2x_Gv_Qya8`G7^D0p{0WCvCgqOAKP`bTe#oe_5~mWVkS zS;aQbd`@7vI5wGyh4QZ+9YT+j2%3h1!3M?X5~?UwgB)wL(q4xL1*ds$=WWStkTo-- z#?MCS?SGz!FR)SGY|kDLN%sefg>u3b(Q&&0oIWdzOxnZ#;HQhTglfVap(@{ld(Nz( z9>BSfLE7j{bUS`YSD^o)){xK*=>3%vVLdbmn*XceI&vemjeg504{Zq#4XcH@MVDa| zFsw7!Ezsg0gVsFAnpuH)1C8AYY70F8wc|L`WlM7~P6tHIjyeJc?-g8c-MEv zn~3ahKJP@I8kiYQR3nIj>|kN8Ni+91Efa?bC((WO7itS}7vBE-M02tyWl-no*R-EL z#Y~1qw*}~n*STN#uILTl9@X`Y^hL@=PAB_A`L>+0GsjttD=V19>Y^+x5b7bHGlOXe z>MttB)K2;|vlX4N8}sMk)#=Yq=PI&E3`?&kn#XAbv3OPOXH)s zI90TnM&o(IS-x7HSpT&>vrV(fRs)^kE1~AQzew3Wn}O&p$iVIM^yM2HnaJ`LFrM z1?B|bg%ZMqyhPqC-;$5W8S)3@Odsm+h<~Uu48?O%PmJ_Wpqmgsb% zGn2W?d|{!vKnNrG0dV}h;g_I3uvYA3dT9D$DsFCK?qwcj?r)xHW-N0og{`}-@2n}- zXlrFlhH0Sq86IVgxlP}MI{yuM74(Uw`X2S2Tn|-~X92(Ox~Ckn#EtUaJ7e>RJk41i z6y$E6SKgLB&3n?j#q-u(8qDn_uI8?~?%zEjU+>^S^lTdqzJ3oQhbVx^7=%vu!{Eg@ z&D4R9g5&OUllf!(E50EZ_fLeO;!x8R(_E9qv|j9rs9`X_N(feAl2BTlBJLLVi49Co zOzEbM<}T)6%w5d~&27v<({WRpcu>qQzT_)l_p=Eq`Utin6GNY%qNq5s5^+>Fq0W*Z zR|SwKhty4U0W#k1Bex!mp?_ZrV> zUx{ER>9yPuz1&lEhHOu@VuFazcZmms<8(o7(o+ zF4*#+gF-{4cQhIbY&@GsE+eU;EwtZa%ioe-CZq}V}-6>9S@ zSc3gVpQd_Jm&kEoDSO~HWV8k99HpfEQJN461&;ai`($?q^Z;&~t7cEirn3LaewkC< zA>=J|pZ2!!JN@PTpS_xAhNq>wzq_Glou{oYe{iqVT4|=&BleS}>8Xs3s|CvQU^p&m z(@Xq*Ps|f~3iaV6yD3&S&#>&bPOxR#ber9N-?q=%(h@e^5?{d&n=BmQX?`5nl}q6o z@h|x{nAxj@`$7{q&aMiAVB({2V*U+zh`*R8bV)iJ>fw#>m_0W7>GicfDl%ZQTN)m^ zg<9L+-hbUyT#93P&cN(BS!!lZmOZC$Zhz-0S3OUvHw8JL1Gp3UK(lTQ+V(ropT6$F zf#J#OFhhpcFhoyhOYrexinvK^g)?AC9BEo@ibVI3H71LBy=ATf1HvDCbI_gF*iS-yTl~*8*qP-1^xwvKnQPS-)hL&jpD; zFU_6dnd@DFj@{*5_w%Cjdgb|C@6a3de4v%|tI|+!3gzoT%Enw`H}l2Bb|Q<_roOmH zENuEF))X&^S4;yelWn){LnH1-yv9zsP{eurTw4`u6=WCUOc%vr;&~wiPuq7c3K8WJ zT8L}Js^T8pjXJ_&eg(IXeZ-7rHo`N|60W3P^bx8qHG`ZEcE@*pmo`Nmt5lKGrHP>> zfhE3Uo>#7PXK6>joU2*2vPxtv$f}sr($Ow&ncMHF?-jhWJk#A5Tpe9;ps^>oIj_^d z47K6wDr+1dc2EIk3fBhL@LJd>zBTPPcQmgBoBOWmi@CD(i}g2KmTf-nM6rk;cAqWD zw#3R>r&%H`HO(IJlkk)8#rMS%WFSrnhxjQ%RdFohj1nyTAk3+~oDaRoiZV55i8?@C zfl_iZXeo_RlmA`MMAc(wczS43@Th-_cbsQ}E8F4E?UHNCd6Atzr$A2C+*s$3yn(K_ zt^uw=u4H$4cOUlzcR9D>eu3-15UeKk51)}QtMFowZRk`+X6y1(g+1aWQz!FWd>l8A zu+*_$vWBc*t#7S$Y!7T*?9c4C?ZfQ@?B(rOZIy8^KAZo=lX9=nK)AyX;cfgroJGs> zo3Wn#E`AYH#d&xysa!tR&io0s-6YiE_iH*du6BK{_EG5}uMA%eH3{7EHS$h#KXvJO z-JQc6eRGTCyw2{K(a4NnEHsj_^Rv~`aZeNXs<7aW2T9AN*kp0 zQ&Z$TX;!EUC>)=CYrLP`hh1@~okck|r?3Eo35rbSYVaBcaGoS@#<{!_kk%vUIb{ua-eI#_DhcH2^|>6U^Pv*iV@zm8cq z)ij69lPrGA9P1jKGh(gr)|*zJb%ix-zKzV%2YB?t@CZ%k7V-_mdFG(yrPXG=XugQE z)e>epy#Obx!_d(mH6o1GdM~Y@nkG}>Q6VGn%13*7cTZ=l+!{HXvX^Gx$vT(yFza;I zn`~G1pq%Tu#~l-0%e@QzS%F!>-NDPjP;dsA^p(P;;R&AhKaS1u>Cv9|7j|2$k4Ua?nGLXped)*=_eR>nN!Rq&3_yB=)3Ei>P~3- zs%LPQnT5y)`hk4H9C{scg64CV_(kfFdI(<*)np%b7CeRB>8EgK{6!WcJ*xJh6TxZz zJ)R{(FK0%9KX*m;m8_>(s_e_zPqWKsj|AOdclHj@98Tqk`Gn9xya}dcBjzr3hzgS& z+-Zx+b#SqcCk~O97)pIm^M~fXcAY-SkYV~^_^R2%RiN*av#CLF&kTga?-sjMov6F6 zEmT+Mi!;yZemrQq^!vVmLjLrMb;qF8h*Q%yguD7`Vt0@?w7WhE?J$<6=P44M`@8g z%6qh|wyw61c9mu~9|valBC$W1{eEeZN}zVp`Ah@aM-C(Vf?%f(9}F*5#nAWo1Nz67 zT9JdI4n-cdjWmU{4fuh~bSR{!vcvf1npiENKdh^&+o5^E-{2}SG&wa?Jvb{+Q+lIx z2@%R$PnZ17>C2OwB|rLU_}wXUbJp=}Hvd}g$DFK;Ng1)(ck^bt&x_r{E%>9_a>lc! zUV59J)25&XJi<(3h9kjXuzHQ{avPtbGqRd-f-ir9`a(Q_ z7iz6)V0e6Jy=oGjq%LQv7xg>(&zK+9bp34g1^N)q3&nmC`*? zTe(LhlK+s%wiSx7R`dtbqjU`96lA1#|9LLyQ*wu2$J0;YGtbX!nbSVIQLdQV7b(E= zo!$Ip!oT=8`fK_Zy2aXUx~}^Ay7%e;6CyW~YEG|B*UvN;Gqu(?Qa7cahU_31{1yi) zkBJ#{Cw3D%2CBxpn5iEI(`u%?P9~_D46V~xyW7r2ca2h+FKY|wSfaA(G_jHDMjoVo zQ6{#Cb|aD-{?@NyHiX{!Ke*lg6-t8YAN;9_@DeQZ6y#~rE2cC{R{w6C5zH`WW#!Dw zev^ACZ(z=mTsq(8d>B{~?#Ncsgw+o;HMA-EJ%&n#%G!Ngb$%avN4-(EMt8*Y$~xP; zL)Vs7;J9cNnn_$F9ueQrpN*qeviZCkZ|5PfI1`jCr8~0i70pG{X6vh{2T@Hd6LgtO zKJ)=IXfRyEiO3zT!Hv}|(tk4S(~h8a$yWbC@0LJq~d%BujAM&XD3y$Yffn!o-I7tga zSEQTXdW8eC$EC0OUHoT4vYedsvu;L8ZvX7U^wqy-r=7{WD0~$^1P=!GNRi<#Oc@?CFZL*Wi@ z4o8LND`8a$PShPV^|D>IZL>|asmu@cahhrLHR8zs{03A8(jMv=+8DaWhUz0$jyG5TG)XZmLP9$KTWvv!7Ni(129hIag<@|T<>sY9LV zznC2OsC&V)*)^OP`l6~tFXg9cPZ(yKMwsFvq9WpKCoE?*>$&bsH!_1N%Z*c;bwv%| zbTj!>?gn#8l^s%0cW9s#=tm4vZBr@-wt8PWj~5WROEVgz4^Equ-aO;a)C#|x$y%Wtq;KscZ=9hjbVnWbM-gOJ0pHZe6+nXb=M4G)?r?+ zV`Mggf35av*6I%E+GsAp%^`sf=9MZaPob@B3QE@GU<;qh+rjU$e$OJ740n?=l5Iww`Q7t(iNdLLRgIxm(XXf+Vj=vL4Ty=%4{e-z ztL=sDv@uHaj@2=vxCuCe_U8^UZdOpg($&{*)I6e1VK4MEDRPu*2M7lF;9-I%fSg{F z|A0HjHL_63?VZ~s`+N4e{2}(~g5oYO>~Xv4EHh zW&(8?d7|r8VJekZ(&3L54o<+}@3*^4=UI4ipBNVaNX&$d?S~{PsM?gvIkE5jpSqM3q6y4$E6|h_B{OPAK3z) z*N#QjoTN2qy7E5m4&O;_(il-O-vF0pm-@ZBuI47rVzJzCW;8vB)-(5*KNx}PMLNMw ztsDAF*&(eKUqScZ&{Ni(CRp8V+#%t%>#}pGlW@*={&Lc;*RD!}S19M6?f&ig?j7bI z8OQ}U&xf`-B?|J-CNxN{TB3o&gjSKll3h2w`MvGPGk$!ZP5vs%-_bfCa`vT z7qtRuNngSPl^)W`z%Ji+uhHAqv&o$yJaA2PK6GT*E%w5~dxeh*$J)Q!L-uoytInRT zk?0AT+*RC9+|@kgy{~;)eqUgDZ~}5*o$%Y3Kzca@p8yA1;YRe}uG4*);b8l8LoetF zUrKXXJ3&9j;4+LdJ~iGo4l@=vN`^~@uwjVtKjR)_*jU+=Xsl)M>CU2}nxd_*y|4L8 zGYZP-q8x!fY#zJ{<%rA>A#W3V1eW`^`p$TNcm}yE3I|=SoRlNSzQ=x-Ss86{&k(@)c0vuHk#q))Y_f~C0SEKUVm%$>}QOdAZx^o{jf zbpv!obW1g})D^(kdWs$dMb{>;t3HKF$yI}{zy-g>|IAw#s=VFkeq41mUA2#16PLYDB#ZS&H;JH82iD)2MF1y6t{Gd4IocqTYo z`X-kN?GOJ$%pk|%DIUR;U~{;I=qqqe|BJi*eLj?)JI;d)CjKw2l@l?lAH+?bg#0kBN;e)OW`Xn7S*VL`}Gi)1XE;SGi)h(fO@(JlX zl%dQ03!p8k<5}z;E);R)BfYSg)!&uqdg>}8n1n)MqUWe* zo@cQ)**o2L#XrtJ!)Nvt@vZPp@%I9srbw_rvMG`9{5K4LR`n+9(H9wpbMm9L5A{va zgF0=>HFdNEFuA^LJ!xHIy=Q%5wOM;xDqAwmU(5~6g7LUvuima}r0c2Op*f*m%1`9p zF*%fs1gtb@;9JRugPp{dfqwp@zU$tho_6jY!VA|h*9Yfu=SrvG{MWS;|5o2M2wAXe zT^e-654v^>@$M{ltoMU=jjtl8Lq~jQ|Q{fpHq54TQ zrv}g#_6Gkzvs%~QFxvRi#97u_vT%>bTW4AOS?gH4Se}?&<{Wba%WHGcG!=PkZ4A5g z**a3!4|%Cs>P37f>@Lmdw$w2~3f~D8Rfb59p|#BmIQ^%6TaaYg*fYZY02=H8LM`EM zAbI)7c-wC9x znBYa}jl33=^76zkvN_$2eZ%KzX6tl@HOAAX<>p?NOO{)fbjx+iHcJ=FJ+s@~%`(c; z5BECVBpI6+&l+roqI#>Y1Nu~xu*3A=8e)Gm)7!`((!Rcd=~gK12(}UH1X}yA!uO!@ zHSj8)rs(x`cPHY>*y|Ep>8@%*31N$CvMW(2hO24e9^gLV4!Wb<2|^dwpRUU;tMC#U zbGQ4aXRSBNx6k*;-#~m7+#;6%8-G1P(a~%QH$wdonwu|rm*KNcaa z!g3J?sj%g##b{|^{*LaSXk2Xc8@d<@_5WbL=+#_TU*NxUpK;QSpsP_%A`4X8^PzZD z_?}=rBylGO_WNJ?=VJBqdNVx@-9e$T`;0J9=qpqZ8VjQYtFYCj6_yIa@NUKkznvAF zL-72!bnbR4&R)2-Qv&0D>3-+Q^zQN(29m^qk`{jT)MC$eyk`#3 zm#7Qm0pg~LR5d_9FHspL)ACwL34RT>!_z(+ddhsP+_9b(p1z)&?*H6IcQ+wXsE2iN zimQ!ln=|BC?)YF|ZjW$mM|$8q=XLb<&bcNDz1{!eO}*#SgBP2QT)uxpH&ri5JH3xR zjikLN>Mt5mtHYj`X?S4VVEV)S&D_Vb+j0`KwRp>5bH3@Ysj4Z}IM%qxa7%C0m(xu^ z`l3qnng7l0XIn6}=y9OO>Bxgf3|^s9gWtCWzRFQbo;*%gq?zz6Y!Ukf4*4qMJ$>z7 z>~88#7d8o9h1Eh0VVmoMYqZPf+~wq)^Blb#=NyADWoVCQpA|j`d)yN}y}cuSz5Fu* zYsJYyWPzhwXe2(6J?ImR$adqKs%vQmX`ARotb~n?Z;UHVt;{#gW=j)`%e>m$%KX@* zHf=ZlHe80nvORK1b2KrUzp$F9z^-h|u4dlQ)#*J{5o#iQ%#*>OSPQRr^KcJ%^(rcQ zSqTmZ#*5Sa@xED}wSwSkATUC2cqSr*R_Htyb^Szs>3ru%=MiU`qnqQ9Lv+-0w#90B zQ`iju##HZR@HF%M&#?mykhaMWFh`CgzL5v%vg{tNg}SV!l(xREvc9WfnDJkeVA^D^ z0Il6*OMgq2`ET<_(=X#7V-I6r!xw#Z{abC8CJucvBfkO+zcg^9e6)(bLUp2&$eSP` zkHnmq_MJvIrUv{b^^pRcsc@KgRw3f3g~-j@#-COH(45n4 z(2q0xGJZ4NH($5ZwtcXjwY87<&$hwxw>iRS(%00Pv>!ANHCD}R^-rGQ|Kv8Yv!M+9 zo9V`EqMZ~?{|n~KS?V{qKC9rftVuLgT?s!$KR7};DSrfS=+9s=v6a7<_otBUs9e}G z?@M+<_8-}2vZcJ|_L{DS?ggHXU>j@PLxoeAvF&x%bJlhaadj2ic#M8C(q^;ejF6&g zNd00|yhFV~yH+<$-^SoE%rG*h!KNLid!{O8gGI0$wzjoRvmLPp%=Jv9;fnUQ`Uj_F z=h7Cc7@moBAeY<&2b{%OcY*2-6iJVXv*a6)_|}r8NaQi%sVPR(2Tf)-curl3y2y#h z2+da3$-MM1@VC$5UhK-ZUxRljEw^g!*IaLI(}LsnSl3qfNzX9v8qZwMa_dt6Y)eE<^s3C;8_IyLJHXO3HX#I@(1ZBtKmcy z0Ut$k%;@`p!tyt93)Gc$Nc(sh`mPAloM1!I1TWG|Omprzwn4dexnDArZGFHXE2V8o@t=TdFO559$(s>2-ROKSS#^A|S`F=d$! zn~$MVX>Zl8J5w92Evk91o`IFH2ltvC!EB^m)DfKW-oeYqg^MCH z1PK<+;7I6c*IA4|N`Jt#l9dHtd1D17rwLJ&Yq*fE!P=mOUGlor!ceNS-~Rck}Q}wKezXCMhnN?v%T~D z7XmlLcG7uiuiOnhm%6IfgiNd;3*oDr&#Yw+^EK2>H8XXC^lHOLqZZFlO>;-{H&X@k za?@<%Qo~k#qApo8Ruiwj#VrF9a}Zd5v*;TjL{$XedmMDH>%m=*0;j(ar`ruk1-XIm zBnyn08sUb>eGrg$P!mj#1h9I3f%kL-&YQ8yelXVZf_uapfj$29zMciSbx*O;cH8h+ztj7$hwXvyjlVOeio9>17IQZ)Md<%Xm-21~(56B?- z6{0thN*zQ$e=tr%qw)6@BTo_cLA9z$5TL5K!&AaK^boFsgp?RA3{?%gLoGu#Fj2}Y zujKx6HaKWg(bakca&8U(EZ;is1y7_W(LDxLLpj$LXNh4JR{AQGnEpc4_yLLX|(D%!H`ABLu64Hvu-ou{$ zo+IuWsEP&(3Bo(q23JGZYbTHI%In$ynEF4R5JYYlY_I;c>a7zP>6=|}56L8I^zJ&e`7 zmz&6qJNpiNK91Q- zHv|Xs3)O;(0$Xi2*&E8gzsWzyCeUN71r2kRN~hWf@|OV5d4teX*T@7|9}s)JkQz9wuI&I>C9r?B1K#xu&>!B^K` zHSjR-LVN_3&}w-eSh9JcqhL3VSFywbVmNwu8PqwtBe;}9SO>cX?Brd15$Jk;s1vaY zjMkjh#A(ZE`$H8oLEAuEQmex^_EE!V^E6vDU7;><^CS6#c)L2Y_m~#U1G*828msUO zhDo^R$qMkZy1_b|tcpc{I2joe%fhw77eL->7<#NsR+=jTTwhVyCryz)f+uVVCW7~8rag{C52fPNC&wlr2_Zzp~)6+wES9mM=6yH04_kdPR6{iMsgD0g9 z@;*6Md7@Mg?E$+gHQWz*8cD=8atH-(By*h^#}?-vaLxGle1G)=b+l#~^fIX$32vF% z+G5&?@y_Mfmw% z;W|%;8h{$NOjR4)!bxNVNVoOrOnMUIWR^li)qy(%{ZJIvzuEi^{CtFxXbJex4WW%V z$4%fGaZb$SN3+%0Oy(>zj%mWMOfFprO0I{_qkn=WxtbnBccNwL0yU57M#WHCN&vT^ zfcy;V-4L+u9K<$aEK#WX2aLp<;8MA9#qpt|VEN_9i)9hdZ82#A-pAH(ZF~G9{P%qI zd^z4l-p1ZK-l5*5-Y?!#zA3&7J{kR_nf`Qt%fMogfBzNFh_W~qglQxE?gyk_QcZa~ z*x(bDGNEf=OL)UOP<{0Ui-#avko!muD|!^Yk(TKR%rB6STd-T%FKh%?0j_%j6Y#6- z5Vjg?WWRwhYK3!lEFFcXGzAl^C)96r1SvWe`!u3m=>)KW%yc4MiH--Gv@tyrM9JIW zlU{||?99)j*{9Obt6gvCUHUgQR*KM1f?fsq{2B zEtoDY7wd~=F%87tXMwbUMro=XCMo{K*CWuRU_nrC4;Tk5d6Kvs0|j9Nn|7HD)!XFbUt01`N_~& z<&s!07||c`Yd1RsTE?aL-4O7PIX0Quz?5VBxGxFxDXJm$8BESf;6nBw^1y?QLIPWz z@K?}0O`#l63GXQfKxMpwyK(}V@);lw`jkRk%YLOlNSDPGBiNu0ISZ7$J7Dv6k!xZN zx5#SQC~Gj4TLA9rLg}D%9K_W_(ln4r`+^R>UP_i)%G2aEAlH6|bM=z^T)r=_kf(#c zIT1|TMA<680*m`ka7$Gnmi~_$-3N@@yULkRb=-v|sbh+edvKiil z14K9E}^C?_S2sEbKVEvwz-^okl2hi9(k>}$sJ;oI-kk^7YdIMA1yP#~V zutRvI4L`8cT`b z`<^8yk`1vp2eAU}Kn3*#S6x=+LsFFny^H_lG_6u?L()|Nm}@NY6r}K(|IfSBq0V|8 zJ^|K#OVp3S5RbFcHBc2Nfzmh@?B2m}R%yT~o3B(s_h23<#Y^OW zkNklm?@c376`SYB^Gs;!zWy0F$yI7`Ba3;Wr7-#+&CvcGfB+BdNi$ z?Fw4v7Vs|LLJR!@gv?vSc98u4BUHFE-Bf9KueX3j=~G@Pua!HX%N_$W?=~o&r@?5v ziTCI}eqT{Z0o(f+II|hj6KS4glm-PwF%sW+zW+M)PP_gAZ zPrDfR98Yy$nLyQGQ~7r2l-POoCe^UsjPbRV^7U;Ss@5&a+fO}Z_*2(4RFS~Ev& zR$oJXP>ov*^0~~c#V0JsK)*+K#HXAF_0M3^NmM4fVuf>r>WA7Z_vOCw3u);8Gm%YL zwKw2t>;#(cNo5~u{LMIQd@ud`E5&&bD>w%G7(z ze}^E^Z=upxzAU+d2ZOtU<%0wgLB5GKgX&;x@Qk=ryelpQ=e=Lxn*S-jw7OR9kaeo2Z#14W31*wMm-V^j zAL~7{V7g=aY}l>uq%&$iqyC=5Nno1yXSRW&Jd}PxC1XduK*gg@Jxn$rvxt0rKfjTK zd|VzS{S~Y#E(z2MjQ7X;OX6p%AM`VEN3gye2j(MB{-)N!!Ih|fsj05B>96QLP&)pt zpQj_Vcho)kJ8UCnDD@WZs5zloO1#`n+7m2>%I~x2M1n-F=n+q0_b(9liC2*wF*1;f zEu*CGu4k6Jfe>_7cV4wGE8K$9JC%PgZ*Shly#4vD3SZdoxN& zcN)i=%iEZUN|BQyM@IM~thNT$j^9cZ9HG}%Xu>25B zuC{|-Hk371hqtM%rQA9=?*EZ9*mCZ>QGz1e4;b!XeB` z?7rP%8+n7$L3NKjPEY54@-doP+Gt&}ZlX@DYoQ&XNm0M%2XQ6XTFeys9BRcluv$kG zors^vhD!|(fVS2RLi&AhCk~3K{`_IZVU>_wazj}xw3xjUkJ?4PmsVg|*o zkDea2EJ7W5$+{lO;5G)AUeZ6;rlX^zXd0obu}0mJTYy@o484e07Df(0aA9D(FTpLj zhPza_+f5w~`)21f!RxNz>xt_nLV98z6{bgU>HIu3L?)Vm=3h-K4JO&#WUeGz6lc`e z)IG|Cn$bl2s75#e)uB&QKTr`}Mq1@wWr4I`tmNP3S&07o>a44Hj(rd&jyh_%-B(B7BQ_0pZz#p-kQ zjSQnK=dHD^e@29?<*n_lPmE1;wbkdD1;iHleIV8oa&~bPK!N*L{)hb6h25OrggozK zv5itO+=TFwdyu3OPya!!#M9bE;?1@D(+eG%d6rHTR)i zD8^Q%BZ>FQN@-DGi!aZUC5(pGroB+!UD6W)4o8bXNPHbkfp7U1&MoJqH^J&6m#xvuSO4vo*6AiRg2skQQC4ruhJZ6_mYRe5&h;X z>Y3)5>|&kooPC_tTzWSJ=fhQhGh`Q9aB}Pkcl1tlrsqm!g1^KP;&ria@Ox0Nya=VJ zYEoYM5oY(-)Nb_`HK`fN2iW1vIkGe|ska5U1~&SdfsE7KqwyAc|MUL}v_qG-fqWRd z|8ubHyI@B3L#zVryxkk}jPm4zFR?`!>nd;#7IL7I%5c@QXJBfdQLx*-*D=Y}-K7s~ zm5+xq*hAcSZ86I$+uF#eXkW~mn7i>m;>W}^jjI*IL=TMVXp4%tV*6#O8qvji-}1=z z*80M--CV`^kM61Z3EPQKg$kte{s>>J=YNyZIhaphaLp2`dFpzT{H4VY;ui2kx5!yI zQCWgx11|yxe2;xe{)PSzVtKi4s0hJRqnTaob?yScfs5eIG4XU!l2WNdb0ll}J{JvO`%d3Z}ZY29^Zp&?~?|8yovi?7p~^;-!j-3BM9X#P5zR z9a|8&-B!zb$|9O~S*lo?ShbOhA}2=wjA{|pGb&_jV{WNi!%ZT~ggS_oJ=a~k94#F6 z9Mv3`oSZPiljsZiUIto-k-;YTE-s5r@wQofxt{%=rRbmR@?7;!3A}|q>Ak9e)Uc^+ zU#>o`SWP>K9pTZ+aCjfWfjNO9;;%rLU`?qrXw8ApAoRliK~Hf%lz_cM|H)r41HS2Z z_!{^V{Kx!yaf>)xDyuZd`Q?iI6P1uQaM9gTs8l#Jzim!bTHo|OX{$45=h_N;dM8SD zhSK-9436m@U!~ZTVtW&^i>`_%;%dg!K%(8S$dxuFLK|s}OpD$hQ!RE*Y))*e*po3Y zqW-kjHa*cyVBV-U23g-g;kDz3UF*;|>$o!91AWZ`!-9*0rGkxu(P9^WYfM{i3ipJ$ zt`DvPOyU0WV73oWzyL`VYN1LYGRP1r+2!O<)$mY5&?Dml=X_bd=DwhRzgS#KQx=7L z6F15E)N=GZ%9Dl2Z9N8h=7V5Vuu8C-*ew_iUXesODnugns5((nwFanSPwktYl)XMT))^7}rdrMYV~nBVmfS zZs2!dw{!-{V(a9o!Q+8rKAUHmu+llx@zZezTBF;-JGa|g75x-@a5Pk|i;y;PM>+uB zQJp|}^kVLNUEa06v;L*x-?%T|!Y#>mNRl$M>zKw279+AD{N2|=$;b~d;AB=3>D*Nb z4Nki~Nh>*)C{1n-cR{WGT_RAYjrCq~x`p=nU$gSEys4wpzNa5aublO-vx~P{$j0^8 zOt;;MnH1ltNM!uqan{)C(W@dn<~&PN+e+Id_-uE?vPEVU)g-hi5?ORrT&!|d(!4*=iM$W@Zasx9Ty;2<$jYN=gs@lk+m@a<@88IPX7ZTkb z`>C9vIp5NcrI${7mD(k1WnNeSL7Hfz>0zoJO~$6je~CX9+bLEZ$=h6(mFC@+PS$(2 zebLq9Y8Gvg@H`S-3|?xeuyms0pAs0SO09kD_Bu(smux= z1z)ffw79(F5480g-RoR;9Tgon9qEq#u17AP$LF2kGY1ACQL84N#A?DK*Ko&GM^{u= zdtEo(d7j1KEDe_Lg>>lrT&68J;kt0b8^ySg6?2AcM`n_RR3oSkYeMnY96j9xhDExH zfoiWx2<=5*@0(;*T8kagalPW7=PfO)DLk8-mYth^GnG#3k&%{j&vDV$BNSp+7}Fv~ z#mtXi5uX&-CboLyP)mgAtg)QYZv16#Z%d7K#hZ&ZOc-CRR02`7MQoD@pD9{@lYdMl zs&2}mV8!68fD)Jl56?3BwxWcpg|8ypYhY-noD?h>_~L8qo$FpIqzLPUlAg72L!b6G z^p5l%^EgmT>YVHBF@+5Zhhs9(k>39{T5Zvd<9RIJ?2P+E8})CsSUV)mN)M7hBo zJx7ma6?pd#a1`&t)KTJFbI-vn$iuX&8etB=I>WQO_#f&T>f-Pb4b{eJkE& zD%b~U8x1k9iY3OA4l;qd94;3gD8CN22?o8VyxV+_y^RBZiE{%!WWf03A@W%9vA>09 zh-*;(@Z1xbzKk`Q2QvQ4X_2?wK2ezODdC@mKC4-DUM}U>8}!%pTXb4&cddvF!0vo3YIz;o6O`0zbS4>1O`+>kO-T>=hiY2*yV5)m=gD^r z$uF6`Br_>>TH4v{2Kki*Li(h7$_Gr#EY~A`+ge-78h2=1+-eHP z{8zkMNv>4ak{gSDjy+<_*QfCv>EX!c9;xg_7DHE6TkN-Vx`WZdDSj8uP-`4F3-=Xvwx1K~`O~BwK5$3bP{_Q~Cz`!@GW-fH_!*C+Sx;9Q~z)d23~K3orNh_^Az7%lmpc*ZP-a{s-dxn*(m zqPVOghJ^P;ql=u2SsHQPv|QJoAIA2h4^j%IV^x^$Yy~z0D{nq>@CK=7DsQDjAQ*N) z4c;YK7MjDL>KT|LT=%S>_KTAapeg~i9buHKhGXpc--~TdsN&D z-|#%&c;Se1RzX^Bi@Y~EZ|ozzZ^Rx%Gxd4>TWgJ|2R3cQ42w&@OLK!;K@Wu5bswF; z`7~E_Zwz%!8;zF?J9L-%kznN|$bX2(y{84yHPSi9@e<_Q0geT(-7d3W@qY3x5_9By zc*4~&5$#A{Ktk?8?kgX!U27O)t{CwsDi){kDY27c#z&>u%369Dzu`Pesei+9 zUz3~1CP0V0hRLAIP5qPmO4*wnPHOkFL-y{RL}9d#@g0*HiH1j`s)NW+&n<<7;p<4{PWJCtP35}j z$68BAHjiSXOy+BbC+b@4ztnl+4^S>1@(nZ}^|j3V&66w|b0z&=?mYEM=@MAvS?F{X zj<)Xujq{+ufkm(k-IxW)T3i(<6R3_H!jfX5q?bp8suM@3YHVrsY%OhQZklVpY3XOJ zW{a_DZLM)Sv05gZHD(zav|omL;J@_Nximl2YQ83$Ne@6zBvMrd>Xine&B&Wqkjyv+ zN}NGZ$OP#jOcvXi3$c~ienfxJw~u;~3cqAe{+*OOqHXBuVxgb=y-(hf7@iy?^L+W)X*a%7oPq2Pq49)|s z{Q%AIskC*d!zb3Z*`x)Bnyr%bu8DHT!<%x~#={Yn@!cs~V-= zVDd(Eif$Mkh^T8Btl!0br}B|L_6y39#_S>WdfgCHd-E8hRo{v4PVETU13QJ94hI+@ z=k4WCH&*gy_+JKhOY`J=n7s9u`Uj5rZ~2D%t#IW!q|u?n)L}Nrdv#9?DW-Rpf#yQf z0yAZrXPjsp4jRCp<~rtnmLJwA>%Z2r)=Q?5#zDFrYCC(D-b*zkNMfE!Lp&qqL32`- zoQ4T`HBeA8$u`t;>VGcL7NB^RH5?0uXk^BJhJB%?z4Y(Cb$Rq){9xt4iClDZIx6qcrUQa z&mgh4Z}2k`kY*4aX@wi3DWPj&7--~8cB9|8*H9lMhj@KE<62`q(`ZX$YY|)Xh@-Zv zmYL>ShK9PXd}UTcH9(f$t#Bz~m#Q3f8(9J4>2$Iwy^s8gSx6RUF>%y(ss_CP^S5-$ z85$fM=1p=8%AJz>Df#GkZF2LUtyAA-l`Ht_9uRPg!^G~s>28bTdhVmlxf!0!v$@4x zm;8f>8QLUMg{WY3Qq<%~(fm`pj_r&5p&*g~R$`srr7dj?n-WZG^d5C>wxcRDc*&dO zjCGnEwH#ZVvxQ^ceE}NN*3t4F=m(}qesPI7GO$jp44&*HkVEa{9p(xfr?u#c8D1K@ zqoWuwjMER&y0lHS=e4CxS;i>Sp$MO~Pt?Q6s@4zY5r$~(F?K2a6;t6|&^(MG&x5*I zot2Yw>AIX3gAEG#a@Me3twK5iE3NUCa3wmAI6gpGTh6n^=Llqghy6+FuGGe)y-G06 ze;x|y80n6@B~+DML;vRXawD{BwVeL2ewlWn?xn`5sRQ@UZp}CCNxjQxwKTO>L?Zhe z+iojsE@5b|`M?!`^mPGqk_O})>IGezeFeA7FSZ|>OSLDL5L%>h1>7C-cE-cJHm2J=XoYpL>O2IbQ zr(k_XtN&^9$CQlyH)e5U8?#YYiqA#1p@#0vR8;?=KW^-48D(y1IHGILohG8?KYWda zKHyl-aqM>{x*vPxz?WbRyyt~V`_O5mRUVaE!++BZJtAjFMd+yF>|qf5dEFA-MO_bc z%-d=*`BB_zb`APeSJiX0M-27NMQlwYzC^GQ6)Z1J>AJq^tE`urM{Xm#Vp2MX+svQi zduy)p)3}Gse`HV9zEE`~Bv(cnOqtM5)n(Od=p`lwJ9#fVUgoY$J(6tF%RC(ht*xxS(>fz2@Wz( zwk)vJH$OI4HB~pv)Ll^bhtErbO5i3E*oX36)f4%1Tp~B0DMpD{S_y z^E%l7@%1MDW1HzI>x77QQHLTrSneB4x@b*9%{$E{-BYL-i`#bDHe0uvFX*zlm83(i zCtmkT!a_JHx(dZT&EQq&D7Fl~6lW=tyjrOlJ{js9KB)2#H;5I;<*`r|nK9s*WKt1G zZyZYvWH#~bHQn_~jAh_ExMHepXbo*=Kdu?O4D)fjrn6CPZeraOan-7rCmM6q71%e_ z9#s%obaz479f13_nUB&e~L!Ecl591 zq*p1se~tg$KC?y6+uUh|j>1XyyAGdotuxv6#Qm@DozzwJlitkDfLGUQxoQ2&`pmM) z(%)*Zw6QETKQj$CH!y8BR@Hyg)aENO*Km?F%l`$Y_*|ZJ_(Ki~Q{2Zr)qMQ}cg3U9 zFlBmpE%Kl5QHwE|Od)+_PogX|!5-;>T%ha=Kcz*ky#9shT7(oC8`;gW!7x=5#T92N zk$tFRtU*7+8WHm-P91+O{#{h5i0Zm|$OwBc^$Mkhju8FW-D-z!tf7y}n%DIA(DlG@PjP33!mR~k3qA19jdVpi{e|=Gm2x^{)=yoN z6q_RbXp_A5r~E5DBPM4`!5(`Fm*3UW&47Sa$NSh@(QlJ>DznJ{xEtCi((0$)B==2%dXIyV4A0)qP!Fm4M-lm?%9-DWxZ@OO;_Xb-i z*Fx2iGCY|m3Cf^@ybM3{2c;|A0H*^-0-MBMax;23m!n;1X=~1~bTmKGT;)zt-;qN) znOH$@(-d0%js6`!zv!O$TCwk}Jq-`Jmt;qH3ka$^r_~KJUN!eK~G>TY<9DmPRKe$rdwnLvC%jn1t{ z8blxoUj2vRm*KRbmi90^pNI?3fYxGwsxGyMpJ{kw{TcN(YJTKN%P4($^?b|;+Y=1P znu(fvhDjzB+=_&5FPobb8N> zMVZ=cCUS2D z&!_8DrgU?AQfS|Zo3xFpuM_1-rmGf&87A1^fAF` z8)HYEU=WalCq@s(_iEKISyC1pr zoSK`+G(nQbL1^A=$joj|-3D{)3213qj6ye}{vj@hvz1wL19<~xQWwJc#B#7-S#}~b zlujf);VYQu-;gTGHA7oe6{sYp33r~W$bDsgBCWNq>M0VIzlSWU5?}^BXPUBA*a}QM zRg-uga-%|sk=i09oPa9jFqHdsLcH>)gnSx*J>OdIXm7UnyD!@>2TqHZ#1rE002$~7 zN8%#iQQu(y(Lk=)N~!=2+6w62Dk`a{HQ&ey%4g+wC{`7RQ)vKM^UaXcvyLoFz9u{b zW}oCysvn(9cZCvqGcwTIFh&07^Z^a>G}8k*g?${ucjB9XvHFc$%(dc*K)I8}USe0W zwV~cQ2%5)ldInt^s_+}Au}4zT(C@QkS)w_n(w9SR!9|Ql0&0ECIo~RO;}eZQqWlje z3>CwjivgXYWC;0Tp^l*-G)IcO8cFF@<;u`tEku=k1^TmYZ~_jLo&}2rw~JLo0q)wY zfK6;Fju&@`=b_!$Ev^(ti@n7W;yUp=m`0a_e<2mB4HSDrl<_J7cU`ZN#xqeobc@`BkxZ?L%31?t0W>2GNG{y_TjFQhfq#y$Nv zG!?1DN1y@Z!#$wQte{$meBVl7!R|n^>u2O+Im1oGEpefVKvQ&VCg}Kn95Yi>MrP~F|1kJ%X zOuniF%R!kG8O#;$i3gFy)L$GUZWmvQdL+E<4892Zf)U6Xse)YiaZqg9@m*hn9(y7b z*#t7C+dvaK3HspOpR4pary_x~3KFHeKrgb8*awUqimow`}tCtblE zJSCl!E=w2j`yWyW9{K;k=uDNPlrFd<8q-(h z8^r%k5m+zslu!8YxAD(ZFjv(f3w~sD_dbA-wj23A`|$T2g1+$!{v?5QsC1}a=%3IC z@dWs(I?(V@Y z5F}`D4H6sz1Pcy<;1E2xOK^90hr^xYIJdQt?(gm8_kTX-*=28cXLh=}y1Kght+x&U zDczMm2JW&gokNz0JNzbU&X>k0JVvguj#@eUD|3qmum& z>p%Wf>>yK68{J8MCR1N3q~zDKM;6lAFq4OD45kg|V(`S3hjsI*sWsiU+6 zWYgxPvw`6*0=Y~?9qOqpT?thSDxI(|(SR^&rvu0mau|D(eo__uHP3;U+`>Pha{D?| zCBF?E?gixM3?L~)p*iR+{6b~q(Uqe6MH^I$9tmXmJdDV$7@d@7fwm_Y|5(GoP4gOA zSMj_CnT{RcnIbdvVK?xZkNJ6e&=54RzaX7+L3#A@EXt2p+!*{0ezODE(wB z^NOdwxD*mKuw2Nv?B5j?vVU^B(g~gU$qsvl+o z^0FkVIQ76>mD_k}Mj3{=-g;rAnoE#_c~RepEY_!}dYcU>g~n!Q<1t?>a#{`f0pQU8 z0+(+!9&L>=Q_KMJzFLDhWFeo*H;4${kHip?yqCJDKudb-Gl{VT4bs?DOduXq*CImMTqEp$lrgku# zE43Asbq8PGR7AlVsh8Bp>=3dVTPPcmr?8AYCd*k{r5QcKDxm)I7}lNc(@ zxIO|A-UMSiH_ctJaZ`*FW{NQy`%NzcHOy!|uQ`vdu;ew@85_-#JPP>Uwq`we1}m&u zn6>x9a{X)7K;^+L!bvBRLa64N!ng6+sJ;DxRl+!}rJ%wiSwL5ai>S#u3Mlz#L=tnt zXRWOY@Eq$(M^q`iLUxfNG#y+rp}=G$C=b-F>^OE`PvJVtNqOK-h5~so0Qt4)A{toh zwb+$UAq9m(e^CAtlYqnVfJJwnIE>hWPG-^n#91yHlPP+ z{g+lRWY8T%<+)F=RmrHrG>512Z)PdfuY0dIMD4CpbO*VN8k3v28|c6C{1j&U>3jwD zuUw=IYsfcQeMm)eo`fS;0u^C-Q?i_wpqs5HxRc4e1gS+b)3#&;#%@7ijJA^Ps8aZW zbRsQiRp7}-();`&M!bnC?7wjv_G0EE!|x6%x*rph!~G;P*0GW;Pj!8VVbyyT*td$M$SP>qJ~}rvYC`4$zmTWbvC1MB81KXrhlI}NBWcZ z#xmr=mm(*`OCC)x0ju4IHo!c3ffl3tU?GN!2;@Coh76C0AtXvvBppOE@e_T8tgjT> z9HaDSuyl;zgRK?xExV4k?y#7^;*E-?dSoBG-sO6p@) zsLq?C7dYxjQB(9sr7v<{G%^5rvH37x z#gdUc9KQBY)SnHbV?=*?Li|W`(I3Q8T9$WWMM)&2Tb2AyX90l}N^8&?{0qH}%B}U- zOT>*wi%4!4Gck(tQwL%d+xRKEm@IyV@DW?xWoG!Zd8VI`5Z7&X!4G7zbuR0m3KhB!zE@gb;$ z8wWjlKzfonOt-=?^Q?sYXVSsQfT=-i0tb8$Sb{yg1@32-2t%G@HY;A~YPQwRi1GRf zRIy93GDI>kqzA=x zMS{q06(wDfJ^44OCHjbm$mG3%TCn%|K@w~JOLmAV*m+tYGWbsAuK7u{An$o3RzpuQ zmKww7j<#wbBDg`kvJ9F}EE0RgW>mNL!lOBYIbbEML2mLI^HE)rm&{a7A%CYR8IIXL zFY7CYTFuBTvJbzhfjcXjgs^wKzF3JJub+_(Qk{iUSrPe-*ukq{UKy^<7OOF)JCGT` z^z2q%^W(^%>xta<$>ccf>T+b#?I9Ynl}2FBO$8IhWRX*IThB>*QOoMiGMiDf1ew4G zSm)p?{RE%w0{ump$nd`^{^gG_C;Wr#Y%kp|;&^BBkF}f56t6`R^@=q(fHLKq$tbmo zF^7#37sV*i548>QLXumtF275+AY!(i3=;?Ohg?U}D57UGWDsoOCjNluWF^f9^fY}d{s13Y4r1{ym_iY+ z!HqLjGzC}MQD9*%iw>d!`DCGFm}o<1@ol7&7=_VYk<1s_k+*XR+EG;$A>-KJ)&zEr zjW=2-fAg|*FrCbk;G>KZ2|(&x2Dkk)@XoElbw86uGz+r4CeSCMlhp}XL1%e)__KBC zpZu9v4qmSe@(@)d3LwkBhgE_w*kn6$%_sB0WDU6pe-!+BVk`ekxx=d>=GKglrF)S- zwoe>`G>(v&;wt}&G1UzBR)#EMi+OHZ9M<{Mttk+qU1%LXQK^iXXflZt@UGE!pG0qH#Bbstd?+8XSyu3Pu+6m$~eTNL2#&o6FPi{b$y5kzx#Vg(bvbYRJ-GIVXfD1r6KH_L#;{>6A-B4SaWvMl)< z`uGdtSWRdRD>pRqFEI^%{wdPJT7y+=CD9gpjQcP$U%?{($&11Y6vLd|3VEN&WH#(q z4jRj=iE7{-JSJY#_53jAw@+B-O`-pQ@%aMl3XX;akXkKNmgtNtoxzHs4!MB&WddZh z7@qzQkmOgg5_4}D3B|l|2DAG>k&_l85kjyt<`v>(&0!TkBjag2qBsho8OtE0i~KjX z0hx~D;hj`Mp7eIGFm(Z{tsnW1|Dx_hE$(@kInF^sjcFq4x*kM-7KX0;O#1OE^bdI8 z9q0#9h_u8W>N=d&No*tKF(L=jQdr-7#;Xm$5Ix{`L@JriOTlZZ1`llxRWVa^BX9U! zWYmGU7#1iF7%L63^P&Fde$gB@v?glewtzR31OAqS)Dy2@x!f4J)#*!~jP++nXiXt{ zggmyEir>jpI^U{F--#Yz6qUAQ3we%t_7eE8j*H=FXEAbueBghP#>g!lNeHsTmJtg1 zwZXb`7IqV7@H~XWO0*((_%!+&E4gd%HZs9OX@T0unn_56E)$u=2HIP+qph*34X2-B+vf?Bj477$LuGUEW z3a|SPaxOR14wz$2%LcznK^{Gl{>+ z8PbAhXYCN1*ht1%ud#lQL2X=)J{wD#ppHm(^g=v7%?EFCy0|UgQw6;VATv-k#ndt2 zAz4em@uTqM60Nnui#Ss$zJm6{Oj4J2z>Jq$6on?d;xoxfaF|5W(&Bes3;WZjL=@dX zZt;`w{S;W2OS}N)`(}JFo6CQsFZe2;;6K7n&V-LWS|o|~KqxoTmT;Sx!>;rHFpmv{ zO?-;EZH{P9;l&e`))fE1Ix<=cBfc0j+aywp`bBNJl3YT6426cSfcLC`@xwu8V8wMA zna0n_RWaL4hOa*wmefrTi)Ew|^u9PNWDTKpz^42ww0)L1VV&d&#I&}e6b^yctSW z>nS_V3*#Q%iM^=F&;%S3^}r>zl@%Ag(2vc@6p;>(d=)&t?U3v(^vfpL=!N2%63Km7 z5mW)MeMii33(RvQ40}SWNeFV%f5)E2a?JRZA=%0pjYGi0@|V?w9<$<6C$25D={@Y- z1o03f;}Y>&(HKKVEk;L@Vq%k6fcTV8lozA<0Cgz#v+Kg{JmK#!D_5bFNK^RGHxNhK zjsDICy*U9%^`s6_4OS#4*^OGf;aG2-q-U(Qbey>z`d7hfgfUkD7QH@vse-H%zelTC z%P@-2B$cpN?7wKB$?&YV; z`CC|&8saV8!jdtX+_X9$O;(u>nuhhQgS_C!t?A%RxWK2Qo?0@WN7iEwF9YwnI$evn zbONISc7)AFMbDj(ug5Awx+)sxCPt%hb|1*&3SwKOX>PESUc*e1gBG*4(s802c~9Hv z%Yd_5Vk*Gi)E5be<@_S9h}Wh^?8aTZ7E$1osv&BDrFbdELP0AhJ1cGl2Eya9cm~)5 zvXgeS5l~VOX%4Y~tP|tJ9Px4tK?)wV3X`*1&PPXxgg?T zbI46&5>P2I$V5-EX3=}1B zpP$BBDw6jV9f1j4%FDt}8w;PVIA)^Om`#VnI+muDQ32Y8SkoOc3Ra@87=_QGMJhdv zm0~E?uvf`lcApgD)gbd~q=A^tf5z_gA6QRhBfr5%vqU}E^$39a$cn4Zndu;Tl~<{=s_q zUqm4ai)q+Fm@mG-r~MD>2pgFSCY1lMMwjdTHzXAG&)$)&tRbxee>)%Th8SK8jKfg6 z0~TZjqIe@jZg{2d#ZAN%n*yPBpS;5SehJs@30qJG>$VtC271#BEN?f!7*UPv1b_4l z%v=}2?LQGL*jo|p)X6aV8S$pOh*{?*_b@lPzyNUq9>@c}T9m*p!%7i`7<7HiS9h^u zstjJ3{8-<$7KIUa`9mBh6=)P<24}$*-<-xmes9rF55)no9Dbz=&o&2bjZtwBYy%Tu zcej&F(6JTp=XyZXcEctQB581+%%BPQwSMq(hhWv+gcKlKz-99o^H&mPzdoWEY||)F6tmDv%nS>$#x}9O zAmTo62bP0PVhQeI2wFD{{`Ml|_dbH(+>bv%47@P*QbzG_ydr%5lEgs#c@B1Yj$pRh zg>_Ol%u>amnPbUwG8ftr#(rlOn9qu^Rdg3BSw5wHnw!;NCDmf~zH%?FQa2v*y#p&_kA1+keg<0la7 z-)gorFM?rZ0rKMN2ezT|@j%~l?;cMz7P+wJ0u<4-s#2DkECBkF6Z2gdEjUs z?pfjK?rr01;9n9rrr$6R@%La9@1uRN=XdROpAXp?dL3ES#oT3_2kqBUpXw5O3TF4V zY#6FYu25fT+;-ny+)>fd%TWOJ@b23p!7h-bRk5XNhirZ9x9qzdQK;#4(HTI^DX(Kc z7{037FQUpxoO(@duM}5iu*qPCTLS)w$6$_V1k}zu)&{%94HQi&4(7orSb5YIlemT* zh#6?*bVRp{n7<;58ww0rvS)UNCw+SQ7vPy+WJGwU`D*zM-)>)X&#JW4Bg*|uM93kojnuo_4%k}Zdw%o2yFn%`;>j~!GA>cZR=w?sR za%t~TscX~Xh(7${ z{hJ>?DzOI=hh#L@ABa$({5OJS^CayIe{}=DNYYWMuaz@L=%w(dnJZ@Lm3e*Ue32!? zcDVA|JF4SY7OV_!BX6S)_BSF~9CkJHAaaKp$pX_4nD85b0jiGwgMVHimjTucs}H#h zWR^#(VxQ~qp~i9+_f&UTcSSIjHg$36$5dyUV?AmSg`>`x7kGypT3@w@GMOd74&Q~f zD2mw519*Ao=_KrrRZ;&|h3Zk0)plS!3R9b)-pvirXBl?m9XEl z&#KK!^Lb)Ft)=v~{pV=q?h%?de0_L)c)jqiV3IrRvN`wLm)QmbBSFC#`5t3OGW0xB z)-jbu!29hZ7W1KC%*urMrZ(M-`XT+m2=;?rx4RsNP}}Ku@IVx|-2zAAC0l*R80U4@ z2=^EFV0Tm3O~-l7!?0UyrUXX&274=bkE1U2`@jg}mU#`e9QGSqeJ9e2Czpu*@-g%+ zdvW^N+2|S3hh9$l(D`%0#0u#Jyxq+=R%L#cuQbo5H;demMhtc zMLo@4H(S5Rd7%fK^EIlTr2kK;5X~WYuqe@rN zz(HfE^#gWdN?}$3&l2{+I%8L{1R@UQ)Nl4tz@{|`cSRJ9SQ;J(9Tig4RoxL`pRW0| zcHs2MZ|h|1X4{Y6|3h7(00#-5eKN1e$M9TO8xNsfl?JM*9k-=|7igjFv6ik5P^T&j zz!cR+%>!<+e9nBX|J;+^PULr7aNM=kP#40hs*T*>IP*5LQvb8|TdCL`_F~WJqQ9*t zGQB|ZFA2%t#(nJ>bLvZhFXg_B|1#-|FXnE{xtNb%#n%PjrYDw9UX#{6z>FR&QCn@h z;9THr>}X(DY_W8&+J5RPhc0V|?E9CZ6ryOnL(=g0y2p)ce2*7uSSx0vwRFfch+oDz|$a zqg^2(5jH)2M|wEwc@OEjp&uc6X*;iNZd z?Yt`jMG=d62L7&&^eruonBop`0y{Uo?H`;YL(4{nXZ}5Fi!8roz8StC^uDXRql0ad zdIlV6zt~4QHaq4zR@%dC50r8w-n<_u=s)J|ajjS?ZO8oZXdIZSYauh`A}gwFP<{e)#!0YPR#MkuKXfYl8#s+T zKqGF#3=$2rkPbN=GHV*4`eXk_@U?$Qk5Ai~Iw9q7@;KC4sGe9Mp=W&AxclEO$MV?e zU{z=mS2tcy%$oWxJ;|HXXkr#Xe3+uo%3wWq80)>8?2UTQmd*V~$bj(s5j65)M9uIM z;K|74MA>)8EcQDMD}3!tj*t!%toh4$M_x z_FN_yb^~8wjwyqh^Ib_ZF%^i%NHdF36+X@y@L@goPxQz6`uaw}&-p83fBIAKZKtFb zNDE6_ou;RqNL!qqExi!fBYnsfY>nCu%YD&-GUjTmEt{|v;3l=Jdz3%fCpMbRRZeT4 zZI2yUUF%&6*b+B5pP?qFSGxlA34s<2vPav0aICQ(u{G5OE4f))vKcIhrLb#rocsi= zUR{=s>bJMB!9??teTF$`!;6;9q=fM6~0c2;-z^d@2SS!M@0^5Z5iy;n^ zj(wG@*u}et*lc#h#$zyhx8enWJjub!z&l9;8s~yF31czLdTNdY&c^^##C)TrQQOF5 zXhuPxp*kD=j2=MxGy~sLvi?{6k!?m?p4fdyy_ zMB^*qer5sJG6)Eu)@o%nhnfUWcL2~YfSFHs6n;@K^>4t{bc_`!9- zD_V!uV^!ca`@vPc2;3%idK-JD!?5lsi1kt`X2P>rK`a&15%m~_8L}<>$buqN7(4;H zesSP^5IhQToN|bJ)IiLmJk|lFurjKQI+HEnS+@}lMQP#2uKz8}tP8QvJpy~=O(9c- zzXI-S8&D)ut+C)StqbTlT8s(~z3XZKM-CMsBF8!EX_ zt$07!bBnRJKui4$hR@50T(?GiCIR}o1emp=h{j#z>##f79IedDvw|7eg~PzTe+91b zF*pw&;=rzgMSzDYfbm<8x8WVZXg8Gq&S&!F*sa^iPeL<4K{sruu~QV1tqK`egk($L zC&o=86)sfY<0SQonRc|okQt7A3X8LY~GVlBLz+yDm84}4T9is&fq zM@NDkb_V?$a$Q1~1B0~~|Nlj2(bVw`tDOra&Pmd&yEQyuzKNtA^S6%QU|fkAAFm|%BA0R|%C^9L{tQ*q1( z9&RJ{U$0`%{}W=YK4Bwn;F<_1L|N0*3PTZ+680+SQ>jGlJOI+QUPnxY$Ochfn#<45yx|^ zz8_)V=Q<=TcT}(9@0Ylb6vPX&qm>0o*&qXPRbUxvLyk2OPiTO6Vn@ic6aH^W8bU`) zBWj!#cTKSVC5S7gVFmsPagN*K0`?7dg1LA-*3WBTwYFj2md6%EGdJPyg=qf_U_B;7 z8dJdZIUDQnMey&J;CZq58)q%Y^UC0{1<~1^sNZ=Saf|D?(+l|j5}wcDozvj_Jch?r zd~y?#yaRo?j##ez#wC1u5yut$eK+{YQ$%B4<9LkYF1~*Y-@PO*2hYBbk@pmmd5hhl zbo2%z03xG5GC|V{V)aoTc*>uGeK83=un02V1VryPpaC|L4X}Gla4bUn?{Dq>Heh z)Gy(lu0e8-AjN3xA%4TyI$5BvQe!E!3q_ZpUvl?a1V^`qcLjb^zK4Q+ z_{Y#!Ahm=pI`QoQ?&%Zm?lxq10UCZ5cYYand=*kWk9W?Z6<5JiDy8xTxQi6@crxS= zgR?$i9K3)QybHdc0ckPBx*2{$$M@pU-sjMw+c^I+t|T89aph}}$>pF_uHpY1xb{O_ zQ3n4PhL+_+%X7oNMPamN$8%Olz>Z(% zIKz)KdGK0(T!{tSZQvar-jllH!F4mx4p2wpm6Q#GEL_mu?6^)IT)hA!UKr2P!sNs| zS@3>H@H>hWVbGFh;QWl>kp^u{!HAZBdGNoEXE`o>XpOYDCVs=jYl=I>h9In$6Tc{N zlU|(h6+Q44l8lD5UP1rl2)qla+<@d{zetUIiqBp{-ibl258#)-_o%erHdy;yI0{1_ ze!x*1mar~HPjx(MKwHXSJeGz1DTSjn{wj$>{#6V|L9{Io&X*R?4$O%iBbh>8^1jp1 z%Srgfm|#rx3-0p^eqHu|AlOS5TAhZ!5?~3wKnLC+cKQze_zwO42G1XH$1l(>V3W{- zCl~|wF=8IzSz4Y)c)p3&J%ENh#5Z1`?H_{L^yz=+zQ zN~?SuXFtI42%kR1@e1F66+Ghu?j;U)rw8R9f;Q$1&K9L0gJO_K<)DnJLShXunrh>y zfpJv>uch4N2&;|f+E~xk!gs6Sj8d>h`61^V(6=Z^Eju3JkeV%M|3F@VUXi1ZV;s2A zGj@F9z_SH?kXkL#Owy`I+x7kbM7*a%%CUGyO7bgSXW;*D_+NfYS|TZTInKTX z$7lRXEUa51`qTrx#AZG;fuRLb!!$gH;8}&{M&h+>Ul@r%+ohKwM@SKj%c2-D#h_co zA){hJO)DIn4T?fL^I=34fV48>+nI0$d43qK=EPr_AtU))0>7LbZ1*SJO(KquXmcDS zAlvvF(s+%w#X?f?I1+-NNZpe&(MNnsqBNvjzU!=y%G(ZC}ecgLY4I&>Z-lb{#UBMU>T!h&8v zBrIkWdRp3MIYZ^cqaY5cDY=5S)D`@;9nYB|jV$OImf+0Z@)&%Ws5(db;)Q& z23mx@DYRZnI3ALSN81zdjt6a)BTmjt?}Ad3)=i!#tzsIk>qG0M-T!{xr=X6=(Hb9= zqa4Mt_`bA)Qn%jWO7DWYBW<^|6j-34x1|QjIZlNXooIpd)1_|6hqNDd{O!i$`(cN4 zvg45BsVK%+DM+Vea3+*8iNYsQ_&+}+QwX0*nHR%1bKsD+N{+OgIC2H0oddrhWhQ4N zsl(s>1{;3aiBDudO3NniQqBn9?^)`yoZa-Gbr5KSJWI~m>5xi1WF@sqjv48#$vM#% z?1MDCmSa|qN2zb}h{YKhkYExdD5WSri9s(t$Kxsb>J=VOf|CFL5|lFh0;zt%m13Zy zaX3#}O3-uQkRy^}R+Xbrw&=Trkuvzdai!sZ>8nZ|QE=BL&XFF18PsEWj*h=%i{$4j zWTW8W4)%iV7bj%nhE(K3N=Wt*P*C_kA}HOE;A|j24GA6*_*?d&4cGg>f8Xte^akWR z|M$L}Z>6OE&mWOi1clUarkp?Iok;sD`(FAzB`_Pw`KbYE1SCXt%SkFv2 z`0uWRM`D#yKxqN&WoPU+t_4zbr8&(=1H*M6^wz_aLI!N z4t0!DWcw@tKIWcz%_s|$j~}s^=fJ#VL0(Qipeu{era+|Hk%!V3XuVM^5o{Wpffs5C zMC32ZYw8Blqk>Wym2xVvCg7QWNZSB!XsVsGt;m27ybyL`UtUrF(geHJ(;cAdm2x$N2n;})o{_kJOKR3cvy#KY$6b8orTAI zNdu0@tf*C91b81H=a#AoN-aXIuf{0;RVl(MSiAW`?ABEQf~5yynN_U_^A+$DaoA}e zOc$yrm94ytVH(>h({_v72C|5ZAB2b9K}2G_RV#3mjzrvkB_h|SiE7ScZJjq+ezPgJ zSRFN!Xo`qoKl6*YicGV?w7j}XD8_Q@ckFB}VkzW{6=LK;K22@Xm|Y9edAX!){tj#u zDs((&v4{x$Y4)@_Ba3V|?qdpFXEZUEkQDF&?F9}hgu0MbyO!8(SGDrA7(X3YWE^0v zZF%TZy(f^)hmd8p6wEjm8_#cNF6b!;5=Y%NLOGf&#MKEAFSufW#C>5nca-u zq^H%3iMdU#L$-KLNfbu(I4dwy5RlA^OLyhuHd{V$A zKDj5z9r^8lAsSfL%I+QJukLJ_xfZJz z=!eonF(IRdq; zx@t>aRz0d&dLB<6@>V-(7Bw>|H?jYhLm8`7<(t4{*xTC5DmjWN)2x=pT|FCC&3`E4 z>~q;gE>P}ivHdsf#tlbcGNb7&9?s3CkIYgXI$$Y+srn;GWuh>OL2fnVZV9WD#hZEu{fQE zo7JskEeFfSS0kRc-K=01(|&d|=5D=^l?F>|Lr(f+YZC9RR28NDaeSWkP`d?$VjfG& z;Iu<{w7Sz<-^j+sVHA{7kJ6s{E~BK?nuI$`+ecb$GivH7_H$Z(>k`RtPY<+6RF#fd zu2AmpLsP7kdQmz~+XFrKQ%l_&$ZEZ3|MER%5&LA={^0Y$nREPHY{Wh-HII8?7R$i%+4!~L`nbMz;5hEHbXS* z6rbo*d?_|h=v$f#737ZdlV&laxOUKx-2+%9+4+ew&`Rw)pb8>W7@R-UqVyZkq2{7qMQO>&T=J@l~

    PIZGsY`9%r%UBoXCRX5X3h%~8m!y5ehTw9t|qE8KT&%~G1AtrvN;yLL;-4QAQXzE?iZHd2og zWiLpEcuSdW>^lF^_%-y2D?a0P+9K-@c1D?D>(57<<5*`pmKQLJSv!<0+ET^Dd(YHu91U(aedKl-ox>-ySz zqiJWyy714A%$~A|_4INfSDj;!!P3RLR zV~kS|S!#qQ^HQyc|QcSOvpl3W( z+lJn9zr&1lz#Ep_Rea9e(>}l)AS$N);+q#fD=dT8NX@Jlhn;K4KG>(R_udn}#jLRV zkiDT*DdTJ)MzguMi*r^*M;_~7Mg@A@G2J?96j8eCSM2HTKm6s>I{J^=&t@v=9_N3W zJRvQ^?CqMKv#v``Y4s&NwX1zxmU@{CH8rDET-EqC&V@xwx?SK`PWP3xW@^u!b?6D~ zXY5lx5Z(CK_+~_?heZV|(NG;EuOfLg1TG;&J-Dh<~X3#>%a=yxQioA68HT(F}sAg}W z#E?v8r3|k=hgPt44SAsr3e2;%ic=Y3fk;+R6Q!oZ((aP7WsQ1ziI-q zpSsXQCQ#qGx0JhXlafqwj9PrufnKvsukm3ZoRUXS9_W51GCJh zN*?VOGuf!k2N*fDT_JglM+t|-BS#KeflgG%xnhmx>3NW`lU-e|tP?Yc;@Cq<`>OEL zwrN@iATx{k`&q}}Uk(R@>-Dsr2?fcMYz^HPtVGYgj9tLpY*eObKZ{1*6?8&qIA54{ z*{r4vbCz+P(0}(1(MPi!u3Fj$WNFz0lf2=2?~q2B-ZEce4&N|$RcoZ@E2|c=Mp@)P z?^}y~h2hpPeoJ#}y04txL7QeKCyldx$QS1*^|e?00QE21OC<*Ar3dEHj48fzVzn)Y z^C4;A>BtRRT{A3@pH)X4p;(SJFW(CE<5tV>%IwH%%s?H&GOp9%XUV@tQJz_U4c@u+ z?7btW*3sCR{DJSX1+a;C;4ZRkYkRspFeM)oS5y5;CT|iS@RcCk*3PqPeg46F!h3_iQvP$jb8RqRr9JV2v%wafskvG!wY@LZHe7R9Kbte@ zKz`6zZ(Br$nu{!_Z9CU}5g}fCB{MeA!8bSEsdULVUYnfK6LVTOI^DLCX;y9JGp}nj zpjFjX$|`Z+ybGLYUeu>L>Ua~@#dLa#>X!s_WOr<$EA=)a9C_5gD5>PE)mG$l{Hav& z&H?XIa$uu9Hd8$wl2#i05UGJG%E6Fc+I-!Xwwu>=jKQiEmB{R`_*b(!Kc$xwozxuU z9lI2gES4txYkbQ3P8sl}skcPLuuK<7Ay2QgeWt_Z4#}ylwP5v9n}5B`zGQD0elERB z@(B^I7SR(DBh~f=XN9^`ZhDgV3I9m{IeV6zd%`w*c7HpOa$b3Ae`51HzR;h%Zg`%9 zNjdGTM42kD-Kw=K*|PE|{mCOob(f1b_ub4m>Rm=Uh1PMGU>B_E8ErBS@aC=y+8|m_ zsV8oFXQbUQ4o0Y%Z9 zou!3!Q^t6TXY@9XsV&{%_98}x&!$K6T1pewpY%q?8hRwGhHr7ApH0dTEVcp5C-f4Kr2Nuf!Zoh$SJRW$nAQr>-#n(}WPCPKe!+pF9C(vVz#84HG93 z1emD(N@`mP$ekWcyyOg@seN@`Q40s&rA7zZJI013+2@m%=@A~?YA7ZNo}f!*J=@P%WB!e--${Bu{uFB_t7+91A+|17e}ky|k-1o3>0pGXwqg~s{AK%^@yXYfFQX$| zn;l2>r>Rr*xoo_>VrVH>thdJ30lxfUpR$ZlZ7Gqy1!9h|m6gyAv%-oU{9vPun*McG zK{iUur`A`ti<h75Y=u-W@pe$;fW* zVa3s0;2>?teW*52lpfKK_^$A};LMIxsDG@tC|%@u;CRpcMr}_&ucj7v=Cr2*)q2Iu zW%VO%l{~h?#*gXGmEz$|*-3K)@~AJFQd!5xxN?&6M8BqLYLYj#Dk2ro&%yx zXaz?%{?v0J?W@`Y~QuYdDFuc^iz+N;)6Z+?r2jaQAxNE$zEWL&Ci!zWeFmAG*?)CO>zC+1J{AWpJ z+e=3-dr2$9zXmKVku1qR)4oF#(NnCfzHh!#S}kWT*D%}9=Gv4IX>R}TbW-+_`PUoc zJ`G6Y<^WpI#EYDq`H*fP~u0YZy(eYxKRy zs-4oRkgj@&zJgy+z4k<<3F~C?v=IL$+W}{F7Ht;64(JD~GgzZukss|vwOu4P9cLBx z7Bf$3W7X=&f;bCK^2z#by^)qnJuWEf^SlEaM^<&5`ofAgpOC`ZPOLfC>qA8a1mk~K zitwaBH~8yaolUbBVlgS*%xua~)Eu8{ucbxtzx00mg#w%tvK|Vujkwk-SmjgZEb>iz zsruNjRv84DCy}3B{oLtR0pIU|4Bn46wO4fyQuCQN{3rBh{>xT9XF_N{*4hk3g}W!b zpf$|gD*gjgZ%cL+tMw7SI{M#^n{Lf`;+=0EGTxKN?6h)^)fL0^6Z#M0sI`|52I^w8 z)>s>7wn(pKIvnTihqb-RS~5)6GA5ZP?5ADBoJYtq@4&R`)=T?K`yF)xi!|;2X;u^& zt4u?M>7J}T`N1me8?2|Z0nXdn5NnVBk*}NiCvD@%X1_yv@do~i{+)D-?Hq7+ca^M0 z_rPvs(w49WYqcFotQbF}kJKmeN!k$GUNRBvJE)~7EHLeE&>EQ;iRHZTWSl+Ji^%NW z%1>Jz%%S`hVjC4u4V{sF+z@T)R5FOpM4o0bYqZq_v9%@ij?$dWK~?JW#zL#P(!sGp z%d7ljdOgSW<4OzrHf1}n2p*Cq$ZBiCN+U{Fi*+FDjM=)I*P$KN1-2Q=Xlsj6)7l`0 z0CD(G8A#U~c5@DS$mXk)l?$RWs@$EoN`tfaxYmqa=G(=5>z;9zABD|{RIiB6=5YNK ze@bs6wp@jEpy8sJRUI|*&hThut#(#C0sA{5hu|$cpgtg{fmd3AOz5A~ENVI;!s|#) z{*zgXzY$&8G$4l~&8r@#c|faW%g#>nqGndTi8)=Qu)ov=j3eXnS1_vdwh)I_j%&ZE zE4c#3w+Jx4998SG#=O7zfgb`cau*Q(3i*SVMLe*%)!SGv_G{&ADYS!`;kO%ZniG{( zBj_of3XUKRd9$yOL0*>E1Zrdqc-r#u4`iw`Tb;ss8I`bZYDV9K@hLO8WrcHHtOU|z zJgEU(dkO5gmla0{JV^Rh2;Ru51+JHEG@II&O+aOcUdVJ!hv#|&`JY3870yg~>%DO53qAgihIuRJPjR?F$nxdtqRsqz6v?s z4Z-5k3;FRCd4_ll#M?IB5OwPtie12?^c9~G-^>jJ;XC9nnm{TP;7fs({e|xYM&T5{ zW8Jk{Sqsc!Ru5hP=*L>TkM)z;&HTq|kKCtjVgkQ}ESf=x-RZzIh9IuI0QFoy(y@qc z^h0&ZLDWQGd@8%CTtS^yZ~_Ad-2yd`qfps(md#=B4%B8CauO_fs{@g3a0k^MCjjA_ zuI;xi&_*fG=qModegF^MSk{gmCl$fgFbH>%gP#LJuCO)UynsritAL1`ZPYVHp@wZo zy%(@>CG^9Ah2VQzjf%H_2b`#2_&U%`PXkZdSO2eIQ2*-x!Qa}m1sKGJ8ArVLJin#i zO{tmeNLi7V$#+ISV%|2t83Cg?uSNvxp-#3nah7mjaVNXqgj^1}7dkO)V_56(^zgh9 zd&94XrG%=Xe)kynJok9_LDxG+A^R+C6h_f*R+~0K24pgS3K_)%Sys_>19R4gcgOA- zL7w1MM4f>?VZ~AZq^?#(Ev6JiP2YUl2HRMB8T)?iDe%BvAX1+r%U~rbAlh-=OgG-@ zO#`YguV-<3YTDzW zC#_ks5q~G~VHypThjqE3wxr3>=i2m*-5FXA4Dq|e&W9fb-gwe?=Ir%0>po3M_0#FAW*YA|8?$khS;0fE~t^pHDupB zq2qz*nultut-;=wRqu_8q6blJr?R)NC)#t~^U9OmGuYe3Gs?HgTiJKh)5^Ee``ViT z#QyvAgK3@8=O#Z&G~?UHlf=Ih)8p61=SrIIdC2c;P1tLwQUuhGu;tk5urI!sh!w!@zH8weL^M$Fo%GF4-JSMPujT5K zqf4IOvqy$ZM7_q4h)bD1MGnn;BkJeeN26AUb+OO0odK(~VlM~};4a&y1c={yst0@x zJT*{bvqYf0`3C6fS^C_-#XzW0$Xvy*vCZ~g&hgFx+9PF@cHN%ph;@u~4t7lrxf62Q z)y%Hh`f01d-Wj2-QS;e`+1jXUz$#Q7?0uiXfKZg3WY3g1ii}Y!VBWUkfUdr5jbWX! z*Ye3YuU{~mS#7LAW>!Smbza;orVlbI0h3z7B6AEb)I*uvE|Wv*v{J$Y^$|$>UK63943u~O?<=3buo6N=5u0J0G>OW zIn^2tG&tp#5nJ7?>w$5`Tk{=w3uEDpWd<&J8S0jAMXxRg8{KwmFzS_80z&+yfmtu` zyD!F5&XdcN-=ldNryodF!1g>mxl~HKj6wQNV?o3N*0mqJp(puWd;#a7iuF<;NO-PL+1edu|h z23BJKc^7c3^;s3{OAZp-xsUGN)PUUUx0k9Te( z$6zejkCu~uU{o7K%7Jy}AE3c6i+=ouCAnU6^GK_-H4s`?7Tn77EWg>Er}Fk<4)%z* zV25ulGN=Y1PkJ(T0#}2NXSMavENxaXZt0`+Lq-W>p!piib4vq1`19#^z@b_MJEm6P zR^V-53_QoPV7@(Ws>lXP0_*;DpyIm#wYD9*z|(j(WCA!aGRFeBK81}(%VZC4!9HL! z?2sLRH`5Zi=CkNm`i*U77h${FVQ;e^s|c+7Ds0{FU@O^W)&fj=W5Aj>5UB4I_L-%F zA-NLjlt(i~i2(EKF}4F&Jb`mQQ_0iU4osUb#0GE#4Ir`bH7^Ng z5ZAO8xZ<6_T_Cd#JVepJ4G%^h>sYXvU@sDTqyLJm;M5v{Yy&S(F zZQxNq!Dlxi@%cdaJOiG>1d?tYaKih5IbDst_87c-5Z@UG-{vj$@Ck4rx3R0A0Bk`W z$tDQ|?kphwyW@_xgD=trgm(lu#Tez_9VrVIbDGkB-#v1j~+TtkkG z{g44@uv;VSzexnsEAT_6$m;eI4);TG=Yn~<0e zcMxNg8(6i{K>2k+7D0R9DXQYp7{AjVxW8t2RSDR?96)|ZM7Z zNW6{2%7J+Zk6XB-J9tP0QvyCo58`4%(B2Zr`ECS>wn2Zk#$M^qXnRk{wObH<-3a5b zI`DiF&z%F96DiRoT=fyIcmvvZ9aoij@YA@%lh8?tzkV2$hD1zBTn}n@;f}N*Vn!nT zB+AW&D_KDdN_r6G@&)JJ2+B#KLa*X_62tijl9EWZL}&|z#2q**587DY(K{ zfO(Rb=~B?`+~~usI9%v+*%uNG`4U%>SYvsd4<3*3$s@cUji{kS-%C`tALt;7>Xm4` zytq<6;87$#nBg~Vzz`+l9%NriJ(6g7i7`Eco;v_elH<^-13-H3#Q$f4D3{}SzKUOe zh4xD%*;gU)uv(GL~GI)XZc@DYVL3**g+9Mxn)0lO-ZEfLBtpB%V!L0oi(q zdX`$o@vK3^B=T0~927(wC8DxY@Q|abCMD9NC{c;@OT%4#$G1tnmPl5A@Q$Q3Wm}~jV$t%qkbp$n zO01xiuS6HhPo#Y06(!DG;`n7QgAbn@u-noOOJtwCvPA8pvJ+Y_rSd&jzzXV+M6pY3 zs>Dx9^qfR#JMfSg@NCe9T97^#Gr%{gRPRiE;C3ZJ~9FC$D_?sauSCsH7gEn zmWX4CLw=9 zDR_N}j+Eb#sAh?El-3i)r647#a}rz1F&D@j6p3h*_V0Ud%J(JiRcfsFe}~i?nKvOZ z@@aTZ#OpLjNurOX1ivFjrJOVHzWhvnI~A?>WS_erYuQSP9}UHK-8jnt{@jM|%g+@& zJ8@2UuwTB96PZ6GuOssU{TSi$n-cjeano`{I$%9Y0!^F=961?i=L_7OiEGNmjpk2kw?) z+&#q1p~IF=#@L;QY``?E#TtR}X%+voe_~S&PoC|$0hAI#;w(XwcbL;C}p<&5Ng=G0bg|k z{#b+}qcK7OHVVusx3JE<#=BUWsK`f|Z>^!$H*kYj2CL3QWdT?R{|4fAzQ&l(_{ay zvh#qm>FWCbIp^M)J53p)MIRA8gdn1h8U)cNLX;o~qt}=R!)PIDFoP(;C<$UPO7va^ z5k!w*^fH)Cx%ZU+cir>e&yeSNKL7Wf^O@UD+h^^y*Is3>-?E13s=3^3u3S@lfwfx0 zxapY!J}l@qAbw5I4|@g~i$Pn7H(Q`1Ct5GTgZ5}m{ab^mokVW<3NYGM83#0q4!d22D2;4R~UJ3E+B`IS4YUn~wPRd*vhc1*k(rt|=XBcD0bED5T1 z2FOJbpq-|fjjV!Tn+(NrXsp&^jnpw}vP@8Ou$ENtmEKl^z>sRAE@OrqVx_X$@fWad zn_Ah-)=C$!9d}wq;j_}Kx^$qqlYg@zTrery5iX3Dh!?;7QDGbFB?ZE$+O^TEBWnjZNej`DEVDVug$5x=%kQxE8lnm1#@=hHo`7dMBiTL?BrI}{_YrIOjeZ1S5IgrIXu1cn zIX5DgW?@U#Mnh{s=5RhF&kx|Ur!)GKzyeQXb@whbw+3Y4TTp8p((G%jwsXjZIG!ZN z+y@>2c&*T}t}=ma)?=)-7>W$1nEZ;w6O4=eBp z*WZl)VmR69+p#FW;JWeDF_>jOG0K?rp+Y^@3OpdfG6M{%n`SEOtUH^D)N_P7A3Oc2>ap8-+;NP|{TIk5O`rqhIreqTK(IXn%mT&^&v)+r?)vUJ?pyAXRR8`fBTvTB^rh+V zQO9;vX7S7@YO9wdHq+W2OwHtYcQvM*09II5HO}kBjBupmCG!+@=KrFa!A`tIe}YRf z$`S48>sX|f0GVN|HV4RJ2kWyYD0fX$FRRb=gb@dOWDGN+^~WBkUf4({3Y?P}Baqe8 zg|vat5 z==T6WzyJN=8&>-Hdx_ z5f~{^X{R$Dy4!g^_w@5r@x;38>ix~F;HW>jBrOQuopf7 z(K4sGP5+W=FK&ImvB-SK_(~tM6LiPDU(k^QI8r*a@Q-UDT@ z0%&`yS$#JPM66Kfb1-D?;|E)yy+`eg_KyC(aNy`eIeM^hs*^= zH++3DM3H*n&0Oc{;#uhl)hikYjTwxYyNsq@f*VYx^BmSsA21E$EBIm#$Zo^*hMsDk zW}aPCxH$;=h31KNZ_jK*4WP^JA)d#cbk9Z4aqu#R>c#bu`V0N2KHTu54oo?twz=P! zYP9?4n@km^BKj8m3? z3YcPcGXFp>G|_T_4EMeEi#l3e$x5UN(9wwwu?)HY2po@Jbbq~)LER5y8`u<`(JTD1 zfj$PyFG0KKsOUG#@3Q|L*8POWKS`aibsCm$(fPjETuG#(_{Nw#DfSEN& zE1^~bl{nHoV-&>mo2-M?13yj0_c>m#1;)c-&`x)=0^>0?n~tj0z>jci55eK6qfJw9 zAjj%~kCAK)r|MW`R{488D?qS0lsO|~U&c@>Ms)!x?MKf-&su$jJ{07UCdOf-nDr5B zc|!Sg-i*TZ8$#5trZvO7W29Iy%$!%jDx7OwLMmyj@F+_an;%)B(9ft&4EcMi49%uO z(QVeHCm1)8cWu!cy5Yx8)rYXQCIl&18^3;WW52HJTY080(03}EEv&AVi&?5aT5~Mb zYFK+jW=nT%q&fzikrd6(Sx(ijdvX_ za8-QiSniksmQghjL06&y1~OWkdA7UHxXXCTc>+8iP>btS=84RM?r-!x)L{F}d~Wmy zhbNNxvj;dK>&*pZ<#ady1xkDqqB!}CJ9;!!_}HjMjzJ?Mo};Q-z3EHU9Av{ zvz{An{U#i+gbGZZKn`qde5O++mzdB8Mqk$V-7_juv1N^U-u#-lfP;E*b@64lLOX8aJu&qLgm+APX_YPmE9 z3-kvpm%31LuHd?Y@7A2QtkpXriQ4FGi752ZqdT1be14)!OE(`>}A2@+=pK6RkrsY%xwzRca8ejtBYWIO`>*vR35; zt2KT{!#Rzn(F!f+H@5Hb1m~rCK?1V1xLMN((T7ln_@d{${=N}l)YngX+IwO>jSQFh z5xOXN)u3Xp1Vw9s<0LB<3OIrt8yqX(p?K#vj$}2)8VN;<~c#gRL$ZYDK z>~895ssBmk9u>_ogNiY$$w@m6UDuhT@i^qg8=lLmYqlcpneO>iZ)VJ^Vy&J3h7 z(hg%PGkI$)zVgVP+FFd3sy)+=QdR7XhLmPqNk8Wmv>MHsgIYzuf?%}{Eg=bX&J^{Y zdL2B^3+giHSVf(x{>t|Xnp=IQf`qFU0`=5VOKAD9c6Ni|lT!`DTIfxTA`DDj1$!?N ze@#&$R0*_t7sx*+u$b;6;U^L!8AtYFIV#wdWj6cRs)*)yj{0P(5r%{tP5fpk7)908 zNXI?L&#q0bOa6ePhGU(h8&z-nSgp+2RuR)-BzbmZj7rzj&ZQY?%QH4*b^*ny zt*4l~vAeXVxxUd@z{pNB^Ma6bo?2EewKVk?cA}5h2Lrc;nxfWM!{Mf})c=Y$kAgN; z2CXs|GgN(SuFut6>RMuUr=<$3R?Be{oXSRyQRLXgFmH`xe-~VK2KjbQ=#f($j5XW_ zETFX@=l)Dg#zfm0$EH!s^nJAR%Xo~RlW~`b4y-6wu)ro$iKsH~LFB;2QN!a5t=P-R zXwCmtvl2B(R-$=GeZ>r`Dv?8f@Noh_HByNbP6K5ySQ|?}M3GT59*WOLe_BJ!np3BK zs@g}*uZ3v&HF5<&RPrZy^#W}q+;Kx&PNrIQkh7@=+WR1k#eKJZjev!mIb2w04{&1h$=*WV$soIuUIQl566Cz+KpBQmOI zBxQ`xh)X}0HZc8``xG$J zgN&viA1A^mqwpepYc8k0Ks43#LRb@(hA#e_*4Ob7b;vapd9PH}&Ny~DKL%s)64>l>vQP^>JyyT@z~8uy3H@+Z|0br^QpJrK*YQ?2j^wUT-P{p5k- zhZfpIy-q}>BGyT3(B?9g7v${Z!GDrT#evVT%X=ydtbEK0yNIer&~IN`;mGV6*e}EJ z;AhfPxu{F{2WZAWVWaK@Wx6e^(LN?B*ao}1EttCfk+x@uL~o{-SJKzZLE6{Jiui;0 z*f-cmi^12qPRwrvxK&A1HK?FgSCc`l?tzu}6MYoJ-TvebDSTdN1#so}$h3vsLQouTo1m)Ph20RSIur2Q@q0smtQfHg?SbM3hM(f+g3_XIiX=3>#BM%y(wSc$JTCZ8e=SR@0 zwNy|UgjXpD|9&@mumE|n4)uVtg{nr+mEvl7>~$ACo`_933}3iH1oI?Rky;nypkN}_ z?NTx<{{RmwoG5W0+Av&Qr7luq!NrPTtzQ_moZ6|~!NTf+6%<6hr@!!iJm&v6xakMz zxhwR-ed@xjf;ZZM(;WjM(njp=-f%`+wvKSgUtUD9a1g#GqH_tB^$4E-SNeD;&#)H> z91o)MT#)}ZVABkZJBb660n8Izp#4p_ zesgOFqgFH5f|1vf)s$7y<|`RtAh^BIFHzC;G^>N|>6v<%QQ!ECt)uY;RW6sJDIYan z8l{McPe4nWPsBT(C;J)A^^sWxp8px&C)FTm>p?{j)H|z7!S@T(T2ke1HZ|~mp@!UY ztiM=#J^`$|tJ)JSh=_hMM`1@O)pBoX2U*cV6(g++`gT4ojT&0>)UU7(i>W$M?Bnq(f90TfcV@n5{D6^2qKUkJi!Wp*o~!8~ zYc0pl{T7PNq;KLG*_Yrt{B^u0!EYyt+aE>aI!%A3Qga{=bhr!h`CYs%x1h11-3n zFwb4yDuM4QCzgXBIpr$Z>N)YJ}ATm*P;{kdK& z6#5I#F=+Ql%>ihMD;c8~!D)Pj@2V8zv>dPR;JYdaqW4FPwI%ow&Y(5KFba!MmuutF84plF!;Hx-}ce4%s zy~+H|ykc%g8`{Sww+WWEwHK*Y4j)v1wjt0v4zFo4IR9Oc0*BOV_(Xok%X3UkpiOaX z3(*#5;jx(mX5M(U8?$+c`V4=>GQ3`Wko}GEmPs^Ia2ph6gxm1M75dwON3J??`7X>2 zU3iyO4|VbRwcz_Co^CzsQucD4OJE&7KzhBv19FZ2UCnGY8SWTO>jv>2g(T~PhTnv! zW_4oag8Ed9sA(Z0iKY0L7^lVjB1l138FeZVusR^Gb)?VBq7_hSgSnv> zEw6boBtyV63`g2Tuytmx>xM6F0@37Op+W%Cr!`m+bCH|Jp|AsOyDsz_hrB(B?wE@e zKvCLeESy(bFm~d5jQvuMvTT*u-bEIe)D%s`Bh^)_$$FrYS}5r7`_Nv#MCUB1J_j#h z662`?-pCQuI!d9A<5RFLHMnvSGN1YeaEl8+U^%NGRdVX^eL9xjV{DiMjPM0!1oK{~ z`5t!A0-kIOb2mOOo^K#|8l$KQ@+~W;mV)CL$vTdeTr(Qp?reRIt+AJR>u_SBpcc52reCB}Q6B^uk*B z9P2WIK4T;fWh_RZadpDm+lmo29({N}_q&JHnuGq%O;ji!ecOmRa4@{G7U^(@9QgCd z_2-PS3wTk_z)c4J>OfXk6`)d~s=i?M}wTg`yBNf5F9409yy%=ckU;0 za+X!zPq4ysBWGGbg??OlEPXH^PtZ!_>_uW%5}_)KPq`y=Uk_&BQEj_ls zK1S8zyVStd@R>9=B8?TsX}ou3nfs)U=@O*mA-qn1vu00p*HzwG%LTs`M%Dyl*%#n$ zHIZMD=z?3BWws-g*Fx>J$oA84R%xv20q9gUkm)Ac%50ukkcX&WOFT6d|C7aQ5;OGo z@N|F1(Rlp%G3Yk?kjg*N2Ls^xDva3>D?g*Q1F@xVnW5Iumet;MRxyk`G6R`cuOsmk zBA+2}Kq&9|k%^@laqSrs(a7}WNS$5cSD{`aS*En;m2PBr$)k;oPar(X-+68EAPi%*?U$WLHMF&UoBL&(EdjH`AVj zXn9l7nC&I*GzwYX9XuM`ql5)Ie=Ogj!^KRzXD^e#QrkU6P0 zbH^mOY%|aQJK8F>wHUwE8SkAK>HQe-eL;a8tqxKLVx5116llfh??iT1xLO%q(ZRaG z`^qyU#%=W9n?!g{L5pQjVGQw**3em2HVG=Mf;Az!&t~|3GGp`;G=j49ULG_C1bE7|7J!;6?%me8UY`g%KK`#hTY12m%y+I<6llwY1B0RpkU3)060c&cQc z<`Mg!FdyCH(<_b^iFp+c*+q{}$42T22leo-qLbI&XirVy`m)G$S-Dgl{_g+>3`Xyc z^y*eU&~HQvwq&2IMas=8%5%_T60$qQ+RW$*Fi%oheWo!A9eX;fwI^d8tHwTki9Ssq zi$`o4{U!329K#pCb=KKL;JvHcKj2U=iW1#m! zP^yoy|2+Fxan4%*g5Ww=;Ox#=lM|WY*P{nqg$Ey^8;0OnYKSzCM#3Cnbf=)dx@eQ2 zA=XB&^uh|Ck5oBFOd>auqZTt}l=>t7luK%+S_Y{cN&IZIHdUKJ{9=Q)mZ|}3&~FZF zr|_Na)lO&+Gy^N=miD{0hdmRt)_BvhBg_4?oZzSL!{4!pOr0jg+-1C$hi8kR(FgH= z7kc$a4nM*6PNaR)sJ%E6eX9ne>=COeuHhM*&n(r9*>*C%-G%rlJK{YpK!ohPvB%hj zU4EI&noDSSe$2WdQ12SsWy4MW+;g-%i1}&2CorwzQCtMG>2ex zi2T?s+0iPt;&VBUjxhvZ(@A*r6~67fS|u#R=2|;#CNZ)f@JN={iedlt!AAU*m|+(x zZgnIMbDFi@GuU$y+9#;{siAWpDyH)(P92F%n@`U)!25Pxsg4#e)n+?E)5CCTW31-J z$X5Ik#0s{+G4qJ8lsA6T2k4W@B(24I?rugOs_o2&yE2ULjb(7s7seVRidF7M(TjH* zXP7(N@Ova@cEtxgj8R$&nlEPtjz`*UMN6K}_f15n6*RdhBwIYO{mN*|3ieVj_&E%# zavK#E7cefyAUjGjdb+^*`SFQfVExcWcz+XC)K%t}xy+7fRJsemmuO<|J-{XkRnIXR zUC8n|WCip_YJUobesg5}-}q*>siry<`b~!aVzdTm&bhU((6nnhK6E^Xa$gZeSO$$- zXm4wmkp*2j?g~_GNv`BBd?6c=6_fFe?}uWi@iZ-H5se4(^nK zO>-ODCKb#7ETd{0TIf?W(9*PH3KIPh7S=iBLI@h%OmzDk%2f2l7qq`7cMXQeA1d#` z**)?67F16lH6zuTAlHwkvO}0Ap7Jm%l&zrU-!Z3-Q7@};tRN_@jZK>#^__v994by5>M6Ye6D#w|KIq zXvgm}Bke*?-=!|W7`#4zGE)bzGG(=yZ1zJ(3AWY|)rf`rS25NNB*ASWVLi|^Dq1(N ziAK_E1C2N&K#=(xqof=$xB5ubKe4QJq85GO?)LC?ICpGeo@F$wBHk1aSLmTsY++~Z!+G@+UhX% zsoDxjdX-T%iij&U~^D={fFcH+`lL^JAy2$XWd1<6vXCvfe*YS?JB{TJc~A0 z2Ay*x()9zp@n@M8)?@3;hZc*GHqH)uQkZjFyfHcp;rTBv&+gLmi-C6lZgMK9eV6|XqQGW{ebVN zDg9g%3QGQuWYwHOLQZ1l>cQ&HgVgzKL62{MU!JgXD+BLqC8~e!U`7zc(5tK-E6g0W z5NW&`@74~spP_9z)^a-deI}mHfsC(y_&ukQr}7@xcX7XqtR`%WE|p*11RbifHKq-f z(F4oSzP|MB04TK^>wFHfcObfiKjZlbJvjm0e+Ye=1JuZ~_#2({dn!FM3(cr5PfLAn zD9{ayMB)fvz>7c9*6GX#gQ4GjtkLIKk*P#KJ23NVtOgf<`+W8{#Ii4nt<@9G?t&#M zTv!|1CW2OMf-)npH40%_+@Kvx;j?^5jmON2JD_7Uz0!}ip2S0O7Oz|Yy6{k*p%**-z2>S0-N zeF@6;Cwkivt9Af({Ve9h)A-6Su;TazJ+c5DW()d!K72KiaL6oVNlRK13jc3_YLA(( zqzXbQs1OYIorTJ=jDr;GbJ}?cYdj7cp#~IE&_TANADqJauR_1Izz!jClZ zhV({nuNPn>y47qvEl*SSH&^`xOJy8<*%up5P^}BG zj=(`IMXCWv2HII@zSry1`G`57GA$Qmd0BC}3+Xo!%Xb^Rq#$YXqZd>}o2lpZ307c~ zSH#cM5r5NHctc9SV>h7WQ`$cZYPIFu4c-{fX?zIh@Q{@BdXpuyp%9WI3^^g619%3h zjv>`L_M#guN8{QA)l;F|Iq&MT0K7ADykx-$K1pSGtqkAt!*7E8E@;DV^Gpxm$lIKC z+nd*O0*buy<{RXM_L3hb9GVyY6u+h)er3sQk*o~KdeRuL7JVY9>5^X{>-4j6<#(Bz znxTum1y@UEg~3^}dR|b^4Y*E)PN_&rli!|jm6F`45LcHhJi(_Hzq#OhyS*}pY;8u) z?>yak`bXipvarJK+6K~w`}9p9ZKYN{@$W12PXaPXGL;TN5f3Y7#aC6y8>79*xH$$l z%If#O(cTJS%~zpcigI^5r$%yNB&$LE_kVKl6P$4nFZNcrWIZ$TN!lX$Xb*XcTu@c; z@x|kxn{!^#mRq#oBFD%k)j_0()4lv4xqjk7lNtk(O<`lJOD3LVaY*))jpQvEDmkHi zL7qu^T`FTp=1LA)_=q-1g$}7SU?DN&ibc7iV0ufwjAR1@@GADI2ii!6iDX(xhJ*!` zB&#l%UNHEUiDXZQJ|_QmE{t6RM4myaEIjnKTQU+PKSwqf_mD~(g0C*f)`EsE*+P=% zBiQPanIkyal2amCXOdT`A%`VX$ga&HSralgWL1-cca65lcl!*fhGP2|3vmT`B9&Kp zmp{~R?jRXNlHnnFMDnX-P#Ely%m_JN&JeofWwZ;9xxtYyyxAD9_+)3INdAtb zT$+Q{=7;{WNv@FO!bt9gv`SXW%f38(mTWp3-Cz1ldQ8ym1@~X_uyS*Z;N43WTM^C) z<>>|EU9uMXFw!MIMrbMdGEyZ#@>N8JODiN>MslBI zlOv=mh#V(*B9fQmW|Kd;n>?ZPlWcNV8MAm8O~vf1zQ zt9@mgvi8w(yzfu;3w`b9_WkO6xB1gqSiAIYD6KRee)_*9M)PWJVhe6st~zOtMn z{V7i-82qvcciYb?DEQJMyRG)U?cSHGN`L(GPwsEiK%O(}&%T0vE&0{gM_JFCb*z0q zn~tyVmve-IHf3bL-8-)zf{w357;NA^R8m zRerJQD!=>o$!}Rl$?rCG?X%@5o3b{AWo%|`QV&GF*?l1Il7A`LZPGWsk>oo@ezC7& z-_dS^+`;Bm`;O8|-_y#we7D=0^%Sq~w?7MgZJOG>=^IDF*Ftl76@K<{w2!BK`)z5E zwO7u`I!2E2^`P{UuaA73Yv0{&!J9p6Kf%AZSDwm#p4a#Je*I5RDpZmEZ~nc0w(qmu zf5J0ztnb&XvweH)E6MM6f7m_YyN~>4k33&%vUj8E zf8O=~_g$_llzsEK|8zGWo!>mxcjTMzK05fWDtrIgf3KgB#mn-moaNi=yF%8x?})5l c{_mf%X#amdH|vc59p$CH_OoZ5`QJbNKf%7=hyVZp literal 0 HcmV?d00001 From a0cd474b62e90871067b847e4b67bf7375ef83ce Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 13:33:35 +0000 Subject: [PATCH 13/67] Formatting --- .../sap/ai/sdk/core/RealtimeParamVoice.java | 6 +- .../foundationmodels/openai/OpenAiClient.java | 1 - .../openai/OpenAiRealtimeClient.java | 186 +++++++++--------- .../openai/TextToSpeechRealtimeClient.java | 8 +- .../openai/ToAudioRealtimeClient.java | 35 ++-- .../openai/WSSOpenAiRealtimeClient.java | 1 - .../ai/sdk/app/services/OpenAiService.java | 10 +- .../sdk/app/controllers/RealtimeApiTest.java | 18 +- 8 files changed, 131 insertions(+), 134 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 10fc818a8..3746a2f3f 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -6,10 +6,8 @@ public final class RealtimeParamVoice implements RealtimeParam { - public static final RealtimeParamVoice DEFAULT_1 = - new RealtimeParamVoice("DEFAULT_1"); - public static final RealtimeParamVoice DEFAULT_2 = - new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); + public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_1"); private final String voice; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 3402e39d8..adb1df725 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -33,7 +33,6 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.HttpDestination; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index f1923347e..4ccbbd12c 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -4,110 +4,110 @@ import com.sap.ai.sdk.core.RealtimeParam; import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; - -import javax.annotation.Nonnull; import java.util.HashMap; +import javax.annotation.Nonnull; public class OpenAiRealtimeClient { - private static final int PATH_BUFFER_SIZE = - 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - - private final Destination destination; + private static final int PATH_BUFFER_SIZE = + 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - OpenAiRealtimeClient(Destination destination) { - this.destination = destination; - } + private final Destination destination; - /** - * Creates realtime channel allowing to input text and voice it (receive audio output) - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-     * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
-     *       textInputChannel.sendText("...");
-     *       ....
-     * }
-     * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @return input channel, allowing for text input - */ - @Nonnull - @Beta - public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); + OpenAiRealtimeClient(Destination destination) { + this.destination = destination; + } - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + /** + * Creates realtime channel allowing to input text and voice it (receive audio output) + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @return input channel, allowing for text input + */ + @Nonnull + @Beta + public TextInputChannel textToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); } + var endpoint = getRealtimeEndpoint(); + return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } - /** - * Creates realtime channel allowing for audio conversation with a model - * - *

    The input channel should be used with a try-with-resources block to ensure that the - * underlying connection is closed. - * - *

    Example: - * - * 

    {@code
-     * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
-     *       audioInputChannel.inputAudio(audioBytesData);
-     *       ....
-     * }
-     * }
    - * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. - * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @param params - optional configuration params - * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) - */ - @Nonnull - @Beta - public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - var endpoint = getRealtimeEndpoint(); - - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + /** + * Creates realtime channel allowing for audio conversation with a model + * + *

    The input channel should be used with a try-with-resources block to ensure that the + * underlying connection is closed. + * + *

    Example: + * + * 

    {@code
+   * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+   *       audioInputChannel.inputAudio(audioBytesData);
+   *       ....
+   * }
+   * }
    + * + * This API implements full duplex (input + output) communication channels. Application should + * logically synchronize their state and close input channel when it is appropriate (e.g. last + * part of the response has been received via output channel and application does not need to send + * any other input). When input channel is closed, output channel will be closed automatically and + * output consumer will not be called anymore. + * + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @param params - optional configuration params + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 + * bit) + */ + @Nonnull + @Beta + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + var extraHeaders = destination.asHttp().getHeaders(); + var headers = new HashMap(extraHeaders.size() + 1); + for (Header header : extraHeaders) { + headers.put(header.getName(), header.getValue()); } + var endpoint = getRealtimeEndpoint(); - private String getRealtimeEndpoint() { - var sb = new StringBuilder(PATH_BUFFER_SIZE); - sb.append("wss://"); - var pathParts = destination.asHttp().getUri().toString().split("//"); - if (pathParts.length != 2) { - throw new IllegalArgumentException( - "Invalid destination URI: " + destination.asHttp().getUri()); - } - sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); - return sb.toString(); - } + return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + } + private String getRealtimeEndpoint() { + var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + var pathParts = destination.asHttp().getUri().toString().split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException( + "Invalid destination URI: " + destination.asHttp().getUri()); + } + sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); + sb.append("v1/realtime"); + return sb.toString(); + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 4ad9f6918..d8f12aa66 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -21,8 +21,8 @@ class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements Text List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = - "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + - "do not answer questions, just read them"; + "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + + "do not answer questions, just read them"; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; @@ -45,7 +45,9 @@ public TextToSpeechRealtimeClient( } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; } else { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.of(param.getValueAsString().toLowerCase()); + voice = + RealtimeAudioConfigOutput.Voice.UnionMember1.of( + param.getValueAsString().toLowerCase()); } } case TURN_DETECTION -> { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 785414a0f..84038bcea 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -2,32 +2,29 @@ import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.SessionUpdateEvent; - import java.util.Map; import java.util.Set; class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String DEFAULT_SYSTEM_PROMPT = ""; - - public ToAudioRealtimeClient(String url, Map httpHeaders) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - } + private static final Set HANDLED_RESPONSE_TYPES = + Set.of("response.output_audio.delta", "response.output_audio.done"); + private static final String DEFAULT_SYSTEM_PROMPT = ""; - @Override - protected String getSystemPrompt() { - return ""; - } + public ToAudioRealtimeClient(String url, Map httpHeaders) { + super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + } - @Override - protected void onResponse(String eventType, JsonNode event) { + @Override + protected String getSystemPrompt() { + return ""; + } - } + @Override + protected void onResponse(String eventType, JsonNode event) {} - @Override - protected SessionUpdateEvent sessionConfiguration() { - return null; - } + @Override + protected SessionUpdateEvent sessionConfiguration() { + return null; + } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java index 213725452..a3f9383e1 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -14,7 +14,6 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; - import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index ad6c92c2a..af6554451 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -5,6 +5,8 @@ import static com.sap.ai.sdk.foundationmodels.openai.OpenAiModel.TEXT_EMBEDDING_3_SMALL; import com.sap.ai.sdk.core.AiCoreService; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionDelta; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionRequest; import com.sap.ai.sdk.foundationmodels.openai.OpenAiChatCompletionResponse; @@ -14,8 +16,6 @@ import com.sap.ai.sdk.foundationmodels.openai.OpenAiImageItem; import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; -import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.ArrayList; import java.util.List; @@ -127,8 +127,10 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp * any other input). When input channel is closed, output channel will be closed automatically and * output consumer will not be called anymore. * - * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 bit depth - * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 bit) + * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 + * bit depth + * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 + * bit) */ public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 8b73f7c77..177d8d208 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -11,7 +11,6 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -28,7 +27,7 @@ public class RealtimeApiTest { @BeforeAll public static void setUp() { - try (var fis = new FileInputStream( "src/test/resources/fixtures/question.pcm")) { + try (var fis = new FileInputStream("src/test/resources/fixtures/question.pcm")) { QUESTION_FIXTURE_PCM = fis.readAllBytes(); } catch (IOException e) { fail(e.getMessage()); @@ -104,14 +103,15 @@ void testSpeechToSpeech() { } catch (IOException e) { throw new RuntimeException(e); } - } - private record AudioMetrics(double rms, double entropy){}; + private record AudioMetrics(double rms, double entropy) {} + ; /** * Audio quality metrics for signed 16-bit little-endian PCM, computed on the mean-subtracted * (DC-offset-removed) signal so they reflect only the varying, audible part of the audio. + * * @param pcm - The raw PCM audio buffer. * @return The RMS amplitude and Shannon entropy (bits) of the zero-mean samples. */ @@ -120,21 +120,21 @@ AudioMetrics pcm16AudioMetrics(byte[] pcm) { return new AudioMetrics(0, 0); } var sum = 0L; - for (var i = 0; i < pcm.length; i+=2) { - sum += (pcm[i]) | ((pcm[i+1]) << 8); + for (var i = 0; i < pcm.length; i += 2) { + sum += (pcm[i]) | ((pcm[i + 1]) << 8); } var mean = sum / (pcm.length / 2); var sumOfSquares = 0d; var counts = new char[Character.MAX_VALUE]; - for (var i = 0; i < pcm.length; i+= 2) { - var residual = ((pcm[i]) | ((pcm[i+1]) << 8)) - mean; + for (var i = 0; i < pcm.length; i += 2) { + var residual = ((pcm[i]) | ((pcm[i + 1]) << 8)) - mean; sumOfSquares += residual * residual; var key = (int) (residual + Short.MAX_VALUE + 1); counts[key]++; } - var rms = Math.sqrt(sumOfSquares / (double) ( pcm.length / 2 )); + var rms = Math.sqrt(sumOfSquares / (double) (pcm.length / 2)); var entropy = 0d; for (var i = 0; i < counts.length; i++) { var p = counts[i] / counts.length; From 8dc8ae60a9db58993acee429f82c87933bb0bccf Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:40:21 +0200 Subject: [PATCH 14/67] remove duplicated dependency --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index b6aaf7cca..ec3bb4f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -280,11 +280,6 @@ openai-java-core ${openai-java.version} - - com.openai - openai-java - ${openai-java.version} - From 42409100a5598e2325c36ebc1f10c423d8ef43be Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:50:20 +0200 Subject: [PATCH 15/67] improve code style --- .../openai/WSSOpenAiRealtimeClient.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java index a3f9383e1..950ce5b57 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java @@ -5,15 +5,23 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.realtime.*; +import com.openai.models.realtime.RealtimeConversationItemSystemMessage; import com.sap.ai.sdk.core.common.ClientException; +import com.openai.models.realtime.ResponseCreateEvent; +import com.openai.models.realtime.SessionUpdateEvent; +import com.openai.models.realtime.ConversationItemCreateEvent; +import com.openai.models.realtime.ConversationItem; + import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; -import java.util.*; +import java.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.Map; +import java.util.Set; +import java.util.Timer; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 0603218aae981e331b2f307d868bbaeb96373ed0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 15:50:59 +0200 Subject: [PATCH 16/67] improve code style --- .../openai/SpeechToSpeechRealtimeClient.java | 2 +- .../openai/SpeechToTextRealtimeClient.java | 2 +- .../openai/TextToSpeechRealtimeClient.java | 2 +- .../openai/ToAudioRealtimeClient.java | 2 +- ...lient.java => WSOpenAiRealtimeClient.java} | 4 +- .../openai/WSSOpenAIRealtimeClient.java | 187 ------------------ 6 files changed, 6 insertions(+), 193 deletions(-) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{WSSOpenAiRealtimeClient.java => WSOpenAiRealtimeClient.java} (98%) delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 59fab1a02..3f3085e9d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -10,7 +10,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java index 377e43625..e8ed82610 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java @@ -7,7 +7,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class SpeechToTextRealtimeClient extends WSSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToTextRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; private static final Set HANDLED_RESPONSE_TYPES = diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d8f12aa66..5734e284d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -10,7 +10,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSSOpenAiRealtimeClient implements TextInputChannel { +class TextToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements TextInputChannel { private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 84038bcea..7033cfec7 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -5,7 +5,7 @@ import java.util.Map; import java.util.Set; -class ToAudioRealtimeClient extends WSSOpenAiRealtimeClient { +class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java similarity index 98% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index 950ce5b57..db13f5030 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -28,7 +28,7 @@ @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) -abstract class WSSOpenAiRealtimeClient implements AutoCloseable { +abstract class WSOpenAiRealtimeClient implements AutoCloseable { private static final int SUCCESS_FINISH_WSS_CODE = 1000; private static final ObjectMapper JACKSON = @@ -56,7 +56,7 @@ abstract class WSSOpenAiRealtimeClient implements AutoCloseable { private final Timer heartbeatTimer; private final Set handleMessageTypes; - public WSSOpenAiRealtimeClient( + public WSOpenAiRealtimeClient( String url, Map httpHeaders, Set handleMessageTypes) { this.client = HttpClient.newHttpClient(); var wsBuilder = this.client.newWebSocketBuilder(); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java deleted file mode 100644 index a78f66d83..000000000 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSSOpenAIRealtimeClient.java +++ /dev/null @@ -1,187 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.openai.models.realtime.*; -import com.sap.ai.sdk.core.common.ClientException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.WebSocket; -import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) -abstract class WSSOpenAIRealtimeClient implements AutoCloseable { - - private static final int SUCCESS_FINISH_WSS_CODE = 1000; - private static final ObjectMapper JACKSON = - new ObjectMapper().setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); - - /** - * WebSocket keeps TCP connection to the server. If connection idles, depending on the provider in - * cloud environments, cloud provider sends hard TCP RST (reset) after 60-300 seconds of - * inactivity, this is normally not configurable and would leave realtime conversation unusable on - * a pause. - * - *

    In mobile networks this period is even shorter and typical CGNAT session can only idle for - * 10-15 seconds before it gets terminated - * - *

    In realtime client we rely on explicit lifetime management (create/close), heartbeat - * mechanism is required to prevent unintended closing of connections while conversation is still - * expected to continue - */ - private static final int HEARTBEAT_INTERVAL_MILLIS = 4500; - - private static final String HEARTBEAT_TIMER_NAME = "wss_realtime_heartbeat"; - - private final HttpClient client; - private final CompletableFuture ws; - private final Timer heartbeatTimer; - private final Set handleMessageTypes; - - public WSSOpenAIRealtimeClient( - String url, Map httpHeaders, Set handleMessageTypes) { - this.client = HttpClient.newHttpClient(); - var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { - wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); - } - this.ws = - wsBuilder.buildAsync( - URI.create(url), new BufferedWebSocketListener(this::onSocketOpen, this::onText)); - this.handleMessageTypes = handleMessageTypes; - this.heartbeatTimer = new Timer(HEARTBEAT_TIMER_NAME, true); - } - - public void askForResponse() { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); - } - ws.request(1); - } - } - - @Override - public void close() { - WebSocket ws = this.ws.join(); - synchronized (this) { - heartbeatTimer.cancel(); - try { - ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { - log.error("Error while closing WebSocket", e); - } - // this.client.close(); // exists only since java 21 - } - } - - protected abstract String getSystemPrompt(); - - protected abstract void onResponse(String eventType, JsonNode event); - - protected abstract SessionUpdateEvent sessionConfiguration(); - - protected void sendMessage(Object message) { - WebSocket ws = this.ws.join(); - synchronized (this) { - try { - ws.sendText(JACKSON.writeValueAsString(message), true); - ws.request(1); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - } - } - - private synchronized void onSocketOpen(WebSocket ws) { - configureSession(ws); - configureConversation(ws); - scheduleHeartbeat(ws); - } - - private void onText(WebSocket webSocket, CharSequence data) { - final JsonNode event; - try { - event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { - throw new ClientException("Error parsing JSON response from speech API", e); - } - var eventType = event.get("type").asText(); - if (handleMessageTypes.contains(eventType)) { - onResponse(eventType, event); - } else { - log.trace("Unhandled event type: {}", eventType); - } - - webSocket.request(1); - } - - private synchronized void sendPing(WebSocket ws) { - if (ws.isInputClosed()) { - return; - } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); - ws.request(1); - } - - private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = sessionConfiguration(); - try { - ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize session request", e); - } - - ws.request(1); - } - - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - if (systemPrompt == null || systemPrompt.isEmpty()) { - return; - } - var systemConversationItem = - ConversationItemCreateEvent.builder() - .item( - ConversationItem.ofRealtimeConversationItemSystemMessage( - RealtimeConversationItemSystemMessage.builder() - .addContent( - RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) - .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) - .build()) - .build())) - .build(); - - try { - var json = JACKSON.writeValueAsString(systemConversationItem); - ws.sendText(json, true); - } catch (JsonProcessingException e) { - throw new ClientException("Failed to serialize message", e); - } - ws.request(1); - } - - private void scheduleHeartbeat(WebSocket ws) { - TimerTask task = - new TimerTask() { - @Override - public void run() { - sendPing(ws); - } - }; - heartbeatTimer.scheduleAtFixedRate(task, 0, HEARTBEAT_INTERVAL_MILLIS); - } -} From fcebe1e6276a768267de2aa4f2d6850fc4142cf7 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 13:51:44 +0000 Subject: [PATCH 17/67] Formatting --- .../openai/WSOpenAiRealtimeClient.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index db13f5030..90cf63bbb 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -5,23 +5,22 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.ConversationItem; +import com.openai.models.realtime.ConversationItemCreateEvent; import com.openai.models.realtime.RealtimeConversationItemSystemMessage; -import com.sap.ai.sdk.core.common.ClientException; import com.openai.models.realtime.ResponseCreateEvent; import com.openai.models.realtime.SessionUpdateEvent; -import com.openai.models.realtime.ConversationItemCreateEvent; -import com.openai.models.realtime.ConversationItem; - +import com.sap.ai.sdk.core.common.ClientException; import java.net.URI; import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; -import java.util.TimerTask; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.Map; import java.util.Set; import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 9a6d251b3b6ba809a650a18295bbb8a063d8faf2 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 16:23:13 +0200 Subject: [PATCH 18/67] extract common realtime api functionality to a superclass --- .../openai/SpeechToSpeechRealtimeClient.java | 124 ++++++------------ .../openai/SpeechToTextRealtimeClient.java | 94 ------------- .../openai/TextToSpeechRealtimeClient.java | 110 ++++------------ .../openai/ToAudioRealtimeClient.java | 80 +++++++++-- 4 files changed, 131 insertions(+), 277 deletions(-) delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 3f3085e9d..31b8611ad 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -1,30 +1,26 @@ package com.sap.ai.sdk.foundationmodels.openai; -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.openai.models.realtime.InputAudioBufferAppendEvent; +import com.openai.models.realtime.InputAudioBufferCommitEvent; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import com.sap.ai.sdk.core.RealtimeParamVoice; -import java.util.*; + import lombok.extern.slf4j.Slf4j; +import java.util.Arrays; +import java.util.Base64; +import java.util.Map; + @Slf4j -class SpeechToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { +class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - - private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final String TASK = ""; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; private final boolean eagerTurnDetection; public SpeechToSpeechRealtimeClient( @@ -32,33 +28,41 @@ public SpeechToSpeechRealtimeClient( Map httpHeaders, AudioOutputChannel outputConsumer, RealtimeParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; + super(url, httpHeaders, outputConsumer, params); - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; var turnDetectionEager = false; for (RealtimeParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - case TURN_DETECTION -> { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } } - this.voice = voice; this.eagerTurnDetection = turnDetectionEager; } + @Override + protected RealtimeAudioConfigInput inputConfig() { + RealtimeAudioInputTurnDetection turnDetection; + if (eagerTurnDetection) { + turnDetection = null; + } else { + turnDetection = + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + } + + return RealtimeAudioConfigInput.builder() + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()).build(); + } + public void inputAudio(byte[] rawAudioChunk) { if (rawAudioChunk.length == 0) { return; @@ -87,56 +91,4 @@ protected String getSystemPrompt() { return TASK; } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - RealtimeAudioInputTurnDetection turnDetection; - if (eagerTurnDetection) { - turnDetection = null; - } else { - turnDetection = - RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); - } - - return SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(turnDetection) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice(voice) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java deleted file mode 100644 index e8ed82610..000000000 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToTextRealtimeClient.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai; - -import com.fasterxml.jackson.databind.JsonNode; -import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import java.util.*; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -class SpeechToTextRealtimeClient extends WSOpenAiRealtimeClient implements AudioInputChannel { - - private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio_transcript.done"); - - private static final String TASK = - "you are a transcript writer and your role is to convert input speech to text as precise as possible (word to word exactly, if possible)"; - - private final TextOutputChannel outputConsumer; - - public SpeechToTextRealtimeClient( - String url, Map httpHeaders, TextOutputChannel outputConsumer) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - } - - public void inputAudio(byte[] rawAudioChunk) { - if (rawAudioChunk.length == 0) { - return; - } - var cursorLeft = 0; - while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = - InputAudioBufferAppendEvent.builder() - .audio(Base64.getEncoder().encodeToString(part)) - .build(); - super.sendMessage(audioInputMessage); - cursorLeft += MAX_DATA_CHUNK_SIZE_BYTES; - } - - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); - super.sendMessage(commitAudioMessage); - askForResponse(); - } - - @Override - protected String getSystemPrompt() { - return TASK; - } - - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio_transcript.done".equals(eventType)) { - this.outputConsumer.outputText( - event.has("transcript") ? event.get("transcript").asText() : "", Boolean.TRUE); - } else { - log.trace("skipping message type: {}", eventType); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofTranscription( - RealtimeTranscriptionSessionCreateRequest.builder() - .audio( - RealtimeTranscriptionSessionAudio.builder() - .input( - RealtimeTranscriptionSessionAudioInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .transcription( - AudioTranscription.builder() - .model( - AudioTranscription.Model.GPT_4O_TRANSCRIBE) - .build()) - .build()) - .build()) - .build()) - .asTranscription()) - .build(); - return sessionUpdateEvent; - } -} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 5734e284d..d2fc08a6a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -1,31 +1,18 @@ package com.sap.ai.sdk.foundationmodels.openai; -import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.*; -import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import com.sap.ai.sdk.core.RealtimeParamVoice; import java.util.*; import lombok.extern.slf4j.Slf4j; @Slf4j -class TextToSpeechRealtimeClient extends WSOpenAiRealtimeClient implements TextInputChannel { - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Set HANDLED_RESPONSE_TYPES = - Set.of("response.output_audio.delta", "response.output_audio.done"); - - private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); +class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextInputChannel { private static final String TASK = "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + "do not answer questions, just read them"; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; private final boolean eagerTurnDetection; public TextToSpeechRealtimeClient( @@ -33,36 +20,34 @@ public TextToSpeechRealtimeClient( Map httpHeaders, AudioOutputChannel outputConsumer, RealtimeParam... params) { - super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - this.outputConsumer = outputConsumer; - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } else { - voice = - RealtimeAudioConfigOutput.Voice.UnionMember1.of( - param.getValueAsString().toLowerCase()); - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } } - case TURN_DETECTION -> { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } } - this.voice = voice; this.eagerTurnDetection = turnDetectionEager; } + @Override + protected RealtimeAudioConfigInput inputConfig() { + return RealtimeAudioConfigInput.builder() + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type( + RealtimeAudioFormats.AudioPcm.Type + .AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); + } + public void sendText(String text) { var message = ConversationItemCreateEvent.builder() @@ -88,55 +73,4 @@ protected String getSystemPrompt() { return TASK; } - @Override - protected void onResponse(String eventType, JsonNode event) { - if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); - } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); - } else { - log.warn("skipping message type: {}", eventType); - } - } - - @Override - protected SessionUpdateEvent sessionConfiguration() { - SessionUpdateEvent sessionUpdateEvent = - SessionUpdateEvent.builder() - .session( - ClientSecretCreateParams.Session.ofRealtime( - RealtimeSessionCreateRequest.builder() - .outputModalities(OUTPUT_MODALITIES) - .audio( - RealtimeAudioConfig.builder() - .input( - RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build()) - .output( - RealtimeAudioConfigOutput.builder() - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .voice(voice) - .build()) - .build()) - .build()) - .asRealtime()) - .build(); - return sessionUpdateEvent; - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 7033cfec7..50b4db464 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -1,30 +1,92 @@ package com.sap.ai.sdk.foundationmodels.openai; import com.fasterxml.jackson.databind.JsonNode; +import com.openai.models.realtime.RealtimeAudioConfig; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioConfigOutput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeSessionCreateRequest; import com.openai.models.realtime.SessionUpdateEvent; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.RealtimeParamVoice; +import lombok.extern.slf4j.Slf4j; + +import java.util.Base64; +import java.util.List; import java.util.Map; import java.util.Set; -class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { +@Slf4j +abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); - private static final String DEFAULT_SYSTEM_PROMPT = ""; + private static final List OUTPUT_MODALITIES = + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private final AudioOutputChannel outputConsumer; + private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - public ToAudioRealtimeClient(String url, Map httpHeaders) { + public ToAudioRealtimeClient(String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + for (RealtimeParam param : params) { + switch (param.getParamName()) { + case VOICE -> { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + } + } + this.outputConsumer = outputConsumer; + this.voice = voice; } - @Override - protected String getSystemPrompt() { - return ""; - } + protected abstract RealtimeAudioConfigInput inputConfig(); @Override - protected void onResponse(String eventType, JsonNode event) {} + protected void onResponse(String eventType, JsonNode event) { + if ("response.output_audio.delta".equals(eventType)) { + var base64Audio = event.get("delta").asText(); + byte[] audio = Base64.getDecoder().decode(base64Audio); + this.outputConsumer.outputAudio(audio, Boolean.FALSE); + } else if ("response.output_audio.done".equals(eventType)) { + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + } else { + log.warn("skipping message type: {}", eventType); + } + } @Override protected SessionUpdateEvent sessionConfiguration() { - return null; + return SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities(OUTPUT_MODALITIES) + .audio( + RealtimeAudioConfig.builder() + .input(inputConfig()) + .output( + RealtimeAudioConfigOutput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice(voice) + .build()) + .build()) + .build()) + .asRealtime()) + .build(); } } From 0f2cd624843813e9d8f40209615fbeefc1edff39 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 14:24:26 +0000 Subject: [PATCH 19/67] Formatting --- .../openai/SpeechToSpeechRealtimeClient.java | 34 +++++++++---------- .../openai/TextToSpeechRealtimeClient.java | 29 +++++++--------- .../openai/ToAudioRealtimeClient.java | 14 ++++---- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 31b8611ad..9de369923 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -7,12 +7,10 @@ import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; - -import lombok.extern.slf4j.Slf4j; - import java.util.Arrays; import java.util.Base64; import java.util.Map; +import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { @@ -32,13 +30,13 @@ public SpeechToSpeechRealtimeClient( var turnDetectionEager = false; for (RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; } + } } this.eagerTurnDetection = turnDetectionEager; } @@ -50,17 +48,18 @@ protected RealtimeAudioConfigInput inputConfig() { turnDetection = null; } else { turnDetection = - RealtimeAudioInputTurnDetection.ofSemanticVad( - RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build()); } return RealtimeAudioConfigInput.builder() - .turnDetection(turnDetection) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()).build(); + .turnDetection(turnDetection) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); } public void inputAudio(byte[] rawAudioChunk) { @@ -90,5 +89,4 @@ public void inputAudio(byte[] rawAudioChunk) { protected String getSystemPrompt() { return TASK; } - } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index d2fc08a6a..6bde180ff 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -23,13 +23,13 @@ public TextToSpeechRealtimeClient( super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; } + } } this.eagerTurnDetection = turnDetectionEager; } @@ -37,15 +37,13 @@ public TextToSpeechRealtimeClient( @Override protected RealtimeAudioConfigInput inputConfig() { return RealtimeAudioConfigInput.builder() - .turnDetection(Optional.empty()) - .format( - RealtimeAudioFormats.AudioPcm.builder() - .type( - RealtimeAudioFormats.AudioPcm.Type - .AUDIO_PCM) - .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) - .build()) - .build(); + .turnDetection(Optional.empty()) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); } public void sendText(String text) { @@ -72,5 +70,4 @@ public void sendText(String text) { protected String getSystemPrompt() { return TASK; } - } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 50b4db464..7279ba465 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -10,12 +10,11 @@ import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamVoice; -import lombok.extern.slf4j.Slf4j; - import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Set; +import lombok.extern.slf4j.Slf4j; @Slf4j abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { @@ -23,16 +22,17 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private static final Set HANDLED_RESPONSE_TYPES = Set.of("response.output_audio.delta", "response.output_audio.done"); private static final List OUTPUT_MODALITIES = - List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); + List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - public ToAudioRealtimeClient(String url, - Map httpHeaders, - AudioOutputChannel outputConsumer, - RealtimeParam... params) { + public ToAudioRealtimeClient( + String url, + Map httpHeaders, + AudioOutputChannel outputConsumer, + RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (RealtimeParam param : params) { From 6082105816f437b0c339b5e18f23eb4a3395f0bf Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 16:34:53 +0200 Subject: [PATCH 20/67] fix misleading javadocs --- .../sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java | 2 -- .../sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index b190fbda5..598fddabf 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -2,8 +2,6 @@ /** * Functional interface representing audio output channel (audio data consumer) - * - *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface AudioOutputChannel { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 5896b85ab..0d67d7af0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -2,8 +2,6 @@ /** * Functional interface representing text output channel (text data consumer) - * - *

    Should be closed by application (try-with-resources) when not needed anymore */ public interface TextOutputChannel { From 537203d538675ce5ce687c39de3d8ad224ab7143 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 17 Jul 2026 14:36:07 +0000 Subject: [PATCH 21/67] Formatting --- .../ai/sdk/foundationmodels/openai/AudioOutputChannel.java | 4 +--- .../sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index 598fddabf..81217d100 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -1,8 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai; -/** - * Functional interface representing audio output channel (audio data consumer) - */ +/** Functional interface representing audio output channel (audio data consumer) */ public interface AudioOutputChannel { /** diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index 0d67d7af0..add9c2460 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,8 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai; -/** - * Functional interface representing text output channel (text data consumer) - */ +/** Functional interface representing text output channel (text data consumer) */ public interface TextOutputChannel { /** From 7a5220eb8a7f0e161c1dec21259e6aaaaa233f2c Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 17 Jul 2026 17:16:37 +0200 Subject: [PATCH 22/67] replace debug output in test with actual assertions --- .../ai/sdk/app/controllers/RealtimeApiTest.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 177d8d208..10f6027e4 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -9,7 +9,6 @@ import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.ByteArrayOutputStream; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -98,11 +97,15 @@ void testSpeechToSpeech() { return; } - try (var fis = new FileOutputStream("/tmp/response.pcm")) { - fis.write(outputBuffer.toByteArray()); - } catch (IOException e) { - throw new RuntimeException(e); - } + assertThat(monitor.getCount()).isEqualTo(0); + assertThat(outputBuffer.size()).isGreaterThan(0); + + var metrics = pcm16AudioMetrics(outputBuffer.toByteArray()); + + // root mean squire (measures the deviation from an average value) + assertThat(metrics.rms).isGreaterThan(500d); + // asserts that variety of deviation is sufficient (not a trivial repeating pattern) + assertThat(metrics.entropy).isGreaterThan(4); } private record AudioMetrics(double rms, double entropy) {} From e7d844d631d749d6a39639d28c6455145b2c6e13 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 20 Jul 2026 11:42:45 +0200 Subject: [PATCH 23/67] fix annotations from invalid package --- .../com/sap/ai/sdk/core/RealtimeParamTurnDetection.java | 6 +++--- .../main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java | 5 ++--- .../ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 5 +++-- .../ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 60f736fd9..4ace8ce1e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -1,7 +1,7 @@ package com.sap.ai.sdk.core; +import javax.annotation.Nonnull; import java.util.Objects; -import org.jspecify.annotations.NonNull; public final class RealtimeParamTurnDetection implements RealtimeParam { @@ -17,12 +17,12 @@ private RealtimeParamTurnDetection(String turnDetectionKind) { } @Override - public @NonNull SpeechOutputParamName getParamName() { + public @Nonnull SpeechOutputParamName getParamName() { return SpeechOutputParamName.TURN_DETECTION; } @Override - public @NonNull String getValueAsString() { + public @Nonnull String getValueAsString() { return turnDetectionKind; } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 3746a2f3f..2a4178a2e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -2,7 +2,6 @@ import java.util.Objects; import javax.annotation.Nonnull; -import org.jspecify.annotations.NonNull; public final class RealtimeParamVoice implements RealtimeParam { @@ -20,12 +19,12 @@ public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { } @Override - public @NonNull SpeechOutputParamName getParamName() { + public @Nonnull SpeechOutputParamName getParamName() { return SpeechOutputParamName.VOICE; } @Override - public @NonNull String getValueAsString() { + public @Nonnull String getValueAsString() { return voice; } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 696d10fab..7ed9ad336 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.socket.BinaryMessage; @@ -15,6 +14,8 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +import javax.annotation.Nonnull; + @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -48,7 +49,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 34469513f..cede97963 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; -import org.jspecify.annotations.NonNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.socket.BinaryMessage; @@ -15,6 +14,8 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +import javax.annotation.Nonnull; + @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -48,7 +49,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @NonNull CloseStatus status) + public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), From 7c6dced7f54fa2f3bb2d7e73cd73158f9e01e063 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 20 Jul 2026 09:43:55 +0000 Subject: [PATCH 24/67] Formatting --- .../java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java | 2 +- .../ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 3 +-- .../sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 4ace8ce1e..a7b247546 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -1,7 +1,7 @@ package com.sap.ai.sdk.core; -import javax.annotation.Nonnull; import java.util.Objects; +import javax.annotation.Nonnull; public final class RealtimeParamTurnDetection implements RealtimeParam { diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 7ed9ad336..64911e08b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -14,8 +15,6 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import javax.annotation.Nonnull; - @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index cede97963..88ce6109b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -14,8 +15,6 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -import javax.annotation.Nonnull; - @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { From 671f96078204e1ec02963e6ab6fa8d0140d671cd Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 20 Jul 2026 11:59:12 +0200 Subject: [PATCH 25/67] add missing javadoc comments --- .../com/sap/ai/sdk/core/RealtimeParam.java | 11 +++++++++++ .../sdk/core/RealtimeParamTurnDetection.java | 12 ++++++++++++ .../sap/ai/sdk/core/RealtimeParamVoice.java | 18 +++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index bb4cd79c4..5c03cba92 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -2,15 +2,26 @@ import javax.annotation.Nonnull; +/** + * Represents possible configuration params of realtime client + */ public interface RealtimeParam { enum SpeechOutputParamName { VOICE, TURN_DETECTION, } + /** + * Returns param name + * @return name + */ @Nonnull SpeechOutputParamName getParamName(); + /** + * Returns string value representation of the param + * @return string value + */ @Nonnull String getValueAsString(); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 4ace8ce1e..cadbbf32b 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -3,10 +3,22 @@ import javax.annotation.Nonnull; import java.util.Objects; +/** + * Allows to configure turn detection (how model responds). + */ public final class RealtimeParamTurnDetection implements RealtimeParam { + /** + * Model tries to recognize if/when it should respond automatically + */ public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); + + /** + * Each call to the provided realtime client is considered a turn (eager explicit turn detection). + * Less convenient than the automatic option but may give lower latency in some cases (model does not need + * to perform additional turn detection analysis). + */ public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 2a4178a2e..1c48040a1 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -3,10 +3,20 @@ import java.util.Objects; import javax.annotation.Nonnull; +/** + * Allows to configure model output voice + */ public final class RealtimeParamVoice implements RealtimeParam { + /** + * Standard voice 1 + */ public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); - public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_1"); + + /** + * Standard voice 2 + */ + public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); private final String voice; @@ -14,6 +24,12 @@ private RealtimeParamVoice(@Nonnull String voice) { this.voice = voice; } + /** + * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify + * in advance if the provided voice name is correct and supported by the chosen model and use case + * @param voiceName as named by model provider + * @return typed voice client configuration param + */ public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { return new RealtimeParamVoice(voiceName); } From b4ba4715be8dcf68f870be52e92732d1cd326b9b Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Mon, 20 Jul 2026 10:00:27 +0000 Subject: [PATCH 26/67] Formatting --- .../java/com/sap/ai/sdk/core/RealtimeParam.java | 6 +++--- .../sap/ai/sdk/core/RealtimeParamTurnDetection.java | 12 ++++-------- .../com/sap/ai/sdk/core/RealtimeParamVoice.java | 13 ++++--------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index 5c03cba92..9c1e4d57e 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -2,9 +2,7 @@ import javax.annotation.Nonnull; -/** - * Represents possible configuration params of realtime client - */ +/** Represents possible configuration params of realtime client */ public interface RealtimeParam { enum SpeechOutputParamName { VOICE, @@ -13,6 +11,7 @@ enum SpeechOutputParamName { /** * Returns param name + * * @return name */ @Nonnull @@ -20,6 +19,7 @@ enum SpeechOutputParamName { /** * Returns string value representation of the param + * * @return string value */ @Nonnull diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 3327f92a2..78fbdfa3a 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -3,21 +3,17 @@ import java.util.Objects; import javax.annotation.Nonnull; -/** - * Allows to configure turn detection (how model responds). - */ +/** Allows to configure turn detection (how model responds). */ public final class RealtimeParamTurnDetection implements RealtimeParam { - /** - * Model tries to recognize if/when it should respond automatically - */ + /** Model tries to recognize if/when it should respond automatically */ public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); /** * Each call to the provided realtime client is considered a turn (eager explicit turn detection). - * Less convenient than the automatic option but may give lower latency in some cases (model does not need - * to perform additional turn detection analysis). + * Less convenient than the automatic option but may give lower latency in some cases (model does + * not need to perform additional turn detection analysis). */ public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 1c48040a1..1743430b0 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -3,19 +3,13 @@ import java.util.Objects; import javax.annotation.Nonnull; -/** - * Allows to configure model output voice - */ +/** Allows to configure model output voice */ public final class RealtimeParamVoice implements RealtimeParam { - /** - * Standard voice 1 - */ + /** Standard voice 1 */ public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); - /** - * Standard voice 2 - */ + /** Standard voice 2 */ public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); private final String voice; @@ -27,6 +21,7 @@ private RealtimeParamVoice(@Nonnull String voice) { /** * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify * in advance if the provided voice name is correct and supported by the chosen model and use case + * * @param voiceName as named by model provider * @return typed voice client configuration param */ From 85120c15fd1c6c29d09dc6408109779b7e8305d4 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 11:38:27 +0200 Subject: [PATCH 27/67] add missing javadocs --- .../src/main/java/com/sap/ai/sdk/core/RealtimeParam.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index 9c1e4d57e..a2f93e9e8 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -4,8 +4,17 @@ /** Represents possible configuration params of realtime client */ public interface RealtimeParam { + /** + * Represents configurable options + */ enum SpeechOutputParamName { + /** + * Voice name to use to produce sound + */ VOICE, + /** + * How model will recognize that it is its turn to respond (e.g. explicitly asked, automatically detected) + */ TURN_DETECTION, } From fa4bb641a6da770f070e25f89a639491534c13b5 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Thu, 23 Jul 2026 09:39:25 +0000 Subject: [PATCH 28/67] Formatting --- .../main/java/com/sap/ai/sdk/core/RealtimeParam.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java index a2f93e9e8..ae31c9739 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java @@ -4,16 +4,13 @@ /** Represents possible configuration params of realtime client */ public interface RealtimeParam { - /** - * Represents configurable options - */ + /** Represents configurable options */ enum SpeechOutputParamName { - /** - * Voice name to use to produce sound - */ + /** Voice name to use to produce sound */ VOICE, /** - * How model will recognize that it is its turn to respond (e.g. explicitly asked, automatically detected) + * How model will recognize that it is its turn to respond (e.g. explicitly asked, automatically + * detected) */ TURN_DETECTION, } From e4e04b35630a80201807e9dd195804075814134f Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 12:34:57 +0200 Subject: [PATCH 29/67] fix style issues --- .../ai/sdk/core/RealtimeParamTurnDetection.java | 11 +++++++---- .../com/sap/ai/sdk/core/RealtimeParamVoice.java | 14 +++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java index 78fbdfa3a..a0d82a3ec 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java @@ -2,6 +2,7 @@ import java.util.Objects; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** Allows to configure turn detection (how model responds). */ public final class RealtimeParamTurnDetection implements RealtimeParam { @@ -20,7 +21,7 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { private final String turnDetectionKind; - private RealtimeParamTurnDetection(String turnDetectionKind) { + private RealtimeParamTurnDetection(final String turnDetectionKind) { this.turnDetectionKind = turnDetectionKind; } @@ -35,9 +36,11 @@ private RealtimeParamTurnDetection(String turnDetectionKind) { } @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - RealtimeParamTurnDetection that = (RealtimeParamTurnDetection) o; + public boolean equals(@Nullable final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final RealtimeParamTurnDetection that = (RealtimeParamTurnDetection) o; return Objects.equals(turnDetectionKind, that.turnDetectionKind); } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java index 1743430b0..c16e675d6 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java @@ -2,6 +2,7 @@ import java.util.Objects; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** Allows to configure model output voice */ public final class RealtimeParamVoice implements RealtimeParam { @@ -14,7 +15,7 @@ public final class RealtimeParamVoice implements RealtimeParam { private final String voice; - private RealtimeParamVoice(@Nonnull String voice) { + private RealtimeParamVoice(@Nonnull final String voice) { this.voice = voice; } @@ -25,7 +26,8 @@ private RealtimeParamVoice(@Nonnull String voice) { * @param voiceName as named by model provider * @return typed voice client configuration param */ - public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { + @Nonnull + public static RealtimeParamVoice unsafeWithExplicitVoice(@Nonnull final String voiceName) { return new RealtimeParamVoice(voiceName); } @@ -40,9 +42,11 @@ public static RealtimeParamVoice unsafeWithExplicitVoice(String voiceName) { } @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - RealtimeParamVoice that = (RealtimeParamVoice) o; + public boolean equals(@Nullable final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final RealtimeParamVoice that = (RealtimeParamVoice) o; return Objects.equals(voice, that.voice); } From 94a0eefddeb14c65251752cfb29d071edbf16887 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 12:59:51 +0200 Subject: [PATCH 30/67] fix dependency issue --- foundation-models/openai/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 91a93f52c..f45744bdf 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -44,8 +44,15 @@ 79% 91% 92% + 1.9.10 + + Used by openai-java, but has to be overridden because of versioning issues<--> + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.stdlib.jdk8.version} + com.sap.cloud.sdk.cloudplatform cloudplatform-connectivity @@ -73,6 +80,12 @@ com.openai openai-java + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + + com.google.code.findbugs From 5fbe0da17ee22ec466a0841b7f55ec38a99ec035 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 13:08:37 +0200 Subject: [PATCH 31/67] fix dependency issue --- foundation-models/openai/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index f45744bdf..c788bdbf7 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -52,6 +52,7 @@ org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin.stdlib.jdk8.version} + runtime com.sap.cloud.sdk.cloudplatform From 50392d640f971cec0911257d538dd4c50a3efd84 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 13:25:15 +0200 Subject: [PATCH 32/67] fix dependency issue --- foundation-models/openai/pom.xml | 13 +++++-------- pom.xml | 7 +++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index c788bdbf7..dcec2e293 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -44,16 +44,8 @@ 79% 91% 92% - 1.9.10 - - Used by openai-java, but has to be overridden because of versioning issues<--> - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.stdlib.jdk8.version} - runtime - com.sap.cloud.sdk.cloudplatform cloudplatform-connectivity @@ -78,6 +70,11 @@ com.sap.ai.sdk core + + Used by openai-java, but has to be overridden because of versioning issues<--> + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + com.openai openai-java diff --git a/pom.xml b/pom.xml index ec3bb4f6d..a1d5b4edb 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,7 @@ /tmp/baseline.jar ${project.build.directory}/${project.build.finalName}.jar + 1.9.10 @@ -173,6 +174,12 @@ httpcore5 ${httpcomponents-core5.version} + + Used by openai-java, but has to be overridden because of versioning issues<--> + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.stdlib.jdk8.version} + io.micrometer micrometer-core From 86cd068055cafce4199a6804bd9e2af0390a6eb5 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 15:35:26 +0200 Subject: [PATCH 33/67] fix dependency issue --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1d5b4edb..95776622f 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ /tmp/baseline.jar ${project.build.directory}/${project.build.finalName}.jar - 1.9.10 + 1.8.10 From e95a5df14b62d1b86f69e37fa8a6cb7b6d2e9929 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 16:11:56 +0200 Subject: [PATCH 34/67] fix dependency issue --- foundation-models/openai/pom.xml | 6 ------ pom.xml | 12 +++++++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index dcec2e293..de1dcfcd6 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -78,12 +78,6 @@ com.openai openai-java - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - - com.google.code.findbugs diff --git a/pom.xml b/pom.xml index 95776622f..9ad3a559c 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ /tmp/baseline.jar ${project.build.directory}/${project.build.finalName}.jar - 1.8.10 + 1.9.10 @@ -180,6 +180,16 @@ kotlin-stdlib-jdk8 ${kotlin.stdlib.jdk8.version} + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.stdlib.jdk8.version} + + + org.jetbrains.kotlin + kotlin-stdlib-common + ${kotlin.stdlib.jdk8.version} + io.micrometer micrometer-core From ac27bb7c34c7fd3fff91ce48c748b743d76f82af Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 16:37:58 +0200 Subject: [PATCH 35/67] fix dependency issue --- foundation-models/openai/pom.xml | 6 +----- pom.xml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index de1dcfcd6..bc5bdeb22 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -70,14 +70,10 @@ com.sap.ai.sdk core - - Used by openai-java, but has to be overridden because of versioning issues<--> - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - com.openai openai-java + runtime com.google.code.findbugs diff --git a/pom.xml b/pom.xml index 9ad3a559c..b001befb5 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ ${httpcomponents-core5.version} - Used by openai-java, but has to be overridden because of versioning issues<--> + Pinned to align all kotlin-stdlib transitive versions pulled via openai-java and okio<--> org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin.stdlib.jdk8.version} From e38a42a00064651485b6c135dfae680cc2a0e818 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 16:58:50 +0200 Subject: [PATCH 36/67] add sample app docs and hints --- .../spring-app/src/main/resources/static/index.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 014fddcf1..bf2a27301 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1455,6 +1455,8 @@

    ⏰ Realtime API

    Realtime API allows for various real time interactions TBD Documentation + In these examples, web socket sessions are created and used + to send text/audio to the local server and receive audio back
      @@ -1467,6 +1469,9 @@

      Text to speech

    +
    + Translate input text into output sound (speech). +
    @@ -1478,6 +1483,9 @@

    Speech to speech

    +
    + Speak with an AI assistant. +
    From 4b789d36bc737dfaa17a026a088b760c6ad0bfa8 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 17:12:27 +0200 Subject: [PATCH 37/67] improve dependency declarations --- foundation-models/openai/pom.xml | 1 - pom.xml | 6 +++--- sample-code/spring-app/src/main/resources/static/index.html | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index bc5bdeb22..91a93f52c 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -73,7 +73,6 @@ com.openai openai-java - runtime com.google.code.findbugs diff --git a/pom.xml b/pom.xml index b001befb5..339726b89 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ 2.1.3 3.5.6 1.1.8 - 4.41.0 + 4.41.0 3.8.6 3.2.0 5.23.0 @@ -121,12 +121,12 @@ pom import + com.openai openai-java - ${com.openai.openai-java.version} + ${openai-java.version} - io.projectreactor reactor-core diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index bf2a27301..1c822a32d 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1455,6 +1455,7 @@

    ⏰ Realtime API

    Realtime API allows for various real time interactions TBD Documentation +
    In these examples, web socket sessions are created and used to send text/audio to the local server and receive audio back From c4f597b15f5473c0a17514e5c493f4a56631d3b4 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Thu, 23 Jul 2026 17:22:01 +0200 Subject: [PATCH 38/67] improve code and docs style --- .../foundationmodels/openai/OpenAiRealtimeClient.java | 6 ++++++ .../ai/sdk/foundationmodels/openai/TextInputChannel.java | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index 4ccbbd12c..de90a2bf9 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -7,6 +7,10 @@ import java.util.HashMap; import javax.annotation.Nonnull; +/** + * OpenAI client implementation of Realtime API. Abstracts technical implementation, + * transport and threading and exposes business-level operations (high level interface) + */ public class OpenAiRealtimeClient { private static final int PATH_BUFFER_SIZE = @@ -41,6 +45,8 @@ public class OpenAiRealtimeClient { * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth + * @param params - allows for various additional features (e.g. voice configuration or conversation + * turn recognition options) * @return input channel, allowing for text input */ @Nonnull diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index e1cfe32db..cac14f061 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,5 +1,14 @@ package com.sap.ai.sdk.foundationmodels.openai; +/** + * Allows to input (send) text to the open channel, must be closed when + * not needed anymore (e.g. try-with-resources) + */ public interface TextInputChannel extends AutoCloseable { + + /** + * Sends input text + * @param text text to send + */ void sendText(String text); } From 611c0ebfb22db0360361c9b51237ef3fa2f48b26 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Thu, 23 Jul 2026 15:22:57 +0000 Subject: [PATCH 39/67] Formatting --- .../sdk/foundationmodels/openai/OpenAiRealtimeClient.java | 8 ++++---- .../ai/sdk/foundationmodels/openai/TextInputChannel.java | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index de90a2bf9..43142beee 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -8,8 +8,8 @@ import javax.annotation.Nonnull; /** - * OpenAI client implementation of Realtime API. Abstracts technical implementation, - * transport and threading and exposes business-level operations (high level interface) + * OpenAI client implementation of Realtime API. Abstracts technical implementation, transport and + * threading and exposes business-level operations (high level interface) */ public class OpenAiRealtimeClient { @@ -45,8 +45,8 @@ public class OpenAiRealtimeClient { * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth - * @param params - allows for various additional features (e.g. voice configuration or conversation - * turn recognition options) + * @param params - allows for various additional features (e.g. voice configuration or + * conversation turn recognition options) * @return input channel, allowing for text input */ @Nonnull diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index cac14f061..ccca95395 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,13 +1,14 @@ package com.sap.ai.sdk.foundationmodels.openai; /** - * Allows to input (send) text to the open channel, must be closed when - * not needed anymore (e.g. try-with-resources) + * Allows to input (send) text to the open channel, must be closed when not needed anymore (e.g. + * try-with-resources) */ public interface TextInputChannel extends AutoCloseable { /** * Sends input text + * * @param text text to send */ void sendText(String text); From b90ab43902f2cc2842a10c16f59cf2ab7d497757 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 12:10:29 +0200 Subject: [PATCH 40/67] use open-ai-core sub dependency as nothing from open-ai dep is explicitly used --- foundation-models/openai/pom.xml | 5 ----- pom.xml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 91a93f52c..6505871af 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -70,10 +70,6 @@ com.sap.ai.sdk core     - - com.openai - openai-java - com.google.code.findbugs jsr305 @@ -128,7 +124,6 @@ com.openai openai-java-core - true diff --git a/pom.xml b/pom.xml index 339726b89..a8502216a 100644 --- a/pom.xml +++ b/pom.xml @@ -122,11 +122,6 @@ import - - com.openai - openai-java - ${openai-java.version} - io.projectreactor reactor-core From dc53160705ce0e7aec8f0b2af7167f55a79b7b47 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 12:38:18 +0200 Subject: [PATCH 41/67] address code style pmd issues --- .../openai/BufferedWebSocketListener.java | 22 +++++++-- .../foundationmodels/openai/OpenAiClient.java | 3 +- .../openai/OpenAiRealtimeClient.java | 18 +++---- .../openai/SpeechToSpeechRealtimeClient.java | 20 ++++---- .../openai/TextInputChannel.java | 4 +- .../openai/TextToSpeechRealtimeClient.java | 17 ++++--- .../openai/ToAudioRealtimeClient.java | 16 ++++--- .../openai/WSOpenAiRealtimeClient.java | 47 +++++++++++-------- 8 files changed, 91 insertions(+), 56 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 1eb449229..b04318f8f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -8,6 +8,8 @@ import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; + @Slf4j class BufferedWebSocketListener implements WebSocket.Listener { @@ -16,20 +18,25 @@ class BufferedWebSocketListener implements WebSocket.Listener { private final StringBuilder buffer; BufferedWebSocketListener( - Consumer onOpen, BiConsumer onText) { + @Nonnull final Consumer onOpen, @Nonnull final BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; this.buffer = new StringBuilder(1024 * 1024); } @Override - public void onOpen(WebSocket webSocket) { + public void onOpen(@Nonnull final WebSocket webSocket) { this.onOpen.accept(webSocket); webSocket.request(1); } @Override - public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean isLast) { + @Nonnull + public CompletionStage onText( + final @Nonnull WebSocket webSocket, + final @Nonnull CharSequence data, + final boolean isLast + ) { buffer.append(data); webSocket.request(1); if (isLast) { @@ -42,12 +49,17 @@ public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean } @Override - public void onError(WebSocket webSocket, Throwable error) { + public void onError(@Nonnull final WebSocket webSocket, @Nonnull final Throwable error) { log.error("Websocket error occurred during realtime communication", error); } @Override - public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean isLast) { + @Nonnull + public CompletionStage onBinary( + @Nonnull final WebSocket webSocket, + @Nonnull final ByteBuffer data, + final boolean isLast + ) { log.warn("Received unexpected binary bytes for WebSocket connection"); return CompletableFuture.completedStage(null); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index f34eb1622..691aee48a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -82,8 +82,9 @@ public static OpenAiClient forModel(@Nonnull final OpenAiModel foundationModel) * @return created client */ @Beta + @Nonnull public static OpenAiRealtimeClient realtimeClient() { - var withResolvedDestination = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); + final var withResolvedDestination = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME); return new OpenAiRealtimeClient(withResolvedDestination.destination); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index 43142beee..355469ec0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -18,7 +18,7 @@ public class OpenAiRealtimeClient { private final Destination destination; - OpenAiRealtimeClient(Destination destination) { + OpenAiRealtimeClient(@Nonnull final Destination destination) { this.destination = destination; } @@ -53,12 +53,12 @@ public class OpenAiRealtimeClient { @Beta public TextInputChannel textToSpeech( @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); + final var extraHeaders = destination.asHttp().getHeaders(); + final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { headers.put(header.getName(), header.getValue()); } - var endpoint = getRealtimeEndpoint(); + final var endpoint = getRealtimeEndpoint(); return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); } @@ -94,20 +94,20 @@ public TextInputChannel textToSpeech( @Beta public AudioInputChannel speechToSpeech( @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { - var extraHeaders = destination.asHttp().getHeaders(); - var headers = new HashMap(extraHeaders.size() + 1); + final var extraHeaders = destination.asHttp().getHeaders(); + final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { headers.put(header.getName(), header.getValue()); } - var endpoint = getRealtimeEndpoint(); + final var endpoint = getRealtimeEndpoint(); return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); } private String getRealtimeEndpoint() { - var sb = new StringBuilder(PATH_BUFFER_SIZE); + final var sb = new StringBuilder(PATH_BUFFER_SIZE); sb.append("wss://"); - var pathParts = destination.asHttp().getUri().toString().split("//"); + final var pathParts = destination.asHttp().getUri().toString().split("//"); if (pathParts.length != 2) { throw new IllegalArgumentException( "Invalid destination URI: " + destination.asHttp().getUri()); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 9de369923..79ee89f3d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -12,6 +12,8 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; + @Slf4j class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { @@ -22,9 +24,9 @@ class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements Audi private final boolean eagerTurnDetection; public SpeechToSpeechRealtimeClient( - String url, - Map httpHeaders, - AudioOutputChannel outputConsumer, + @Nonnull final String url, + @Nonnull final Map httpHeaders, + @Nonnull final AudioOutputChannel outputConsumer, RealtimeParam... params) { super(url, httpHeaders, outputConsumer, params); @@ -42,6 +44,7 @@ public SpeechToSpeechRealtimeClient( } @Override + @Nonnull protected RealtimeAudioConfigInput inputConfig() { RealtimeAudioInputTurnDetection turnDetection; if (eagerTurnDetection) { @@ -62,15 +65,15 @@ protected RealtimeAudioConfigInput inputConfig() { .build(); } - public void inputAudio(byte[] rawAudioChunk) { + public void inputAudio(@Nonnull final byte[] rawAudioChunk) { if (rawAudioChunk.length == 0) { return; } var cursorLeft = 0; while (cursorLeft < rawAudioChunk.length) { - var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); - var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); - var audioInputMessage = + final var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + final var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); + final var audioInputMessage = InputAudioBufferAppendEvent.builder() .audio(Base64.getEncoder().encodeToString(part)) .build(); @@ -79,13 +82,14 @@ public void inputAudio(byte[] rawAudioChunk) { } if (eagerTurnDetection) { - var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); + final var commitAudioMessage = InputAudioBufferCommitEvent.builder().build(); super.sendMessage(commitAudioMessage); askForResponse(); } } @Override + @Nonnull protected String getSystemPrompt() { return TASK; } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index ccca95395..f1d2dadd2 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,5 +1,7 @@ package com.sap.ai.sdk.foundationmodels.openai; +import javax.annotation.Nonnull; + /** * Allows to input (send) text to the open channel, must be closed when not needed anymore (e.g. * try-with-resources) @@ -11,5 +13,5 @@ public interface TextInputChannel extends AutoCloseable { * * @param text text to send */ - void sendText(String text); + void sendText(@Nonnull final String text); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 6bde180ff..1fa0f4a3e 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,9 +3,12 @@ import com.openai.models.realtime.*; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import java.util.*; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; +import java.util.Map; +import java.util.Optional; + @Slf4j class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextInputChannel { @@ -16,9 +19,9 @@ class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextIn private final boolean eagerTurnDetection; public TextToSpeechRealtimeClient( - String url, - Map httpHeaders, - AudioOutputChannel outputConsumer, + @Nonnull final String url, + @Nonnull final Map httpHeaders, + @Nonnull final AudioOutputChannel outputConsumer, RealtimeParam... params) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; @@ -35,6 +38,7 @@ public TextToSpeechRealtimeClient( } @Override + @Nonnull protected RealtimeAudioConfigInput inputConfig() { return RealtimeAudioConfigInput.builder() .turnDetection(Optional.empty()) @@ -46,8 +50,8 @@ protected RealtimeAudioConfigInput inputConfig() { .build(); } - public void sendText(String text) { - var message = + public void sendText(@Nonnull final String text) { + final var message = ConversationItemCreateEvent.builder() .item( ConversationItem.ofRealtimeConversationItemUserMessage( @@ -67,6 +71,7 @@ public void sendText(String text) { } @Override + @Nonnull protected String getSystemPrompt() { return TASK; } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 7279ba465..93c958c82 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -16,6 +16,8 @@ import java.util.Set; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; + @Slf4j abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { @@ -29,9 +31,9 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; public ToAudioRealtimeClient( - String url, - Map httpHeaders, - AudioOutputChannel outputConsumer, + @Nonnull final String url, + @Nonnull final Map httpHeaders, + @Nonnull final AudioOutputChannel outputConsumer, RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; @@ -50,13 +52,14 @@ public ToAudioRealtimeClient( this.voice = voice; } + @Nonnull protected abstract RealtimeAudioConfigInput inputConfig(); @Override - protected void onResponse(String eventType, JsonNode event) { + protected void onResponse(@Nonnull final String eventType, @Nonnull final JsonNode event) { if ("response.output_audio.delta".equals(eventType)) { - var base64Audio = event.get("delta").asText(); - byte[] audio = Base64.getDecoder().decode(base64Audio); + final var base64Audio = event.get("delta").asText(); + final byte[] audio = Base64.getDecoder().decode(base64Audio); this.outputConsumer.outputAudio(audio, Boolean.FALSE); } else if ("response.output_audio.done".equals(eventType)) { this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); @@ -66,6 +69,7 @@ protected void onResponse(String eventType, JsonNode event) { } @Override + @Nonnull protected SessionUpdateEvent sessionConfiguration() { return SessionUpdateEvent.builder() .session( diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index 90cf63bbb..9cc0d54a0 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -25,6 +25,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import javax.annotation.Nonnull; + @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) abstract class WSOpenAiRealtimeClient implements AutoCloseable { @@ -56,7 +58,10 @@ abstract class WSOpenAiRealtimeClient implements AutoCloseable { private final Set handleMessageTypes; public WSOpenAiRealtimeClient( - String url, Map httpHeaders, Set handleMessageTypes) { + @Nonnull final String url, + @Nonnull final Map httpHeaders, + @Nonnull final Set handleMessageTypes + ) { this.client = HttpClient.newHttpClient(); var wsBuilder = this.client.newWebSocketBuilder(); for (Map.Entry entry : httpHeaders.entrySet()) { @@ -73,13 +78,13 @@ public void askForResponse() { WebSocket ws; try { ws = this.ws.join(); - } catch (CompletionException e) { + } catch (final CompletionException e) { throw new ClientException("failed to establish web socket connection", e); } synchronized (this) { try { ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { throw new RuntimeException("Failed to serialize ask for response", e); } ws.request(1); @@ -91,27 +96,29 @@ public void close() { WebSocket ws; try { ws = this.ws.join(); - } catch (CompletionException e) { + } catch (final CompletionException e) { throw new ClientException("failed to establish web socket connection", e); } synchronized (this) { heartbeatTimer.cancel(); try { ws.sendClose(SUCCESS_FINISH_WSS_CODE, "done").join(); - } catch (Exception e) { + } catch (final Exception e) { log.error("Error while closing WebSocket", e); } // this.client.close(); // exists only since java 21 } } + @Nonnull protected abstract String getSystemPrompt(); - protected abstract void onResponse(String eventType, JsonNode event); + protected abstract void onResponse(@Nonnull final String eventType, @Nonnull final JsonNode event); + @Nonnull protected abstract SessionUpdateEvent sessionConfiguration(); - protected void sendMessage(Object message) { + protected void sendMessage(@Nonnull final Object message) { WebSocket ws; try { ws = this.ws.join(); @@ -122,23 +129,23 @@ protected void sendMessage(Object message) { try { ws.sendText(JACKSON.writeValueAsString(message), true); ws.request(1); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { throw new ClientException("Failed to serialize message", e); } } } - private synchronized void onSocketOpen(WebSocket ws) { + private synchronized void onSocketOpen(@Nonnull final WebSocket ws) { configureSession(ws); configureConversation(ws); scheduleHeartbeat(ws); } - private void onText(WebSocket webSocket, CharSequence data) { + private void onText(@Nonnull final WebSocket webSocket, @Nonnull final CharSequence data) { final JsonNode event; try { event = JACKSON.readTree(data.toString()); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { throw new ClientException("Error parsing JSON response from speech API", e); } var eventType = event.get("type").asText(); @@ -151,7 +158,7 @@ private void onText(WebSocket webSocket, CharSequence data) { webSocket.request(1); } - private synchronized void sendPing(WebSocket ws) { + private synchronized void sendPing(@Nonnull final WebSocket ws) { if (ws.isInputClosed()) { return; } @@ -159,20 +166,20 @@ private synchronized void sendPing(WebSocket ws) { ws.request(1); } - private void configureSession(WebSocket ws) { - SessionUpdateEvent sue = sessionConfiguration(); + private void configureSession(@Nonnull final WebSocket ws) { + final SessionUpdateEvent sue = sessionConfiguration(); try { ws.sendText(JACKSON.writeValueAsString(sue), true); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { throw new ClientException("Failed to serialize session request", e); } ws.request(1); } - private void configureConversation(WebSocket ws) { - var systemPrompt = getSystemPrompt(); - if (systemPrompt == null || systemPrompt.isEmpty()) { + private void configureConversation(@Nonnull final WebSocket ws) { + final var systemPrompt = getSystemPrompt(); + if (systemPrompt.isEmpty()) { return; } var systemConversationItem = @@ -191,13 +198,13 @@ private void configureConversation(WebSocket ws) { try { var json = JACKSON.writeValueAsString(systemConversationItem); ws.sendText(json, true); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { throw new ClientException("Failed to serialize message", e); } ws.request(1); } - private void scheduleHeartbeat(WebSocket ws) { + private void scheduleHeartbeat(@Nonnull final WebSocket ws) { TimerTask task = new TimerTask() { @Override From c6a0e949cd83b3807d66ffac2c8b2e60eb4c1dd0 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 10:39:14 +0000 Subject: [PATCH 42/67] Formatting --- .../openai/BufferedWebSocketListener.java | 16 +++++----------- .../openai/SpeechToSpeechRealtimeClient.java | 6 +++--- .../openai/TextToSpeechRealtimeClient.java | 5 ++--- .../openai/ToAudioRealtimeClient.java | 3 +-- .../openai/WSOpenAiRealtimeClient.java | 13 ++++++------- 5 files changed, 17 insertions(+), 26 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index b04318f8f..2916b0f83 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -6,9 +6,8 @@ import java.util.concurrent.CompletionStage; import java.util.function.BiConsumer; import java.util.function.Consumer; -import lombok.extern.slf4j.Slf4j; - import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; @Slf4j class BufferedWebSocketListener implements WebSocket.Listener { @@ -18,7 +17,8 @@ class BufferedWebSocketListener implements WebSocket.Listener { private final StringBuilder buffer; BufferedWebSocketListener( - @Nonnull final Consumer onOpen, @Nonnull final BiConsumer onText) { + @Nonnull final Consumer onOpen, + @Nonnull final BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; this.buffer = new StringBuilder(1024 * 1024); @@ -33,10 +33,7 @@ public void onOpen(@Nonnull final WebSocket webSocket) { @Override @Nonnull public CompletionStage onText( - final @Nonnull WebSocket webSocket, - final @Nonnull CharSequence data, - final boolean isLast - ) { + final @Nonnull WebSocket webSocket, final @Nonnull CharSequence data, final boolean isLast) { buffer.append(data); webSocket.request(1); if (isLast) { @@ -56,10 +53,7 @@ public void onError(@Nonnull final WebSocket webSocket, @Nonnull final Throwable @Override @Nonnull public CompletionStage onBinary( - @Nonnull final WebSocket webSocket, - @Nonnull final ByteBuffer data, - final boolean isLast - ) { + @Nonnull final WebSocket webSocket, @Nonnull final ByteBuffer data, final boolean isLast) { log.warn("Received unexpected binary bytes for WebSocket connection"); return CompletableFuture.completedStage(null); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 79ee89f3d..61baee6a9 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -10,9 +10,8 @@ import java.util.Arrays; import java.util.Base64; import java.util.Map; -import lombok.extern.slf4j.Slf4j; - import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; @Slf4j class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements AudioInputChannel { @@ -71,7 +70,8 @@ public void inputAudio(@Nonnull final byte[] rawAudioChunk) { } var cursorLeft = 0; while (cursorLeft < rawAudioChunk.length) { - final var cursorRight = Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); + final var cursorRight = + Math.min(cursorLeft + MAX_DATA_CHUNK_SIZE_BYTES, rawAudioChunk.length); final var part = Arrays.copyOfRange(rawAudioChunk, cursorLeft, cursorRight); final var audioInputMessage = InputAudioBufferAppendEvent.builder() diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 1fa0f4a3e..727da6853 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -3,11 +3,10 @@ import com.openai.models.realtime.*; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nonnull; import java.util.Map; import java.util.Optional; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; @Slf4j class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextInputChannel { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 93c958c82..dc1ab5f18 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -14,9 +14,8 @@ import java.util.List; import java.util.Map; import java.util.Set; -import lombok.extern.slf4j.Slf4j; - import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; @Slf4j abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index 9cc0d54a0..77f6b2ebb 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -21,12 +21,11 @@ import java.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nonnull; - @Slf4j @RequiredArgsConstructor(access = AccessLevel.PACKAGE) abstract class WSOpenAiRealtimeClient implements AutoCloseable { @@ -58,10 +57,9 @@ abstract class WSOpenAiRealtimeClient implements AutoCloseable { private final Set handleMessageTypes; public WSOpenAiRealtimeClient( - @Nonnull final String url, - @Nonnull final Map httpHeaders, - @Nonnull final Set handleMessageTypes - ) { + @Nonnull final String url, + @Nonnull final Map httpHeaders, + @Nonnull final Set handleMessageTypes) { this.client = HttpClient.newHttpClient(); var wsBuilder = this.client.newWebSocketBuilder(); for (Map.Entry entry : httpHeaders.entrySet()) { @@ -113,7 +111,8 @@ public void close() { @Nonnull protected abstract String getSystemPrompt(); - protected abstract void onResponse(@Nonnull final String eventType, @Nonnull final JsonNode event); + protected abstract void onResponse( + @Nonnull final String eventType, @Nonnull final JsonNode event); @Nonnull protected abstract SessionUpdateEvent sessionConfiguration(); From 1400efcde7bc641b155afdc10a59eb3d0b303a96 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 13:07:22 +0200 Subject: [PATCH 43/67] address code style pmd issues --- .../openai/BufferedWebSocketListener.java | 2 +- .../openai/OpenAiRealtimeClient.java | 5 +++-- .../openai/SpeechToSpeechRealtimeClient.java | 5 +++-- .../foundationmodels/openai/TextOutputChannel.java | 4 +++- .../openai/TextToSpeechRealtimeClient.java | 9 +++++++-- .../openai/ToAudioRealtimeClient.java | 5 +++-- .../openai/WSOpenAiRealtimeClient.java | 12 ++++++------ 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index b04318f8f..9fd0da1f6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -40,7 +40,7 @@ public CompletionStage onText( buffer.append(data); webSocket.request(1); if (isLast) { - var completeMessage = buffer.toString(); + final var completeMessage = buffer.toString(); buffer.setLength(0); this.onText.accept(webSocket, completeMessage); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index 355469ec0..b63e586b6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -52,7 +52,8 @@ public class OpenAiRealtimeClient { @Nonnull @Beta public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params + ) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -93,7 +94,7 @@ public TextInputChannel textToSpeech( @Nonnull @Beta public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, RealtimeParam... params) { + @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 79ee89f3d..1e6af4f76 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -27,11 +27,12 @@ public SpeechToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - RealtimeParam... params) { + @Nonnull final RealtimeParam... params + ) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = false; - for (RealtimeParam param : params) { + for (final RealtimeParam param : params) { if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index add9c2460..b403097e8 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,5 +1,7 @@ package com.sap.ai.sdk.foundationmodels.openai; +import javax.annotation.Nonnull; + /** Functional interface representing text output channel (text data consumer) */ public interface TextOutputChannel { @@ -12,5 +14,5 @@ public interface TextOutputChannel { * single logical entity (e.g. gets called at the end when all parts of a single message get * passed) */ - void outputText(CharSequence textChunk, boolean isLast); + void outputText(@Nonnull final CharSequence textChunk, final boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index 1fa0f4a3e..451ff5f04 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -1,6 +1,10 @@ package com.sap.ai.sdk.foundationmodels.openai; -import com.openai.models.realtime.*; +import com.openai.models.realtime.ConversationItem; +import com.openai.models.realtime.ConversationItemCreateEvent; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeConversationItemUserMessage; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; import lombok.extern.slf4j.Slf4j; @@ -22,7 +26,8 @@ public TextToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - RealtimeParam... params) { + @Nonnull final RealtimeParam... params + ) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index 93c958c82..7bdfc1c02 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -34,10 +34,11 @@ public ToAudioRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - RealtimeParam... params) { + @Nonnull final RealtimeParam... params + ) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - for (RealtimeParam param : params) { + for (final RealtimeParam param : params) { switch (param.getParamName()) { case VOICE -> { if (RealtimeParamVoice.DEFAULT_2.equals(param)) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java index 9cc0d54a0..77ab383e3 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java @@ -64,7 +64,7 @@ public WSOpenAiRealtimeClient( ) { this.client = HttpClient.newHttpClient(); var wsBuilder = this.client.newWebSocketBuilder(); - for (Map.Entry entry : httpHeaders.entrySet()) { + for (final Map.Entry entry : httpHeaders.entrySet()) { wsBuilder = wsBuilder.header(entry.getKey(), entry.getValue()); } this.ws = @@ -85,7 +85,7 @@ public void askForResponse() { try { ws.sendText(JACKSON.writeValueAsString(ResponseCreateEvent.builder().build()), true); } catch (final JsonProcessingException e) { - throw new RuntimeException("Failed to serialize ask for response", e); + throw new ClientException("Failed to serialize ask for response", e); } ws.request(1); } @@ -148,7 +148,7 @@ private void onText(@Nonnull final WebSocket webSocket, @Nonnull final CharSeque } catch (final JsonProcessingException e) { throw new ClientException("Error parsing JSON response from speech API", e); } - var eventType = event.get("type").asText(); + final var eventType = event.get("type").asText(); if (handleMessageTypes.contains(eventType)) { onResponse(eventType, event); } else { @@ -182,7 +182,7 @@ private void configureConversation(@Nonnull final WebSocket ws) { if (systemPrompt.isEmpty()) { return; } - var systemConversationItem = + final var systemConversationItem = ConversationItemCreateEvent.builder() .item( ConversationItem.ofRealtimeConversationItemSystemMessage( @@ -196,7 +196,7 @@ private void configureConversation(@Nonnull final WebSocket ws) { .build(); try { - var json = JACKSON.writeValueAsString(systemConversationItem); + final var json = JACKSON.writeValueAsString(systemConversationItem); ws.sendText(json, true); } catch (final JsonProcessingException e) { throw new ClientException("Failed to serialize message", e); @@ -205,7 +205,7 @@ private void configureConversation(@Nonnull final WebSocket ws) { } private void scheduleHeartbeat(@Nonnull final WebSocket ws) { - TimerTask task = + final TimerTask task = new TimerTask() { @Override public void run() { From 51b35a0ca192377314969d0a67e58ab08bc174a4 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 11:08:40 +0000 Subject: [PATCH 44/67] Formatting --- .../sdk/foundationmodels/openai/OpenAiRealtimeClient.java | 7 ++++--- .../openai/SpeechToSpeechRealtimeClient.java | 3 +-- .../openai/TextToSpeechRealtimeClient.java | 3 +-- .../sdk/foundationmodels/openai/ToAudioRealtimeClient.java | 3 +-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java index b63e586b6..4ac272e00 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java @@ -52,8 +52,8 @@ public class OpenAiRealtimeClient { @Nonnull @Beta public TextInputChannel textToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params - ) { + @Nonnull final AudioOutputChannel audioOutputConsumer, + @Nonnull final RealtimeParam... params) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { @@ -94,7 +94,8 @@ public TextInputChannel textToSpeech( @Nonnull @Beta public AudioInputChannel speechToSpeech( - @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params) { + @Nonnull final AudioOutputChannel audioOutputConsumer, + @Nonnull final RealtimeParam... params) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (Header header : extraHeaders) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java index 12cf3ea1c..27a5ea618 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java @@ -26,8 +26,7 @@ public SpeechToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - @Nonnull final RealtimeParam... params - ) { + @Nonnull final RealtimeParam... params) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = false; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java index dab25cbec..8a3286c47 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java @@ -25,8 +25,7 @@ public TextToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - @Nonnull final RealtimeParam... params - ) { + @Nonnull final RealtimeParam... params) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; for (RealtimeParam param : params) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java index ddf82af62..b196cf59e 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java @@ -33,8 +33,7 @@ public ToAudioRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, - @Nonnull final RealtimeParam... params - ) { + @Nonnull final RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (final RealtimeParam param : params) { From f92d32718bec789191d70694a2c32e631b337592 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 15:35:34 +0200 Subject: [PATCH 45/67] reduce default response buffer size --- .../sdk/foundationmodels/openai/BufferedWebSocketListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java index 2c6d3b475..cc56903e4 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java @@ -21,7 +21,7 @@ class BufferedWebSocketListener implements WebSocket.Listener { @Nonnull final BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; - this.buffer = new StringBuilder(1024 * 1024); + this.buffer = new StringBuilder(128 * 1024); } @Override From ff3007b80f45a53bfbfb1649ea5c25c1fe63e8c8 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 15:44:26 +0200 Subject: [PATCH 46/67] move realtime api classed to a dedicated package --- .../ai/sdk/foundationmodels/openai/OpenAiClient.java | 1 + .../{ => realtime}/BufferedWebSocketListener.java | 2 +- .../openai/{ => realtime}/OpenAiRealtimeClient.java | 11 +++++++++-- .../{ => realtime}/SpeechToSpeechRealtimeClient.java | 5 ++++- .../{ => realtime}/TextToSpeechRealtimeClient.java | 5 ++++- .../openai/{ => realtime}/ToAudioRealtimeClient.java | 4 +++- .../openai/{ => realtime}/WSOpenAiRealtimeClient.java | 3 ++- 7 files changed, 24 insertions(+), 7 deletions(-) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/BufferedWebSocketListener.java (96%) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/OpenAiRealtimeClient.java (91%) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/SpeechToSpeechRealtimeClient.java (94%) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/TextToSpeechRealtimeClient.java (93%) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/ToAudioRealtimeClient.java (96%) rename foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/{ => realtime}/WSOpenAiRealtimeClient.java (99%) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java index 691aee48a..bd3b4176a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java @@ -26,6 +26,7 @@ import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage; import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput; import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters; +import com.sap.ai.sdk.foundationmodels.openai.realtime.OpenAiRealtimeClient; import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor; import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination; import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java similarity index 96% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java index cc56903e4..82d59429a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import java.net.http.WebSocket; import java.nio.ByteBuffer; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java similarity index 91% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index 4ac272e00..cd21fb052 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -1,7 +1,10 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import java.util.HashMap; @@ -18,7 +21,11 @@ public class OpenAiRealtimeClient { private final Destination destination; - OpenAiRealtimeClient(@Nonnull final Destination destination) { + /** + * Created OpenAI Realtime client for a specific destination + * @param destination - destination to use + */ + public OpenAiRealtimeClient(@Nonnull final Destination destination) { this.destination = destination; } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java similarity index 94% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java index 27a5ea618..16519fac6 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.openai.models.realtime.InputAudioBufferAppendEvent; import com.openai.models.realtime.InputAudioBufferCommitEvent; @@ -11,6 +11,9 @@ import java.util.Base64; import java.util.Map; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java similarity index 93% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 8a3286c47..0cc8aed05 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.openai.models.realtime.ConversationItem; import com.openai.models.realtime.ConversationItemCreateEvent; @@ -10,6 +10,9 @@ import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java similarity index 96% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index b196cf59e..c4a5d1d5d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.fasterxml.jackson.databind.JsonNode; import com.openai.models.realtime.RealtimeAudioConfig; @@ -15,6 +15,8 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; + +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java similarity index 99% rename from foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index d24084e77..7f2730462 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.foundationmodels.openai; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; @@ -22,6 +22,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import javax.annotation.Nonnull; + import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 2060668976571c335e44db8dd6558ea0371c8674 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 13:45:27 +0000 Subject: [PATCH 47/67] Formatting --- .../openai/realtime/OpenAiRealtimeClient.java | 1 + .../openai/realtime/SpeechToSpeechRealtimeClient.java | 5 ++--- .../openai/realtime/TextToSpeechRealtimeClient.java | 5 ++--- .../openai/realtime/ToAudioRealtimeClient.java | 3 +-- .../openai/realtime/WSOpenAiRealtimeClient.java | 1 - 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index cd21fb052..a226f6e6b 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -23,6 +23,7 @@ public class OpenAiRealtimeClient { /** * Created OpenAI Realtime client for a specific destination + * * @param destination - destination to use */ public OpenAiRealtimeClient(@Nonnull final Destination destination) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java index 16519fac6..4cf86d509 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java @@ -7,13 +7,12 @@ import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Arrays; import java.util.Base64; import java.util.Map; import javax.annotation.Nonnull; - -import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 0cc8aed05..dc2ab4fbb 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -7,12 +7,11 @@ import com.openai.models.realtime.RealtimeConversationItemUserMessage; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; - -import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; -import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index c4a5d1d5d..93e6ee35e 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -10,13 +10,12 @@ import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.core.RealtimeParam; import com.sap.ai.sdk.core.RealtimeParamVoice; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; - -import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index 7f2730462..13b4f5609 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -22,7 +22,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import javax.annotation.Nonnull; - import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; From 1e74df5c54613eb87865883e140f158f79513ead Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 16:30:11 +0200 Subject: [PATCH 48/67] fix pmd issues --- .../openai/realtime/OpenAiRealtimeClient.java | 4 ++-- .../openai/realtime/TextToSpeechRealtimeClient.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index cd21fb052..2770e782a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -63,7 +63,7 @@ public TextInputChannel textToSpeech( @Nonnull final RealtimeParam... params) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { + for (final Header header : extraHeaders) { headers.put(header.getName(), header.getValue()); } final var endpoint = getRealtimeEndpoint(); @@ -105,7 +105,7 @@ public AudioInputChannel speechToSpeech( @Nonnull final RealtimeParam... params) { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); - for (Header header : extraHeaders) { + for (final Header header : extraHeaders) { headers.put(header.getName(), header.getValue()); } final var endpoint = getRealtimeEndpoint(); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 0cc8aed05..26d6c2287 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -31,7 +31,7 @@ public TextToSpeechRealtimeClient( @Nonnull final RealtimeParam... params) { super(url, httpHeaders, outputConsumer, params); var turnDetectionEager = true; - for (RealtimeParam param : params) { + for (final RealtimeParam param : params) { if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { turnDetectionEager = true; From 0fb2646ef543d966425c543fb02b92a10323aa46 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 16:41:05 +0200 Subject: [PATCH 49/67] address spotbugs plugin issues --- .../openai/realtime/ToAudioRealtimeClient.java | 14 ++++++-------- .../openai/realtime/WSOpenAiRealtimeClient.java | 3 ++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 93e6ee35e..5f918a9bf 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -38,15 +38,13 @@ public ToAudioRealtimeClient( super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (final RealtimeParam param : params) { - switch (param.getParamName()) { - case VOICE -> { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.VOICE) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } } - } } this.outputConsumer = outputConsumer; this.voice = voice; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index 13b4f5609..afd8b134f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -15,6 +15,7 @@ import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Set; import java.util.Timer; @@ -161,7 +162,7 @@ private synchronized void sendPing(@Nonnull final WebSocket ws) { if (ws.isInputClosed()) { return; } - ws.sendPing(ByteBuffer.wrap("ping".getBytes())).join(); + ws.sendPing(ByteBuffer.wrap("ping".getBytes(StandardCharsets.UTF_8))).join(); ws.request(1); } From 54b151d6f1e2b8f6f9a2e3fd535b677c18ead89e Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 14:42:00 +0000 Subject: [PATCH 50/67] Formatting --- .../openai/realtime/ToAudioRealtimeClient.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 5f918a9bf..3decb6cd2 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -38,13 +38,13 @@ public ToAudioRealtimeClient( super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.VOICE) { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } + if (param.getParamName() == RealtimeParam.SpeechOutputParamName.VOICE) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; } + } } this.outputConsumer = outputConsumer; this.voice = voice; From f6698c55966d2a9cadb3d9d76db9442117203174 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 17:01:09 +0200 Subject: [PATCH 51/67] address dependency issue --- sample-code/spring-app/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index 75f32a353..ac97990da 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -140,8 +140,8 @@ spring-ai-commons - org.springframework.boot - spring-boot-starter-websocket + org.springframework + spring-websocket org.springframework.ai From 40971d49b955e568e4be533e525679203083bcad Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 17:15:15 +0200 Subject: [PATCH 52/67] fix checkstyle issues --- .../src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java | 7 +++++++ .../sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 8 ++++++++ .../ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 9273342c8..780c3da58 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -7,6 +7,9 @@ import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +/** + * Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API + */ @Configuration @EnableWebSocket public class WebsocketConfig implements WebSocketConfigurer { @@ -20,6 +23,10 @@ public WebsocketConfig( this.speechToSpeech = speechToSpeech; } + /** + * Registers websocket handlers, implements WebSocketConfigurer contract + * @param registry - registry where to register handlers + */ @Override public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech"); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 64911e08b..7eee2b04e 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -15,6 +15,9 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +/** + * Implements handler (Web Socket messages handling) for speech to speech realtime api operation + */ @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -22,6 +25,11 @@ public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; private final Map channels; + /** + * Constructs handler object + * + * @param service - handling service + */ @Autowired public SpeechToSpeechWebsocketHandler(OpenAiService service) { this.service = service; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 88ce6109b..6b91b41aa 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -15,6 +15,9 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; +/** + * Implements handler (Web Socket messages handling) for text to speech realtime api operation + */ @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -22,6 +25,11 @@ public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; private final Map channels; + /** + * Constructs handler object + * + * @param service - handling service + */ @Autowired public TextToSpeechWebsocketHandler(OpenAiService service) { this.service = service; From 7f656aaa20b96b9701af6e1aa31af71b7d716d6d Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 15:16:13 +0000 Subject: [PATCH 53/67] Formatting --- .../java/com/sap/ai/sdk/app/WebsocketConfig.java | 5 ++--- .../realtime/SpeechToSpeechWebsocketHandler.java | 14 ++++++-------- .../app/realtime/TextToSpeechWebsocketHandler.java | 14 ++++++-------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 780c3da58..0d24f52f8 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -7,9 +7,7 @@ import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; -/** - * Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API - */ +/** Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API */ @Configuration @EnableWebSocket public class WebsocketConfig implements WebSocketConfigurer { @@ -25,6 +23,7 @@ public WebsocketConfig( /** * Registers websocket handlers, implements WebSocketConfigurer contract + * * @param registry - registry where to register handlers */ @Override diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 7eee2b04e..86fe5cd2b 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -15,9 +15,7 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -/** - * Implements handler (Web Socket messages handling) for speech to speech realtime api operation - */ +/** Implements handler (Web Socket messages handling) for speech to speech realtime api operation */ @Component @Slf4j public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -25,11 +23,11 @@ public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; private final Map channels; - /** - * Constructs handler object - * - * @param service - handling service - */ + /** + * Constructs handler object + * + * @param service - handling service + */ @Autowired public SpeechToSpeechWebsocketHandler(OpenAiService service) { this.service = service; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 6b91b41aa..5fb96d5cf 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -15,9 +15,7 @@ import org.springframework.web.socket.WebSocketSession; import org.springframework.web.socket.handler.BinaryWebSocketHandler; -/** - * Implements handler (Web Socket messages handling) for text to speech realtime api operation - */ +/** Implements handler (Web Socket messages handling) for text to speech realtime api operation */ @Component @Slf4j public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { @@ -25,11 +23,11 @@ public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; private final Map channels; - /** - * Constructs handler object - * - * @param service - handling service - */ + /** + * Constructs handler object + * + * @param service - handling service + */ @Autowired public TextToSpeechWebsocketHandler(OpenAiService service) { this.service = service; From 7eed889e8d2f4d5f14393b2dd9ea605658db6743 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 17:21:53 +0200 Subject: [PATCH 54/67] fix checkstyle issues --- .../src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 780c3da58..761bc0b66 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -17,6 +17,11 @@ public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; private final SpeechToSpeechWebsocketHandler speechToSpeech; + /** + * Constructs configuration object + * @param textToSpeech - text to speech realtime api handler + * @param speechToSpeech - speech to speech realtime api handler + */ public WebsocketConfig( TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { this.textToSpeech = textToSpeech; From a3d4641ccf1dee6ffea7b5dfe62961b5b6f65d7e Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 15:23:06 +0000 Subject: [PATCH 55/67] Formatting --- .../src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index ae7233798..1c32e53cc 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -17,6 +17,7 @@ public class WebsocketConfig implements WebSocketConfigurer { /** * Constructs configuration object + * * @param textToSpeech - text to speech realtime api handler * @param speechToSpeech - speech to speech realtime api handler */ From bd8cb90e83f964e5578036550159435552d60399 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 18:37:01 +0200 Subject: [PATCH 56/67] fix checkstyle issues --- .../com/sap/ai/sdk/app/WebsocketConfig.java | 8 ++++-- .../SpeechToSpeechWebsocketHandler.java | 25 ++++++++++++------- .../TextToSpeechWebsocketHandler.java | 21 ++++++++++------ .../ai/sdk/app/services/OpenAiService.java | 1 + 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index ae7233798..4c496bfae 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -7,6 +7,8 @@ import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; +import javax.annotation.Nonnull; + /** Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API */ @Configuration @EnableWebSocket @@ -21,7 +23,9 @@ public class WebsocketConfig implements WebSocketConfigurer { * @param speechToSpeech - speech to speech realtime api handler */ public WebsocketConfig( - TextToSpeechWebsocketHandler textToSpeech, SpeechToSpeechWebsocketHandler speechToSpeech) { + @Nonnull final TextToSpeechWebsocketHandler textToSpeech, + @Nonnull final SpeechToSpeechWebsocketHandler speechToSpeech + ) { this.textToSpeech = textToSpeech; this.speechToSpeech = speechToSpeech; } @@ -32,7 +36,7 @@ public WebsocketConfig( * @param registry - registry where to register handlers */ @Override - public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + public void registerWebSocketHandlers(@Nonnull final WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech"); registry.addHandler(speechToSpeech, "/speech-to-speech"); } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 86fe5cd2b..8eaf9cdf2 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -29,16 +29,22 @@ public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired - public SpeechToSpeechWebsocketHandler(OpenAiService service) { + public SpeechToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); } @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] chunkBytes = payload.array(); - AudioInputChannel channel = + // The channel MUST NOT be closed here, its lifecycle is managed by the WebSocket container (RAII) + // closing performed in afterConnectionClosed method + @SuppressWarnings("PMD.CloseResource") + protected void handleBinaryMessage( + @Nonnull final WebSocketSession session, + @Nonnull final BinaryMessage message + ) { + final ByteBuffer payload = message.getPayload(); + final byte[] chunkBytes = payload.array(); + final AudioInputChannel channel = channels.computeIfAbsent( session.getId(), sessionId -> @@ -46,7 +52,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa (rawBytesChunk, isLast) -> { try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (final IOException e) { log.error("failed to send audio data to realtime api", e); } })); @@ -54,14 +60,15 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) - throws Exception { + public void afterConnectionClosed(@Nonnull final WebSocketSession session, + @Nonnull final CloseStatus status + ) throws Exception { channels.computeIfPresent( session.getId(), (sessionId, inputChannel) -> { try { inputChannel.close(); - } catch (Exception e) { + } catch (final Exception e) { log.warn("failed to close input channel for session {}", sessionId, e); } return null; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 5fb96d5cf..ebc67ffed 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -29,16 +29,21 @@ public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired - public TextToSpeechWebsocketHandler(OpenAiService service) { + public TextToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); } @Override - protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { - ByteBuffer payload = message.getPayload(); - byte[] textBytes = payload.array(); - TextInputChannel channel = + // The channel MUST NOT be closed here, its lifecycle is managed by the WebSocket container (RAII) + // closing performed in afterConnectionClosed method + @SuppressWarnings("PMD.CloseResource") + protected void handleBinaryMessage(@Nonnull final WebSocketSession session, + @Nonnull final BinaryMessage message + ) { + final ByteBuffer payload = message.getPayload(); + final byte[] textBytes = payload.array(); + final TextInputChannel channel = channels.computeIfAbsent( session.getId(), sessionId -> @@ -46,7 +51,7 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa (rawBytesChunk, isLast) -> { try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); - } catch (IOException e) { + } catch (final IOException e) { log.error("failed to send text message to realtime api", e); } })); @@ -54,7 +59,9 @@ protected void handleBinaryMessage(WebSocketSession session, BinaryMessage messa } @Override - public void afterConnectionClosed(WebSocketSession session, @Nonnull CloseStatus status) + public void afterConnectionClosed(@Nonnull final WebSocketSession session, + @Nonnull final CloseStatus status + ) throws Exception { channels.computeIfPresent( session.getId(), diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index e8c83a5f8..dbe16d8db 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -132,6 +132,7 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 * bit) */ + @Nonnull public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); } From 7e008bb007f6ffe1994841f87c27a233b22b1f04 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 24 Jul 2026 16:38:17 +0000 Subject: [PATCH 57/67] Formatting --- .../main/java/com/sap/ai/sdk/app/WebsocketConfig.java | 8 +++----- .../app/realtime/SpeechToSpeechWebsocketHandler.java | 9 +++------ .../app/realtime/TextToSpeechWebsocketHandler.java | 11 ++++------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 105098453..0b8a56d22 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -2,13 +2,12 @@ import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; +import javax.annotation.Nonnull; import org.springframework.context.annotation.Configuration; import org.springframework.web.socket.config.annotation.EnableWebSocket; import org.springframework.web.socket.config.annotation.WebSocketConfigurer; import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; -import javax.annotation.Nonnull; - /** Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API */ @Configuration @EnableWebSocket @@ -24,9 +23,8 @@ public class WebsocketConfig implements WebSocketConfigurer { * @param speechToSpeech - speech to speech realtime api handler */ public WebsocketConfig( - @Nonnull final TextToSpeechWebsocketHandler textToSpeech, - @Nonnull final SpeechToSpeechWebsocketHandler speechToSpeech - ) { + @Nonnull final TextToSpeechWebsocketHandler textToSpeech, + @Nonnull final SpeechToSpeechWebsocketHandler speechToSpeech) { this.textToSpeech = textToSpeech; this.speechToSpeech = speechToSpeech; } diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 8eaf9cdf2..956ad5cd5 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -39,9 +39,7 @@ public SpeechToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { // closing performed in afterConnectionClosed method @SuppressWarnings("PMD.CloseResource") protected void handleBinaryMessage( - @Nonnull final WebSocketSession session, - @Nonnull final BinaryMessage message - ) { + @Nonnull final WebSocketSession session, @Nonnull final BinaryMessage message) { final ByteBuffer payload = message.getPayload(); final byte[] chunkBytes = payload.array(); final AudioInputChannel channel = @@ -60,9 +58,8 @@ protected void handleBinaryMessage( } @Override - public void afterConnectionClosed(@Nonnull final WebSocketSession session, - @Nonnull final CloseStatus status - ) throws Exception { + public void afterConnectionClosed( + @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), (sessionId, inputChannel) -> { diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index ebc67ffed..5c3d8f423 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -38,9 +38,8 @@ public TextToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { // The channel MUST NOT be closed here, its lifecycle is managed by the WebSocket container (RAII) // closing performed in afterConnectionClosed method @SuppressWarnings("PMD.CloseResource") - protected void handleBinaryMessage(@Nonnull final WebSocketSession session, - @Nonnull final BinaryMessage message - ) { + protected void handleBinaryMessage( + @Nonnull final WebSocketSession session, @Nonnull final BinaryMessage message) { final ByteBuffer payload = message.getPayload(); final byte[] textBytes = payload.array(); final TextInputChannel channel = @@ -59,10 +58,8 @@ protected void handleBinaryMessage(@Nonnull final WebSocketSession session, } @Override - public void afterConnectionClosed(@Nonnull final WebSocketSession session, - @Nonnull final CloseStatus status - ) - throws Exception { + public void afterConnectionClosed( + @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent( session.getId(), (sessionId, inputChannel) -> { From b36ed95c8d9d850ece6af0181ae3047d8be3da1f Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 24 Jul 2026 18:44:37 +0200 Subject: [PATCH 58/67] fix checkstyle issues --- .../sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index ebc67ffed..97eb6d1ff 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -4,6 +4,7 @@ import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; @@ -55,7 +56,7 @@ protected void handleBinaryMessage(@Nonnull final WebSocketSession session, log.error("failed to send text message to realtime api", e); } })); - channel.sendText(new String(textBytes)); + channel.sendText(new String(textBytes, StandardCharsets.UTF_8)); } @Override From 7f2387027dab13ec7cfa7f2af5312c1b4ea0c8f9 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 27 Jul 2026 15:40:47 +0200 Subject: [PATCH 59/67] fix test coverage issues --- .../{ => client/realtime}/RealtimeParam.java | 2 +- .../realtime}/RealtimeParamTurnDetection.java | 4 +- .../realtime}/RealtimeParamVoice.java | 4 +- .../RealtimeParamTurnDetectionUnitTest.java | 59 +++++++++++++++++++ .../realtime/RealtimeParamVoiceUnitTest.java | 59 +++++++++++++++++++ .../openai/realtime/OpenAiRealtimeClient.java | 2 +- .../SpeechToSpeechRealtimeClient.java | 4 +- .../realtime/TextToSpeechRealtimeClient.java | 4 +- .../realtime/ToAudioRealtimeClient.java | 4 +- .../realtime/OpenAiRealtimeClientTest.java | 37 ++++++++++++ .../sdk/app/controllers/RealtimeApiTest.java | 2 +- 11 files changed, 168 insertions(+), 13 deletions(-) rename core/src/main/java/com/sap/ai/sdk/core/{ => client/realtime}/RealtimeParam.java (93%) rename core/src/main/java/com/sap/ai/sdk/core/{ => client/realtime}/RealtimeParamTurnDetection.java (93%) rename core/src/main/java/com/sap/ai/sdk/core/{ => client/realtime}/RealtimeParamVoice.java (93%) create mode 100644 core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java create mode 100644 core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java similarity index 93% rename from core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java rename to core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java index ae31c9739..e614132f1 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParam.java +++ b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core; +package com.sap.ai.sdk.core.client.realtime; import javax.annotation.Nonnull; diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java similarity index 93% rename from core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java rename to core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java index a0d82a3ec..8eacfb80f 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamTurnDetection.java +++ b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core; +package com.sap.ai.sdk.core.client.realtime; import java.util.Objects; import javax.annotation.Nonnull; @@ -21,7 +21,7 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { private final String turnDetectionKind; - private RealtimeParamTurnDetection(final String turnDetectionKind) { + RealtimeParamTurnDetection(final String turnDetectionKind) { this.turnDetectionKind = turnDetectionKind; } diff --git a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java similarity index 93% rename from core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java rename to core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java index c16e675d6..6d45dd4f4 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/RealtimeParamVoice.java +++ b/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core; +package com.sap.ai.sdk.core.client.realtime; import java.util.Objects; import javax.annotation.Nonnull; @@ -15,7 +15,7 @@ public final class RealtimeParamVoice implements RealtimeParam { private final String voice; - private RealtimeParamVoice(@Nonnull final String voice) { + RealtimeParamVoice(@Nonnull final String voice) { this.voice = voice; } diff --git a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java b/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java new file mode 100644 index 000000000..adc8ad835 --- /dev/null +++ b/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java @@ -0,0 +1,59 @@ +package com.sap.ai.sdk.core.client.realtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RealtimeParamTurnDetectionUnitTest { + + private RealtimeParamTurnDetection random; + private String expectedRandomValue; + + @BeforeEach + void setUp() { + expectedRandomValue = UUID.randomUUID().toString().substring(0, 20); + random = new RealtimeParamTurnDetection(expectedRandomValue); + } + + @Test + void getParamName() { + assertThat(random.getParamName()).isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + assertThat(RealtimeParamTurnDetection.BY_MODEL_AUTO.getParamName()) + .isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.getParamName()) + .isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + } + + @Test + void getValueAsString() { + assertThat(random.getValueAsString()).isEqualTo(expectedRandomValue); + assertThat(RealtimeParamTurnDetection.BY_MODEL_AUTO.getValueAsString()) + .isEqualTo("BY_MODEL_AUTO"); + assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.getValueAsString()) + .isEqualTo("EACH_CALL_IS_A_TURN"); + } + + @Test + void testEquals() { + assertThat(random).isEqualTo(random); + assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN) + .isEqualTo(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN); + assertThat(RealtimeParamTurnDetection.BY_MODEL_AUTO) + .isEqualTo(RealtimeParamTurnDetection.BY_MODEL_AUTO); + assertThat(random).isNotEqualTo(RealtimeParamTurnDetection.BY_MODEL_AUTO); + assertThat(random).isNotEqualTo(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN); + assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN) + .isNotEqualTo(RealtimeParamTurnDetection.BY_MODEL_AUTO); + } + + @Test + void testHashCode() { + assertThat(random.hashCode()).isEqualTo(random.hashCode()); + assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.hashCode()) + .isEqualTo(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.hashCode()); + assertThat(RealtimeParamTurnDetection.BY_MODEL_AUTO.hashCode()) + .isEqualTo(RealtimeParamTurnDetection.BY_MODEL_AUTO.hashCode()); + } +} diff --git a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java b/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java new file mode 100644 index 000000000..06b516708 --- /dev/null +++ b/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java @@ -0,0 +1,59 @@ +package com.sap.ai.sdk.core.client.realtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RealtimeParamVoiceUnitTest { + + private RealtimeParamVoice random; + private String expectedRandomValue; + + @BeforeEach + void setUp() { + expectedRandomValue = UUID.randomUUID().toString().substring(0, 20); + random = new RealtimeParamVoice(expectedRandomValue); + } + + @Test + void unsafeWithExplicitVoice() { + assertThat(random).isEqualTo(RealtimeParamVoice.unsafeWithExplicitVoice(expectedRandomValue)); + } + + @Test + void getParamName() { + assertThat(random.getParamName()).isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + assertThat(RealtimeParamVoice.DEFAULT_1.getParamName()) + .isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + assertThat(RealtimeParamVoice.DEFAULT_1.getParamName()) + .isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + } + + @Test + void getValueAsString() { + assertThat(random.getValueAsString()).isEqualTo(expectedRandomValue); + assertThat(RealtimeParamVoice.DEFAULT_1.getValueAsString()).isEqualTo("DEFAULT_1"); + assertThat(RealtimeParamVoice.DEFAULT_2.getValueAsString()).isEqualTo("DEFAULT_2"); + } + + @Test + void testEquals() { + assertThat(random).isEqualTo(random); + assertThat(RealtimeParamVoice.DEFAULT_1).isEqualTo(RealtimeParamVoice.DEFAULT_1); + assertThat(RealtimeParamVoice.DEFAULT_2).isEqualTo(RealtimeParamVoice.DEFAULT_2); + assertThat(random).isNotEqualTo(RealtimeParamVoice.DEFAULT_1); + assertThat(random).isNotEqualTo(RealtimeParamVoice.DEFAULT_2); + assertThat(RealtimeParamVoice.DEFAULT_1).isNotEqualTo(RealtimeParamVoice.DEFAULT_2); + } + + @Test + void testHashCode() { + assertThat(random.hashCode()).isEqualTo(random.hashCode()); + assertThat(RealtimeParamVoice.DEFAULT_1.hashCode()) + .isEqualTo(RealtimeParamVoice.DEFAULT_1.hashCode()); + assertThat(RealtimeParamVoice.DEFAULT_2.hashCode()) + .isEqualTo(RealtimeParamVoice.DEFAULT_2.hashCode()); + } +} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index 96678c2d5..a3be0f884 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -1,7 +1,7 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; -import com.sap.ai.sdk.core.RealtimeParam; +import com.sap.ai.sdk.core.client.realtime.RealtimeParam; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java index 4cf86d509..0ff6bccfb 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java @@ -5,8 +5,8 @@ import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioFormats; import com.openai.models.realtime.RealtimeAudioInputTurnDetection; -import com.sap.ai.sdk.core.RealtimeParam; -import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.client.realtime.RealtimeParam; +import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Arrays; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 6fd355469..159a86285 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -5,8 +5,8 @@ import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioFormats; import com.openai.models.realtime.RealtimeConversationItemUserMessage; -import com.sap.ai.sdk.core.RealtimeParam; -import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.client.realtime.RealtimeParam; +import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.Map; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 3decb6cd2..28f1de460 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -8,8 +8,8 @@ import com.openai.models.realtime.RealtimeSessionCreateRequest; import com.openai.models.realtime.SessionUpdateEvent; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.RealtimeParam; -import com.sap.ai.sdk.core.RealtimeParamVoice; +import com.sap.ai.sdk.core.client.realtime.RealtimeParam; +import com.sap.ai.sdk.core.client.realtime.RealtimeParamVoice; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Base64; import java.util.List; diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java new file mode 100644 index 000000000..2f24dc75c --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java @@ -0,0 +1,37 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class OpenAiRealtimeClientTest { + + private OpenAiRealtimeClient client; + + @BeforeEach + void setUp() { + client = OpenAiClient.realtimeClient(); + } + + @Test + void textToSpeech() { + try (TextInputChannel ic = client.textToSpeech((a, b) -> {})) { + assertThat(ic).isNotNull(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + void speechToSpeech() { + try (AudioInputChannel ic = client.speechToSpeech((a, b) -> {})) { + assertThat(ic).isNotNull(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index 10f6027e4..cf8ad26ef 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -3,7 +3,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import com.sap.ai.sdk.core.RealtimeParamTurnDetection; +import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; From 07ea836595cdad74461df026b1ee9ee1b83baa03 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 27 Jul 2026 20:18:54 +0200 Subject: [PATCH 60/67] address pr feedback, fix linter issues --- core-services/batch/pom.xml | 2 +- core-services/document-grounding/pom.xml | 2 +- core-services/prompt-registry/pom.xml | 2 +- core/pom.xml | 2 +- foundation-models/openai/pom.xml | 3 +- .../openai/AudioInputChannel.java | 4 ++ .../openai/AudioOutputChannel.java | 4 ++ .../openai/TextInputChannel.java | 3 ++ .../openai/TextOutputChannel.java | 3 ++ .../openai/realtime/OpenAiRealtimeClient.java | 21 ++++---- .../openai}/realtime/RealtimeParam.java | 10 ++-- .../realtime/RealtimeParamSystemPrompt.java | 44 ++++++++++++++++ .../realtime/RealtimeParamTurnDetection.java | 6 +-- .../openai}/realtime/RealtimeParamVoice.java | 6 +-- .../SpeechToSpeechRealtimeClient.java | 26 +--------- .../realtime/TextToSpeechRealtimeClient.java | 34 ++++-------- .../realtime/ToAudioRealtimeClient.java | 52 +++++++++++++++++-- .../realtime/WSOpenAiRealtimeClient.java | 16 +++--- .../RealtimeParamSystemPromptUnitTest.java | 48 +++++++++++++++++ .../RealtimeParamTurnDetectionUnitTest.java | 8 +-- .../realtime/RealtimeParamVoiceUnitTest.java | 8 +-- foundation-models/sap-rpt/pom.xml | 2 +- orchestration/pom.xml | 2 +- pom.xml | 2 +- sample-code/spring-app/pom.xml | 2 +- .../SpeechToSpeechWebsocketHandler.java | 4 +- .../TextToSpeechWebsocketHandler.java | 4 +- .../src/main/resources/static/index.html | 4 +- .../sdk/app/controllers/RealtimeApiTest.java | 2 +- 29 files changed, 223 insertions(+), 103 deletions(-) rename {core/src/main/java/com/sap/ai/sdk/core/client => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/realtime/RealtimeParam.java (74%) create mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java rename {core/src/main/java/com/sap/ai/sdk/core/client => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/realtime/RealtimeParamTurnDetection.java (90%) rename {core/src/main/java/com/sap/ai/sdk/core/client => foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai}/realtime/RealtimeParamVoice.java (91%) create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPromptUnitTest.java rename {core/src/test/java/com/sap/ai/sdk/core/client => foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai}/realtime/RealtimeParamTurnDetectionUnitTest.java (87%) rename {core/src/test/java/com/sap/ai/sdk/core/client => foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai}/realtime/RealtimeParamVoiceUnitTest.java (87%) diff --git a/core-services/batch/pom.xml b/core-services/batch/pom.xml index cac9c9a9b..2dd14a064 100644 --- a/core-services/batch/pom.xml +++ b/core-services/batch/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml batch diff --git a/core-services/document-grounding/pom.xml b/core-services/document-grounding/pom.xml index 0fd55b018..9bd73f38d 100644 --- a/core-services/document-grounding/pom.xml +++ b/core-services/document-grounding/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml document-grounding diff --git a/core-services/prompt-registry/pom.xml b/core-services/prompt-registry/pom.xml index f9d462ede..b4b831b5f 100644 --- a/core-services/prompt-registry/pom.xml +++ b/core-services/prompt-registry/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml prompt-registry diff --git a/core/pom.xml b/core/pom.xml index 16061ae0c..522b6dbfc 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT core AI Core client diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 6505871af..68140484c 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.foundationmodels @@ -124,6 +124,7 @@ com.openai openai-java-core + true diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java index 50bea1504..7d0f822bd 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java @@ -1,10 +1,13 @@ package com.sap.ai.sdk.foundationmodels.openai; +import com.google.common.annotations.Beta; + /** * Functional interface representing audio input channel (audio data consumer) * *

    Should be closed by application (try-with-resources) when not needed anymore */ +@Beta public interface AudioInputChannel extends AutoCloseable { /** @@ -14,5 +17,6 @@ public interface AudioInputChannel extends AutoCloseable { * * @param rawBytesChunk binary data in the depending on the use case format */ + @Beta void inputAudio(byte[] rawBytesChunk); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java index 81217d100..7c15fb759 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java @@ -1,6 +1,9 @@ package com.sap.ai.sdk.foundationmodels.openai; +import com.google.common.annotations.Beta; + /** Functional interface representing audio output channel (audio data consumer) */ +@Beta public interface AudioOutputChannel { /** @@ -13,5 +16,6 @@ public interface AudioOutputChannel { * single logical entity (e.g. gets called at the end when all byte parts of a single message * get passed) */ + @Beta void outputAudio(byte[] rawBytesChunk, boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java index f1d2dadd2..bbd78c40a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java @@ -1,11 +1,13 @@ package com.sap.ai.sdk.foundationmodels.openai; +import com.google.common.annotations.Beta; import javax.annotation.Nonnull; /** * Allows to input (send) text to the open channel, must be closed when not needed anymore (e.g. * try-with-resources) */ +@Beta public interface TextInputChannel extends AutoCloseable { /** @@ -13,5 +15,6 @@ public interface TextInputChannel extends AutoCloseable { * * @param text text to send */ + @Beta void sendText(@Nonnull final String text); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java index b403097e8..973d9a151 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java @@ -1,8 +1,10 @@ package com.sap.ai.sdk.foundationmodels.openai; +import com.google.common.annotations.Beta; import javax.annotation.Nonnull; /** Functional interface representing text output channel (text data consumer) */ +@Beta public interface TextOutputChannel { /** @@ -14,5 +16,6 @@ public interface TextOutputChannel { * single logical entity (e.g. gets called at the end when all parts of a single message get * passed) */ + @Beta void outputText(@Nonnull final CharSequence textChunk, final boolean isLast); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index a3be0f884..fb89f66a3 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -1,13 +1,13 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; -import com.sap.ai.sdk.core.client.realtime.RealtimeParam; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import com.sap.cloud.sdk.cloudplatform.connectivity.Destination; import com.sap.cloud.sdk.cloudplatform.connectivity.Header; import java.util.HashMap; +import java.util.Map; import javax.annotation.Nonnull; /** @@ -62,14 +62,8 @@ public OpenAiRealtimeClient(@Nonnull final Destination destination) { public TextInputChannel textToSpeech( @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params) { - final var extraHeaders = destination.asHttp().getHeaders(); - final var headers = new HashMap(extraHeaders.size() + 1); - for (final Header header : extraHeaders) { - headers.put(header.getName(), header.getValue()); - } - final var endpoint = getRealtimeEndpoint(); - - return new TextToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + return new TextToSpeechRealtimeClient( + getRealtimeEndpoint(), buildRealtimeHeaders(), audioOutputConsumer, params); } /** @@ -104,14 +98,17 @@ public TextInputChannel textToSpeech( public AudioInputChannel speechToSpeech( @Nonnull final AudioOutputChannel audioOutputConsumer, @Nonnull final RealtimeParam... params) { + return new SpeechToSpeechRealtimeClient( + getRealtimeEndpoint(), buildRealtimeHeaders(), audioOutputConsumer, params); + } + + private Map buildRealtimeHeaders() { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (final Header header : extraHeaders) { headers.put(header.getName(), header.getValue()); } - final var endpoint = getRealtimeEndpoint(); - - return new SpeechToSpeechRealtimeClient(endpoint, headers, audioOutputConsumer, params); + return headers; } private String getRealtimeEndpoint() { diff --git a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java similarity index 74% rename from core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java index e614132f1..76fe3e656 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParam.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java @@ -1,18 +1,20 @@ -package com.sap.ai.sdk.core.client.realtime; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import javax.annotation.Nonnull; /** Represents possible configuration params of realtime client */ public interface RealtimeParam { /** Represents configurable options */ - enum SpeechOutputParamName { + enum ParamName { /** Voice name to use to produce sound */ - VOICE, + OUTPUT_VOICE, /** * How model will recognize that it is its turn to respond (e.g. explicitly asked, automatically * detected) */ TURN_DETECTION, + /** Override or specify system prompt given to a model */ + SYSTEM_PROMPT, } /** @@ -21,7 +23,7 @@ enum SpeechOutputParamName { * @return name */ @Nonnull - SpeechOutputParamName getParamName(); + ParamName getParamName(); /** * Returns string value representation of the param diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java new file mode 100644 index 000000000..a6d08b144 --- /dev/null +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java @@ -0,0 +1,44 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** Allows to configure model system prompt */ +public final class RealtimeParamSystemPrompt implements RealtimeParam { + + private final String systemPrompt; + + /** + * Constructs RealtimeParamSystemPrompt object + * + * @param systemPrompt system prompt to use + */ + public RealtimeParamSystemPrompt(@Nonnull final String systemPrompt) { + this.systemPrompt = systemPrompt; + } + + @Override + public @Nonnull ParamName getParamName() { + return ParamName.SYSTEM_PROMPT; + } + + @Override + public @Nonnull String getValueAsString() { + return systemPrompt; + } + + @Override + public boolean equals(@Nullable final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final RealtimeParamSystemPrompt that = (RealtimeParamSystemPrompt) o; + return Objects.equals(systemPrompt, that.systemPrompt); + } + + @Override + public int hashCode() { + return Objects.hashCode(systemPrompt); + } +} diff --git a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java similarity index 90% rename from core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java index 8eacfb80f..c0dfddbfa 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetection.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core.client.realtime; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import java.util.Objects; import javax.annotation.Nonnull; @@ -26,8 +26,8 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { } @Override - public @Nonnull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.TURN_DETECTION; + public @Nonnull ParamName getParamName() { + return ParamName.TURN_DETECTION; } @Override diff --git a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java similarity index 91% rename from core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java rename to foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java index 6d45dd4f4..4b28fd561 100644 --- a/core/src/main/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoice.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core.client.realtime; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import java.util.Objects; import javax.annotation.Nonnull; @@ -32,8 +32,8 @@ public static RealtimeParamVoice unsafeWithExplicitVoice(@Nonnull final String v } @Override - public @Nonnull SpeechOutputParamName getParamName() { - return SpeechOutputParamName.VOICE; + public @Nonnull ParamName getParamName() { + return ParamName.OUTPUT_VOICE; } @Override diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java index 0ff6bccfb..5516b26f1 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java @@ -5,8 +5,6 @@ import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioFormats; import com.openai.models.realtime.RealtimeAudioInputTurnDetection; -import com.sap.ai.sdk.core.client.realtime.RealtimeParam; -import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Arrays; @@ -20,28 +18,12 @@ class SpeechToSpeechRealtimeClient extends ToAudioRealtimeClient implements Audi private static final int MAX_DATA_CHUNK_SIZE_BYTES = 8192; - private static final String TASK = ""; - - private final boolean eagerTurnDetection; - public SpeechToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, @Nonnull final RealtimeParam... params) { - super(url, httpHeaders, outputConsumer, params); - - var turnDetectionEager = false; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - this.eagerTurnDetection = turnDetectionEager; + super(url, httpHeaders, outputConsumer, false, params); } @Override @@ -89,10 +71,4 @@ public void inputAudio(@Nonnull final byte[] rawAudioChunk) { askForResponse(); } } - - @Override - @Nonnull - protected String getSystemPrompt() { - return TASK; - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 159a86285..2db0d1361 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -5,41 +5,35 @@ import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioFormats; import com.openai.models.realtime.RealtimeConversationItemUserMessage; -import com.sap.ai.sdk.core.client.realtime.RealtimeParam; -import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @Slf4j class TextToSpeechRealtimeClient extends ToAudioRealtimeClient implements TextInputChannel { - private static final String TASK = + private static final String SYSTEM_PROMPT = "you are a speaker and your role is to read (produce audio) of the user input speech. voice user text input, " + "do not answer questions, just read them"; - private final boolean eagerTurnDetection; - public TextToSpeechRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, @Nonnull final RealtimeParam... params) { - super(url, httpHeaders, outputConsumer, params); - var turnDetectionEager = true; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - this.eagerTurnDetection = turnDetectionEager; + super( + url, + httpHeaders, + outputConsumer, + true, + Stream.concat( + Stream.of((RealtimeParam) new RealtimeParamSystemPrompt(SYSTEM_PROMPT)), + Stream.of(params)) + .toArray(RealtimeParam[]::new)); } @Override @@ -74,10 +68,4 @@ public void sendText(@Nonnull final String text) { askForResponse(); } } - - @Override - @Nonnull - protected String getSystemPrompt() { - return TASK; - } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 28f1de460..895f260f2 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -8,16 +8,16 @@ import com.openai.models.realtime.RealtimeSessionCreateRequest; import com.openai.models.realtime.SessionUpdateEvent; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; -import com.sap.ai.sdk.core.client.realtime.RealtimeParam; -import com.sap.ai.sdk.core.client.realtime.RealtimeParamVoice; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; +/** Implements common functionality for realtime api clients that output audio */ @Slf4j abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { @@ -30,15 +30,34 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { private final AudioOutputChannel outputConsumer; private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + /** + * defines default eagerness (if EACH_CALL_IS_A_TURN) behavior if specific turn detection config + * not provided + */ + protected final boolean eagerTurnDetection; + + private final String systemPrompt; + + /** + * Constructs the object + * + * @param url - realtime api endpoint url + * @param httpHeaders - http headers (key - value) for client to use + * @param outputConsumer - consumer of audio bytes in pcm 24000 Hz mono little endian format + * @param defaultTurnDetectionEager - if explicit cfg for turn detection was not specified, this + * turn detection eagerness flag will be used (true results in EACH_CALL_IS_A_TURN handling) + * @param params - possible overrides for default params (e.g. voice, system prompt, etc.) + */ public ToAudioRealtimeClient( @Nonnull final String url, @Nonnull final Map httpHeaders, @Nonnull final AudioOutputChannel outputConsumer, + final boolean defaultTurnDetectionEager, @Nonnull final RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.SpeechOutputParamName.VOICE) { + if (param.getParamName() == RealtimeParam.ParamName.OUTPUT_VOICE) { if (RealtimeParamVoice.DEFAULT_2.equals(param)) { voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { @@ -46,13 +65,40 @@ public ToAudioRealtimeClient( } } } + + var turnDetectionEager = defaultTurnDetectionEager; + for (final RealtimeParam param : params) { + if (param.getParamName() == RealtimeParam.ParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + + var systemPrompt = ""; + for (final RealtimeParam param : params) { + if (param.getParamName() == RealtimeParam.ParamName.SYSTEM_PROMPT) { + systemPrompt = param.getValueAsString(); + } + } + this.outputConsumer = outputConsumer; this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + this.systemPrompt = systemPrompt; } @Nonnull protected abstract RealtimeAudioConfigInput inputConfig(); + @Override + @Nonnull + protected Optional getSystemPrompt() { + return systemPrompt.isEmpty() ? Optional.empty() : Optional.of(systemPrompt); + } + @Override protected void onResponse(@Nonnull final String eventType, @Nonnull final JsonNode event) { if ("response.output_audio.delta".equals(eventType)) { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index afd8b134f..7ddf957be 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -17,6 +17,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.Timer; import java.util.TimerTask; @@ -78,7 +79,7 @@ public void askForResponse() { try { ws = this.ws.join(); } catch (final CompletionException e) { - throw new ClientException("failed to establish web socket connection", e); + throw new ClientException("Failed to establish web socket connection", e); } synchronized (this) { try { @@ -96,7 +97,7 @@ public void close() { try { ws = this.ws.join(); } catch (final CompletionException e) { - throw new ClientException("failed to establish web socket connection", e); + throw new ClientException("Failed to establish web socket connection", e); } synchronized (this) { heartbeatTimer.cancel(); @@ -110,7 +111,10 @@ public void close() { } @Nonnull - protected abstract String getSystemPrompt(); + protected Optional getSystemPrompt() { + return Optional.empty(); + } + ; protected abstract void onResponse( @Nonnull final String eventType, @Nonnull final JsonNode event); @@ -123,7 +127,7 @@ protected void sendMessage(@Nonnull final Object message) { try { ws = this.ws.join(); } catch (CompletionException e) { - throw new ClientException("failed to establish web socket connection", e); + throw new ClientException("Failed to establish web socket connection", e); } synchronized (this) { try { @@ -179,7 +183,7 @@ private void configureSession(@Nonnull final WebSocket ws) { private void configureConversation(@Nonnull final WebSocket ws) { final var systemPrompt = getSystemPrompt(); - if (systemPrompt.isEmpty()) { + if (systemPrompt.isEmpty() || systemPrompt.get().isEmpty()) { return; } final var systemConversationItem = @@ -189,7 +193,7 @@ private void configureConversation(@Nonnull final WebSocket ws) { RealtimeConversationItemSystemMessage.builder() .addContent( RealtimeConversationItemSystemMessage.Content.builder() - .text(systemPrompt) + .text(systemPrompt.get()) .type(RealtimeConversationItemSystemMessage.Content.Type.INPUT_TEXT) .build()) .build())) diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPromptUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPromptUnitTest.java new file mode 100644 index 000000000..ee0f3a4b8 --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPromptUnitTest.java @@ -0,0 +1,48 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class RealtimeParamSystemPromptUnitTest { + + @Test + void getParamName() { + assertThat(new RealtimeParamSystemPrompt(randomString()).getParamName()) + .isEqualTo(RealtimeParam.ParamName.SYSTEM_PROMPT); + } + + @Test + void getValueAsString() { + final var randomValue = randomString(); + final var prompt = new RealtimeParamSystemPrompt(randomValue); + assertThat(prompt.getValueAsString()).isEqualTo(randomValue); + } + + @Test + void testEquals() { + final var randomValue = randomString(); + final var prompt1 = new RealtimeParamSystemPrompt(randomValue); + final var prompt2 = new RealtimeParamSystemPrompt(randomValue); + assertThat(prompt1).isEqualTo(prompt2); + + final var otherRandomValue = randomString(); + final var prompt3 = new RealtimeParamSystemPrompt(otherRandomValue); + assertThat(prompt1).isNotEqualTo(prompt3); + assertThat(prompt2).isNotEqualTo(prompt3); + } + + @Test + void testHashCode() { + final var randomValue = randomString(); + final var prompt1 = new RealtimeParamSystemPrompt(randomValue); + final var prompt2 = new RealtimeParamSystemPrompt(randomValue); + + assertThat(prompt1.hashCode()).isEqualTo(prompt2.hashCode()); + } + + private String randomString() { + return UUID.randomUUID().toString().substring(0, 20).replace("-", " "); + } +} diff --git a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetectionUnitTest.java similarity index 87% rename from core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java rename to foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetectionUnitTest.java index adc8ad835..4e1abaa9a 100644 --- a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamTurnDetectionUnitTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetectionUnitTest.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core.client.realtime; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import static org.assertj.core.api.Assertions.assertThat; @@ -19,11 +19,11 @@ void setUp() { @Test void getParamName() { - assertThat(random.getParamName()).isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + assertThat(random.getParamName()).isEqualTo(RealtimeParam.ParamName.TURN_DETECTION); assertThat(RealtimeParamTurnDetection.BY_MODEL_AUTO.getParamName()) - .isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + .isEqualTo(RealtimeParam.ParamName.TURN_DETECTION); assertThat(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.getParamName()) - .isEqualTo(RealtimeParam.SpeechOutputParamName.TURN_DETECTION); + .isEqualTo(RealtimeParam.ParamName.TURN_DETECTION); } @Test diff --git a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java similarity index 87% rename from core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java rename to foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java index 06b516708..99164fcea 100644 --- a/core/src/test/java/com/sap/ai/sdk/core/client/realtime/RealtimeParamVoiceUnitTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java @@ -1,4 +1,4 @@ -package com.sap.ai.sdk.core.client.realtime; +package com.sap.ai.sdk.foundationmodels.openai.realtime; import static org.assertj.core.api.Assertions.assertThat; @@ -24,11 +24,11 @@ void unsafeWithExplicitVoice() { @Test void getParamName() { - assertThat(random.getParamName()).isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + assertThat(random.getParamName()).isEqualTo(RealtimeParam.ParamName.OUTPUT_VOICE); assertThat(RealtimeParamVoice.DEFAULT_1.getParamName()) - .isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + .isEqualTo(RealtimeParam.ParamName.OUTPUT_VOICE); assertThat(RealtimeParamVoice.DEFAULT_1.getParamName()) - .isEqualTo(RealtimeParam.SpeechOutputParamName.VOICE); + .isEqualTo(RealtimeParam.ParamName.OUTPUT_VOICE); } @Test diff --git a/foundation-models/sap-rpt/pom.xml b/foundation-models/sap-rpt/pom.xml index 7403a9a79..ba3a13607 100644 --- a/foundation-models/sap-rpt/pom.xml +++ b/foundation-models/sap-rpt/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.foundationmodels diff --git a/orchestration/pom.xml b/orchestration/pom.xml index 525b19258..0cca802c8 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT orchestration Orchestration client diff --git a/pom.xml b/pom.xml index d3e7290c6..58e1cf8a8 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT pom Parent pom SAP Cloud SDK for AI is the official Software Development Kit (SDK) for SAP AI Core, SAP Generative AI Hub, and Orchestration Service. The parent pom defines dependency versions and plugin configurations diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index ac97990da..9b71dcabc 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.23.0-SNAPSHOT + 1.24.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.app diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 956ad5cd5..3a43b191d 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -51,7 +51,7 @@ protected void handleBinaryMessage( try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); } catch (final IOException e) { - log.error("failed to send audio data to realtime api", e); + log.error("Failed to send audio data to realtime api", e); } })); channel.inputAudio(chunkBytes); @@ -66,7 +66,7 @@ public void afterConnectionClosed( try { inputChannel.close(); } catch (final Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); + log.warn("Failed to close input channel for session {}", sessionId, e); } return null; }); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 7bd8a45b8..c15f13a03 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -52,7 +52,7 @@ protected void handleBinaryMessage( try { session.sendMessage(new BinaryMessage(rawBytesChunk, isLast)); } catch (final IOException e) { - log.error("failed to send text message to realtime api", e); + log.error("Failed to send text message to realtime api", e); } })); channel.sendText(new String(textBytes, StandardCharsets.UTF_8)); @@ -67,7 +67,7 @@ public void afterConnectionClosed( try { inputChannel.close(); } catch (Exception e) { - log.warn("failed to close input channel for session {}", sessionId, e); + log.warn("Failed to close input channel for session {}", sessionId, e); } return null; }); diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 967120463..8c9c7f2b3 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -1489,8 +1489,8 @@

    📂 Batch API

    ⏰ Realtime API

    Realtime API allows for various real time interactions TBD Documentation + href="https://developers.openai.com/api/docs/guides/realtime" + target="_blank">('Voice agents' and 'speech generation' scenarios are currently supported)
    In these examples, web socket sessions are created and used to send text/audio to the local server and receive audio back diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java index cf8ad26ef..20fa264b4 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/RealtimeApiTest.java @@ -3,10 +3,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import com.sap.ai.sdk.core.client.realtime.RealtimeParamTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.realtime.RealtimeParamTurnDetection; import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.IOException; From 69bbd72dc8d718a4bdb571a0717f511cbf1900c3 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 27 Jul 2026 20:36:15 +0200 Subject: [PATCH 61/67] remove unit test which is alredy duplicated by an integration test --- .../realtime/OpenAiRealtimeClientTest.java | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java deleted file mode 100644 index 2f24dc75c..000000000 --- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai.realtime; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; -import com.sap.ai.sdk.foundationmodels.openai.OpenAiClient; -import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class OpenAiRealtimeClientTest { - - private OpenAiRealtimeClient client; - - @BeforeEach - void setUp() { - client = OpenAiClient.realtimeClient(); - } - - @Test - void textToSpeech() { - try (TextInputChannel ic = client.textToSpeech((a, b) -> {})) { - assertThat(ic).isNotNull(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - void speechToSpeech() { - try (AudioInputChannel ic = client.speechToSpeech((a, b) -> {})) { - assertThat(ic).isNotNull(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} From 6f93d30ae1dc8882e6b04d4175a484b77a2f5562 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Mon, 27 Jul 2026 20:51:17 +0200 Subject: [PATCH 62/67] revert version change to workaround ci caching issue --- core-services/batch/pom.xml | 2 +- core-services/document-grounding/pom.xml | 2 +- core-services/prompt-registry/pom.xml | 2 +- core/pom.xml | 2 +- foundation-models/openai/pom.xml | 2 +- foundation-models/sap-rpt/pom.xml | 2 +- orchestration/pom.xml | 2 +- pom.xml | 2 +- sample-code/spring-app/pom.xml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core-services/batch/pom.xml b/core-services/batch/pom.xml index 2dd14a064..cac9c9a9b 100644 --- a/core-services/batch/pom.xml +++ b/core-services/batch/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml batch diff --git a/core-services/document-grounding/pom.xml b/core-services/document-grounding/pom.xml index 9bd73f38d..0fd55b018 100644 --- a/core-services/document-grounding/pom.xml +++ b/core-services/document-grounding/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml document-grounding diff --git a/core-services/prompt-registry/pom.xml b/core-services/prompt-registry/pom.xml index b4b831b5f..f9d462ede 100644 --- a/core-services/prompt-registry/pom.xml +++ b/core-services/prompt-registry/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml prompt-registry diff --git a/core/pom.xml b/core/pom.xml index 522b6dbfc..16061ae0c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT core AI Core client diff --git a/foundation-models/openai/pom.xml b/foundation-models/openai/pom.xml index 68140484c..9d5280f45 100644 --- a/foundation-models/openai/pom.xml +++ b/foundation-models/openai/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.foundationmodels diff --git a/foundation-models/sap-rpt/pom.xml b/foundation-models/sap-rpt/pom.xml index ba3a13607..7403a9a79 100644 --- a/foundation-models/sap-rpt/pom.xml +++ b/foundation-models/sap-rpt/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.foundationmodels diff --git a/orchestration/pom.xml b/orchestration/pom.xml index 0cca802c8..525b19258 100644 --- a/orchestration/pom.xml +++ b/orchestration/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT orchestration Orchestration client diff --git a/pom.xml b/pom.xml index 58e1cf8a8..d3e7290c6 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT pom Parent pom SAP Cloud SDK for AI is the official Software Development Kit (SDK) for SAP AI Core, SAP Generative AI Hub, and Orchestration Service. The parent pom defines dependency versions and plugin configurations diff --git a/sample-code/spring-app/pom.xml b/sample-code/spring-app/pom.xml index 9b71dcabc..ac97990da 100644 --- a/sample-code/spring-app/pom.xml +++ b/sample-code/spring-app/pom.xml @@ -4,7 +4,7 @@ com.sap.ai.sdk sdk-parent - 1.24.0-SNAPSHOT + 1.23.0-SNAPSHOT ../../pom.xml com.sap.ai.sdk.app From c81910b19a4b4eec02135cf051de41534f05d68c Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Tue, 28 Jul 2026 16:53:40 +0200 Subject: [PATCH 63/67] add unit tests, fix test coverage issue --- .../openai/realtime/OpenAiRealtimeClient.java | 8 +- .../SpeechToSpeechRealtimeClient.java | 13 + .../realtime/TextToSpeechRealtimeClient.java | 22 + .../realtime/ToAudioRealtimeClient.java | 50 +- .../realtime/WSOpenAiRealtimeClient.java | 6 +- .../BufferedWebSocketListenerUnitTest.java | 139 ++++++ .../OpenAiRealtimeClientUnitTest.java | 138 ++++++ .../SpeechToSpeechRealtimeClientUnitTest.java | 202 ++++++++ .../TextToSpeechRealtimeClientTest.java | 185 +++++++ .../ToAudioRealtimeClientUnitTest.java | 304 ++++++++++++ .../WSOpenAiRealtimeClientUnitTest.java | 454 ++++++++++++++++++ 11 files changed, 1511 insertions(+), 10 deletions(-) create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListenerUnitTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClientUnitTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java create mode 100644 foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClientUnitTest.java diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index fb89f66a3..36e60c3e9 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -16,10 +16,10 @@ */ public class OpenAiRealtimeClient { - private static final int PATH_BUFFER_SIZE = + static final int PATH_BUFFER_SIZE = 400; // existing URLs are ~120 symbols long, 400 has reasonable margin - private final Destination destination; + final Destination destination; /** * Created OpenAI Realtime client for a specific destination @@ -102,7 +102,7 @@ public AudioInputChannel speechToSpeech( getRealtimeEndpoint(), buildRealtimeHeaders(), audioOutputConsumer, params); } - private Map buildRealtimeHeaders() { + Map buildRealtimeHeaders() { final var extraHeaders = destination.asHttp().getHeaders(); final var headers = new HashMap(extraHeaders.size() + 1); for (final Header header : extraHeaders) { @@ -111,7 +111,7 @@ private Map buildRealtimeHeaders() { return headers; } - private String getRealtimeEndpoint() { + String getRealtimeEndpoint() { final var sb = new StringBuilder(PATH_BUFFER_SIZE); sb.append("wss://"); final var pathParts = destination.asHttp().getUri().toString().split("//"); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java index 5516b26f1..3d5d40b93 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClient.java @@ -7,9 +7,13 @@ import com.openai.models.realtime.RealtimeAudioInputTurnDetection; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import java.net.http.HttpClient; +import java.net.http.WebSocket; import java.util.Arrays; import java.util.Base64; import java.util.Map; +import java.util.Timer; +import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -26,6 +30,15 @@ public SpeechToSpeechRealtimeClient( super(url, httpHeaders, outputConsumer, false, params); } + SpeechToSpeechRealtimeClient( + @Nonnull final HttpClient httpClient, + @Nonnull final CompletableFuture ws, + @Nonnull final Timer timer, + @Nonnull final AudioOutputChannel outputConsumer, + @Nonnull final RealtimeParam... params) { + super(httpClient, ws, timer, outputConsumer, false, params); + } + @Override @Nonnull protected RealtimeAudioConfigInput inputConfig() { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java index 2db0d1361..d0971fc44 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClient.java @@ -7,8 +7,12 @@ import com.openai.models.realtime.RealtimeConversationItemUserMessage; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import java.net.http.HttpClient; +import java.net.http.WebSocket; import java.util.Map; import java.util.Optional; +import java.util.Timer; +import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -36,6 +40,24 @@ public TextToSpeechRealtimeClient( .toArray(RealtimeParam[]::new)); } + TextToSpeechRealtimeClient( + @Nonnull final HttpClient httpClient, + @Nonnull final CompletableFuture ws, + @Nonnull final Timer timer, + @Nonnull final AudioOutputChannel outputConsumer, + @Nonnull final RealtimeParam... params) { + super( + httpClient, + ws, + timer, + outputConsumer, + true, + Stream.concat( + Stream.of((RealtimeParam) new RealtimeParamSystemPrompt(SYSTEM_PROMPT)), + Stream.of(params)) + .toArray(RealtimeParam[]::new)); + } + @Override @Nonnull protected RealtimeAudioConfigInput inputConfig() { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 895f260f2..e5da6dc7f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -9,11 +9,15 @@ import com.openai.models.realtime.SessionUpdateEvent; import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import java.net.http.HttpClient; +import java.net.http.WebSocket; import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.Timer; +import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -27,8 +31,8 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - private final AudioOutputChannel outputConsumer; - private final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; + final AudioOutputChannel outputConsumer; + final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; /** * defines default eagerness (if EACH_CALL_IS_A_TURN) behavior if specific turn detection config @@ -36,7 +40,7 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { */ protected final boolean eagerTurnDetection; - private final String systemPrompt; + final String systemPrompt; /** * Constructs the object @@ -90,6 +94,46 @@ public ToAudioRealtimeClient( this.systemPrompt = systemPrompt; } + ToAudioRealtimeClient( + @Nonnull final HttpClient httpClient, + @Nonnull final CompletableFuture ws, + @Nonnull final Timer timer, + @Nonnull final AudioOutputChannel outputConsumer, + final boolean eagerTurnDetection, + @Nonnull final RealtimeParam... params) { + super(httpClient, ws, timer, HANDLED_RESPONSE_TYPES); + var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + for (final RealtimeParam param : params) { + if (param.getParamName() == RealtimeParam.ParamName.OUTPUT_VOICE) { + if (RealtimeParamVoice.DEFAULT_2.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { + voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + } + } + var turnDetectionEager = eagerTurnDetection; + for (final RealtimeParam param : params) { + if (param.getParamName() == RealtimeParam.ParamName.TURN_DETECTION) { + if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { + turnDetectionEager = true; + } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { + turnDetectionEager = false; + } + } + } + var systemPrompt = ""; + for (final RealtimeParam param : params) { + if (param.getParamName() == RealtimeParam.ParamName.SYSTEM_PROMPT) { + systemPrompt = param.getValueAsString(); + } + } + this.outputConsumer = outputConsumer; + this.voice = voice; + this.eagerTurnDetection = turnDetectionEager; + this.systemPrompt = systemPrompt; + } + @Nonnull protected abstract RealtimeAudioConfigInput inputConfig(); diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index 7ddf957be..30b75eca4 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -139,13 +139,13 @@ protected void sendMessage(@Nonnull final Object message) { } } - private synchronized void onSocketOpen(@Nonnull final WebSocket ws) { + synchronized void onSocketOpen(@Nonnull final WebSocket ws) { configureSession(ws); configureConversation(ws); scheduleHeartbeat(ws); } - private void onText(@Nonnull final WebSocket webSocket, @Nonnull final CharSequence data) { + protected void onText(@Nonnull final WebSocket webSocket, @Nonnull final CharSequence data) { final JsonNode event; try { event = JACKSON.readTree(data.toString()); @@ -162,7 +162,7 @@ private void onText(@Nonnull final WebSocket webSocket, @Nonnull final CharSeque webSocket.request(1); } - private synchronized void sendPing(@Nonnull final WebSocket ws) { + protected synchronized void sendPing(@Nonnull final WebSocket ws) { if (ws.isInputClosed()) { return; } diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListenerUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListenerUnitTest.java new file mode 100644 index 000000000..fcb379c42 --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListenerUnitTest.java @@ -0,0 +1,139 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BufferedWebSocketListenerUnitTest { + + private WebSocket webSocketMock; + private List openCallbacks; + private List textCallbacks; + + @BeforeEach + void setUp() { + webSocketMock = mock(WebSocket.class); + openCallbacks = new ArrayList<>(); + textCallbacks = new ArrayList<>(); + } + + private BufferedWebSocketListener build() { + return new BufferedWebSocketListener( + openCallbacks::add, (ws, data) -> textCallbacks.add(data.toString())); + } + + @Test + void onOpenInvokesConsumerWithWebSocket() { + build().onOpen(webSocketMock); + + assertThat(openCallbacks).containsExactly(webSocketMock); + } + + @Test + void onOpenRequestsNextMessage() { + build().onOpen(webSocketMock); + + verify(webSocketMock).request(1L); + } + + @Test + void onTextInvokesConsumerWhenLastFrameReceived() { + build().onText(webSocketMock, "hello", true); + + assertThat(textCallbacks).containsExactly("hello"); + } + + @Test + void onTextDoesNotInvokeConsumerForPartialFrame() { + build().onText(webSocketMock, "hel", false); + + assertThat(textCallbacks).isEmpty(); + } + + @Test + void onTextRequestsNextMessageForPartialFrame() { + build().onText(webSocketMock, "hel", false); + + verify(webSocketMock).request(1L); + } + + @Test + void onTextReturnedFutureIsAlreadyCompleted() { + final var future = build().onText(webSocketMock, "hello", true); + + assertThat(future.toCompletableFuture()).isDone(); + } + + @Test + void onTextBuffersPartialsAndDeliversOnLastFrame() { + final var listener = build(); + listener.onText(webSocketMock, "foo", false); + listener.onText(webSocketMock, "bar", false); + listener.onText(webSocketMock, "baz", true); + + assertThat(textCallbacks).containsExactly("foobarbaz"); + } + + @Test + void onTextResetsBufferAfterCompleteMessage() { + final var listener = build(); + listener.onText(webSocketMock, "first", true); + listener.onText(webSocketMock, "second", true); + + assertThat(textCallbacks).containsExactly("first", "second"); + } + + @Test + void onTextBufferDoesNotLeakAcrossMessages() { + final var listener = build(); + listener.onText(webSocketMock, "part1", false); + listener.onText(webSocketMock, "part2", true); // completes first message + listener.onText(webSocketMock, "part3", true); // second message starts clean + + assertThat(textCallbacks).containsExactly("part1part2", "part3"); + } + + @Test + void onErrorDoesNotThrow() { + assertThatNoException() + .isThrownBy(() -> build().onError(webSocketMock, new RuntimeException("boom"))); + } + + @Test + void onErrorDoesNotInvokeTextConsumer() { + build().onError(webSocketMock, new RuntimeException("boom")); + + assertThat(textCallbacks).isEmpty(); + } + + @Test + void onBinaryDoesNotInvokeTextConsumer() { + build().onBinary(webSocketMock, ByteBuffer.wrap(new byte[] {0x01}), true); + + assertThat(textCallbacks).isEmpty(); + } + + @Test + void onBinaryReturnedFutureIsAlreadyCompleted() { + final var future = build().onBinary(webSocketMock, ByteBuffer.wrap(new byte[] {0x01}), true); + + assertThat(future.toCompletableFuture()).isDone(); + } + + @Test + void onBinaryDoesNotRequestNextMessage() { + build().onBinary(webSocketMock, ByteBuffer.wrap(new byte[] {0x01}), true); + + verify(webSocketMock, never()).request(anyLong()); + } +} diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java new file mode 100644 index 000000000..d1201f40e --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java @@ -0,0 +1,138 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination; +import org.junit.jupiter.api.Test; + +class OpenAiRealtimeClientUnitTest { + + private static final AudioOutputChannel NO_OP_OUTPUT = (audio, done) -> {}; + + @Test + void constructorStoresDestination() { + final var destination = DefaultHttpDestination.builder("https://api.example.com").build(); + final var client = new OpenAiRealtimeClient(destination); + + assertThat(client.destination).isSameAs(destination); + } + + @Test + void buildRealtimeHeadersReturnsEmptyMapWhenDestinationHasNoHeaders() { + final var destination = DefaultHttpDestination.builder("https://api.example.com").build(); + final var client = new OpenAiRealtimeClient(destination); + + assertThat(client.buildRealtimeHeaders()).isEmpty(); + } + + @Test + void buildRealtimeHeadersCopiesAllDestinationHeaders() { + final var destination = + DefaultHttpDestination.builder("https://api.example.com") + .header("Authorization", "Bearer token-123") + .header("X-Custom-Header", "custom-value") + .build(); + final var client = new OpenAiRealtimeClient(destination); + + final var headers = client.buildRealtimeHeaders(); + + assertThat(headers) + .containsEntry("Authorization", "Bearer token-123") + .containsEntry("X-Custom-Header", "custom-value"); + } + + @Test + void buildRealtimeHeadersReturnsModifiableCopy() { + final var destination = + DefaultHttpDestination.builder("https://api.example.com") + .header("Authorization", "Bearer token") + .build(); + final var client = new OpenAiRealtimeClient(destination); + + final var headers = client.buildRealtimeHeaders(); + headers.put("injected", "value"); + + assertThat(client.buildRealtimeHeaders()).doesNotContainKey("injected"); + } + + @Test + void getRealtimeEndpointBuildsWssUrlAndAppendsSuffix() { + final var destination = + DefaultHttpDestination.builder( + "https://my-resource.openai.azure.com/openai/deployments/gpt-4o") + .build(); + final var client = new OpenAiRealtimeClient(destination); + + assertThat(client.getRealtimeEndpoint()).startsWith("wss://").endsWith("v1/realtime"); + } + + @Test + void getRealtimeEndpointReplacesApiSubdomainWithRealtime() { + final var destination = + DefaultHttpDestination.builder("https://api.example.com/some/path").build(); + final var client = new OpenAiRealtimeClient(destination); + + final var endpoint = client.getRealtimeEndpoint(); + + assertThat(endpoint).startsWith("wss://realtime.example.com"); + assertThat(endpoint).doesNotContain("api.example.com"); + } + + @Test + void getRealtimeEndpointPreservesNonApiSubdomain() { + final var destination = + DefaultHttpDestination.builder("https://my-resource.openai.azure.com/openai").build(); + final var client = new OpenAiRealtimeClient(destination); + + final var endpoint = client.getRealtimeEndpoint(); + + assertThat(endpoint).startsWith("wss://my-resource.openai.azure.com"); + } + + @Test + void getRealtimeEndpointThrowsOnUriWithoutDoubleSlash() { + final var destination = DefaultHttpDestination.builder("https://host").build(); + final var client = + new OpenAiRealtimeClient(destination) { + @Override + public String getRealtimeEndpoint() { + // Force a URI that produces != 2 parts when split on "//" + final var sb = new StringBuilder(PATH_BUFFER_SIZE); + sb.append("wss://"); + final var pathParts = "no-double-slash".split("//"); + if (pathParts.length != 2) { + throw new IllegalArgumentException("Invalid destination URI: no-double-slash"); + } + return sb.toString(); + } + }; + + assertThatThrownBy(client::getRealtimeEndpoint) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid destination URI"); + } + + @Test + void textToSpeechReturnsTextInputChannel() { + final var destination = DefaultHttpDestination.builder("https://api.example.com").build(); + final var client = new OpenAiRealtimeClient(destination); + + final var channel = client.textToSpeech(NO_OP_OUTPUT); + + assertThat(channel).isInstanceOf(TextInputChannel.class); + } + + @Test + void speechToSpeechReturnsAudioInputChannel() { + final var destination = DefaultHttpDestination.builder("https://api.example.com").build(); + final var client = new OpenAiRealtimeClient(destination); + + final var channel = client.speechToSpeech(NO_OP_OUTPUT); + + assertThat(channel).isInstanceOf(AudioInputChannel.class); + } +} diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClientUnitTest.java new file mode 100644 index 000000000..8c91d6610 --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/SpeechToSpeechRealtimeClientUnitTest.java @@ -0,0 +1,202 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Timer; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class SpeechToSpeechRealtimeClientUnitTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private AudioOutputChannel outputConsumerMock; + private WebSocket webSocketMock; + + @BeforeEach + void setUp() { + outputConsumerMock = mock(AudioOutputChannel.class); + webSocketMock = mock(WebSocket.class); + when(webSocketMock.sendText(any(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendClose(anyInt(), anyString())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendPing(any())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + } + + private SpeechToSpeechRealtimeClient build(final RealtimeParam... params) { + return new SpeechToSpeechRealtimeClient( + mock(HttpClient.class), + CompletableFuture.completedFuture(webSocketMock), + mock(Timer.class), + outputConsumerMock, + params); + } + + private List captureAllSentTexts() { + final var captor = ArgumentCaptor.forClass(CharSequence.class); + try { + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), anyBoolean()); + return captor.getAllValues().stream().map(CharSequence::toString).toList(); + } catch (final org.mockito.exceptions.verification.WantedButNotInvoked e) { + return List.of(); + } + } + + @Test + void defaultTurnDetectionIsNotEager() { + assertThat(build().eagerTurnDetection).isFalse(); + } + + @Test + void turnDetectionBecomesEagerWhenParamProvided() { + assertThat(build(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN).eagerTurnDetection).isTrue(); + } + + @Test + void turnDetectionRemainsNonEagerWhenByModelAutoProvided() { + assertThat(build(RealtimeParamTurnDetection.BY_MODEL_AUTO).eagerTurnDetection).isFalse(); + } + + @Test + void outputConsumerIsStoredFromConstructor() { + assertThat(build().outputConsumer).isSameAs(outputConsumerMock); + } + + @Test + void inputConfigHasPcm24000Format() { + final var format = build().inputConfig().format().orElseThrow().audioPcm().orElseThrow(); + + assertThat(format.type().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM); + assertThat(format.rate().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Rate._24000); + } + + @Test + void inputConfigHasSemanticVadTurnDetectionWhenNotEager() { + final var config = build(RealtimeParamTurnDetection.BY_MODEL_AUTO).inputConfig(); + + assertThat(config.turnDetection().orElseThrow().semanticVad()).isPresent(); + } + + @Test + void inputConfigHasNoTurnDetectionWhenEager() { + final var config = build(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN).inputConfig(); + + assertThat(config.turnDetection()).isEmpty(); + } + + @Test + void inputAudioDoesNothingForEmptyArray() { + build().inputAudio(new byte[0]); + + verify(webSocketMock, never()).sendText(any(), anyBoolean()); + } + + @Test + void inputAudioSendsOneAppendEventForSmallChunk() throws Exception { + final byte[] audio = {0x01, 0x02, 0x03}; // simple test fixture + build().inputAudio(audio); + + final var sent = captureAllSentTexts(); + assertThat(sent).hasSize(1); + final var node = MAPPER.readTree(sent.get(0)); + assertThat(node.get("type").asText()).isEqualTo("input_audio_buffer.append"); + assertThat(Base64.getDecoder().decode(node.get("audio").asText())).isEqualTo(audio); + } + + @Test + void inputAudioDoesNotSendCommitWhenNotEager() throws Exception { + build().inputAudio(new byte[] {0x01}); + + final var sent = captureAllSentTexts(); + assertThat(sent).hasSize(1); + assertThat(MAPPER.readTree(sent.get(0)).get("type").asText()) + .isEqualTo("input_audio_buffer.append"); + } + + @Test + void inputAudioSendsCommitAndResponseCreateWhenEager() throws Exception { + build(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN) + .inputAudio(new byte[] {0x01}); // simple test fixture + + final var sent = captureAllSentTexts(); + // append + commit + response.create (from askForResponse) + assertThat(sent).hasSize(3); + assertThat(MAPPER.readTree(sent.get(0)).get("type").asText()) + .isEqualTo("input_audio_buffer.append"); + assertThat(MAPPER.readTree(sent.get(1)).get("type").asText()) + .isEqualTo("input_audio_buffer.commit"); + assertThat(MAPPER.readTree(sent.get(2)).get("type").asText()).isEqualTo("response.create"); + } + + @Test + void inputAudioSplitsLargeInputInto8192ByteChunks() throws Exception { + final int chunkSize = 8192; + final byte[] audio = new byte[chunkSize * 2 + 100]; + Arrays.fill(audio, (byte) 0x42); // simple test fixture + build().inputAudio(audio); + + final var sent = captureAllSentTexts(); + assertThat(sent).hasSize(3); + + final byte[] chunk1 = + Base64.getDecoder().decode(MAPPER.readTree(sent.get(0)).get("audio").asText()); + final byte[] chunk2 = + Base64.getDecoder().decode(MAPPER.readTree(sent.get(1)).get("audio").asText()); + final byte[] chunk3 = + Base64.getDecoder().decode(MAPPER.readTree(sent.get(2)).get("audio").asText()); + assertThat(chunk1).hasSize(chunkSize); + assertThat(chunk2).hasSize(chunkSize); + assertThat(chunk3).hasSize(100); + + final byte[] reassembled = new byte[audio.length]; + System.arraycopy(chunk1, 0, reassembled, 0, chunkSize); + System.arraycopy(chunk2, 0, reassembled, chunkSize, chunkSize); + System.arraycopy(chunk3, 0, reassembled, chunkSize * 2, 100); + assertThat(reassembled).isEqualTo(audio); + } + + @Test + void inputAudioSendsExactlyOneChunkWhenSizeEqualsChunkSize() { + build().inputAudio(new byte[8192]); + + assertThat(captureAllSentTexts()).hasSize(1); + } + + @Test + void inputAudioWithEagerTurnDetectionSendsCommitAfterAllChunks() throws Exception { + final byte[] audio = new byte[8192 + 1]; + build(RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN).inputAudio(audio); + + final var sent = captureAllSentTexts(); + // 2 appends + 1 commit + 1 response.create + assertThat(sent).hasSize(4); + assertThat(MAPPER.readTree(sent.get(0)).get("type").asText()) + .isEqualTo("input_audio_buffer.append"); + assertThat(MAPPER.readTree(sent.get(1)).get("type").asText()) + .isEqualTo("input_audio_buffer.append"); + assertThat(MAPPER.readTree(sent.get(2)).get("type").asText()) + .isEqualTo("input_audio_buffer.commit"); + assertThat(MAPPER.readTree(sent.get(3)).get("type").asText()).isEqualTo("response.create"); + } +} diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java new file mode 100644 index 000000000..bec50d848 --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java @@ -0,0 +1,185 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.List; +import java.util.Timer; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class TextToSpeechRealtimeClientTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private AudioOutputChannel outputConsumerMock; + private WebSocket webSocketMock; + + @BeforeEach + void setUp() { + outputConsumerMock = mock(AudioOutputChannel.class); + webSocketMock = mock(WebSocket.class); + when(webSocketMock.sendText(any(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendClose(anyInt(), anyString())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendPing(any())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + } + + private TextToSpeechRealtimeClient build(final RealtimeParam... params) { + return new TextToSpeechRealtimeClient( + mock(HttpClient.class), + CompletableFuture.completedFuture(webSocketMock), + mock(Timer.class), + outputConsumerMock, + params); + } + + private List captureAllSentTexts() { + final var captor = ArgumentCaptor.forClass(CharSequence.class); + try { + org.mockito.Mockito.verify(webSocketMock, atLeastOnce()) + .sendText(captor.capture(), anyBoolean()); + return captor.getAllValues().stream().map(CharSequence::toString).toList(); + } catch (final org.mockito.exceptions.verification.WantedButNotInvoked e) { + return List.of(); + } + } + + @Test + void defaultTurnDetectionIsEager() { + assertThat(build().eagerTurnDetection).isTrue(); + } + + @Test + void turnDetectionCanBeOverriddenToNonEagerByParam() { + assertThat(build(RealtimeParamTurnDetection.BY_MODEL_AUTO).eagerTurnDetection).isFalse(); + } + + @Test + void outputConsumerIsStoredFromConstructor() { + assertThat(build().outputConsumer).isSameAs(outputConsumerMock); + } + + @Test + void defaultSystemPromptIsSetByConstructor() { + assertThat(build().systemPrompt).isNotEmpty(); + } + + @Test + void callerSystemPromptOverridesDefaultBecauseItComesLast() { + final var custom = "My custom prompt."; + assertThat(build(new RealtimeParamSystemPrompt(custom)).systemPrompt).isEqualTo(custom); + } + + @Test + void inputConfigHasPcm24000Format() { + final var format = build().inputConfig().format().orElseThrow().audioPcm().orElseThrow(); + + assertThat(format.type().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM); + assertThat(format.rate().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Rate._24000); + } + + @Test + void inputConfigHasNoTurnDetection() { + assertThat(build().inputConfig().turnDetection()).isEmpty(); + } + + @Test + void sendTextSendsConversationItemCreateEvent() { + build().sendText("Hello world"); + + final var sent = captureAllSentTexts(); + assertThat(sent).isNotEmpty(); + final var conversationCreate = + sent.stream() + .map( + s -> { + try { + return MAPPER.readTree(s); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(n -> "conversation.item.create".equals(n.get("type").asText())) + .findFirst(); + assertThat(conversationCreate).isPresent(); + assertThat(conversationCreate.get().at("/item/content/0/text").asText()) + .isEqualTo("Hello world"); + } + + @Test + void sendTextSendsResponseCreateWhenEager() throws Exception { + build().sendText("Hello"); + + final var sent = captureAllSentTexts(); + final var types = + sent.stream() + .map( + s -> { + try { + return MAPPER.readTree(s).get("type").asText(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .toList(); + assertThat(types).contains("conversation.item.create", "response.create"); + } + + @Test + void sendTextDoesNotSendResponseCreateWhenNotEager() throws Exception { + build(RealtimeParamTurnDetection.BY_MODEL_AUTO).sendText("Hello"); + + final var sent = captureAllSentTexts(); + final var types = + sent.stream() + .map( + s -> { + try { + return MAPPER.readTree(s).get("type").asText(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .toList(); + assertThat(types).contains("conversation.item.create"); + assertThat(types).doesNotContain("response.create"); + } + + @Test + void sendTextPreservesFullTextInPayload() throws Exception { + final var longText = "a".repeat(500); + build().sendText(longText); + + final var sent = captureAllSentTexts(); + final var conversationCreate = + sent.stream() + .map( + s -> { + try { + return MAPPER.readTree(s); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(n -> "conversation.item.create".equals(n.get("type").asText())) + .findFirst() + .orElseThrow(); + assertThat(conversationCreate.at("/item/content/0/text").asText()).isEqualTo(longText); + } +} diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java new file mode 100644 index 000000000..b6fb22154 --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java @@ -0,0 +1,304 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioConfigOutput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeSessionCreateRequest; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import java.util.Base64; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nonnull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class ToAudioRealtimeClientUnitTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final RealtimeAudioConfigInput FIXED_INPUT_CONFIG = + RealtimeAudioConfigInput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); + + private AudioOutputChannel outputConsumerMock; + + @BeforeEach + void setUp() { + outputConsumerMock = mock(AudioOutputChannel.class); + } + + /** + * Concrete subclass that calls the URL-based constructor with a stub address. {@code buildAsync} + * is non-blocking so the constructor returns immediately without attempting a real connection. + */ + private static class DirectClient extends ToAudioRealtimeClient { + + DirectClient( + final AudioOutputChannel outputConsumer, + final boolean defaultEager, + final RealtimeParam... params) { + super("ws://localhost:0", Map.of(), outputConsumer, defaultEager, params); + } + + @Override + @Nonnull + protected RealtimeAudioConfigInput inputConfig() { + return FIXED_INPUT_CONFIG; + } + } + + private DirectClient build( + final AudioOutputChannel outputConsumer, + final boolean defaultEager, + final RealtimeParam... params) { + return new DirectClient(outputConsumer, defaultEager, params); + } + + @Test + void outputConsumerIsStoredFromConstructor() { + assertThat(build(outputConsumerMock, false).outputConsumer).isSameAs(outputConsumerMock); + } + + @Test + void defaultVoiceIsMarinWhenNoVoiceParamProvided() { + assertThat(build(outputConsumerMock, false).voice) + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + } + + @Test + void voiceIsMarinWhenDefault2Provided() { + assertThat(build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_2).voice) + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + } + + @Test + void voiceIsEchoWhenDefault1Provided() { + assertThat(build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_1).voice) + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); + } + + @Test + void lastVoiceParamWinsWhenMultipleProvided() { + assertThat( + build( + outputConsumerMock, + false, + RealtimeParamVoice.DEFAULT_1, + RealtimeParamVoice.DEFAULT_2) + .voice) + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + } + + @Test + void turnDetectionFollowsDefaultFalseWhenNoParamProvided() { + assertThat(build(outputConsumerMock, false).eagerTurnDetection).isFalse(); + } + + @Test + void turnDetectionFollowsDefaultTrueWhenNoParamProvided() { + assertThat(build(outputConsumerMock, true).eagerTurnDetection).isTrue(); + } + + @Test + void turnDetectionIsEagerWhenEachCallIsATurnParamProvided() { + assertThat( + build(outputConsumerMock, false, RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN) + .eagerTurnDetection) + .isTrue(); + } + + @Test + void turnDetectionIsByModelWhenByModelAutoParamProvided() { + assertThat( + build(outputConsumerMock, true, RealtimeParamTurnDetection.BY_MODEL_AUTO) + .eagerTurnDetection) + .isFalse(); + } + + @Test + void lastTurnDetectionParamWinsWhenMultipleProvided() { + assertThat( + build( + outputConsumerMock, + false, + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN, + RealtimeParamTurnDetection.BY_MODEL_AUTO) + .eagerTurnDetection) + .isFalse(); + } + + @Test + void systemPromptIsEmptyWhenNoParamProvided() { + assertThat(build(outputConsumerMock, false).systemPrompt).isEmpty(); + } + + @Test + void systemPromptIsStoredWhenParamProvided() { + assertThat( + build(outputConsumerMock, false, new RealtimeParamSystemPrompt("You are helpful.")) + .systemPrompt) + .isEqualTo("You are helpful."); + } + + @Test + void lastSystemPromptParamWinsWhenMultipleProvided() { + assertThat( + build( + outputConsumerMock, + false, + new RealtimeParamSystemPrompt("first"), + new RealtimeParamSystemPrompt("second")) + .systemPrompt) + .isEqualTo("second"); + } + + @Test + void getSystemPromptReturnsEmptyWhenSystemPromptIsBlank() { + assertThat(build(outputConsumerMock, false).getSystemPrompt()).isEqualTo(Optional.empty()); + } + + @Test + void getSystemPromptReturnsValueWhenSystemPromptIsSet() { + final var prompt = "You are a helpful assistant."; + assertThat( + build(outputConsumerMock, false, new RealtimeParamSystemPrompt(prompt)) + .getSystemPrompt()) + .isEqualTo(Optional.of(prompt)); + } + + @Test + void onResponseDecodesBase64AudioAndForwardsWithNotDoneFlag() throws Exception { + final var client = build(outputConsumerMock, false); + final byte[] rawAudio = {0x01, 0x02, 0x03}; + final var base64 = Base64.getEncoder().encodeToString(rawAudio); + final var event = + MAPPER.readTree("{\"type\":\"response.output_audio.delta\",\"delta\":\"" + base64 + "\"}"); + + client.onResponse("response.output_audio.delta", event); + + final var captor = ArgumentCaptor.forClass(byte[].class); + verify(outputConsumerMock).outputAudio(captor.capture(), eq(Boolean.FALSE)); + assertThat(captor.getValue()).isEqualTo(rawAudio); + } + + @Test + void onResponseForwardsEmptyBytesWithDoneFlagOnAudioDoneEvent() throws Exception { + final var client = build(outputConsumerMock, false); + final var event = MAPPER.readTree("{\"type\":\"response.output_audio.done\"}"); + + client.onResponse("response.output_audio.done", event); + + final var captor = ArgumentCaptor.forClass(byte[].class); + verify(outputConsumerMock).outputAudio(captor.capture(), eq(Boolean.TRUE)); + assertThat(captor.getValue()).isEmpty(); + } + + @Test + void onResponseDoesNotCallOutputConsumerForUnknownEventType() throws Exception { + final var client = build(outputConsumerMock, false); + final var event = MAPPER.readTree("{\"type\":\"session.created\"}"); + + client.onResponse("session.created", event); + + verifyNoInteractions(outputConsumerMock); + } + + @Test + void sessionConfigurationHasAudioOutputModality() { + final var realtimeSession = + build(outputConsumerMock, false) + .sessionConfiguration() + .session() + .realtimeSessionCreateRequest() + .orElseThrow(); + + assertThat(realtimeSession.outputModalities().orElseThrow()) + .containsExactly(RealtimeSessionCreateRequest.OutputModality.AUDIO); + } + + @Test + void sessionConfigurationOutputFormatIsPcm24000() { + final var output = + build(outputConsumerMock, false) + .sessionConfiguration() + .session() + .realtimeSessionCreateRequest() + .orElseThrow() + .audio() + .orElseThrow() + .output() + .orElseThrow(); + final var format = output.format().orElseThrow().audioPcm().orElseThrow(); + + assertThat(format.type().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM); + assertThat(format.rate().orElseThrow()).isEqualTo(RealtimeAudioFormats.AudioPcm.Rate._24000); + } + + @Test + void sessionConfigurationOutputVoiceIsMarinByDefault() { + final var voice = + build(outputConsumerMock, false) + .sessionConfiguration() + .session() + .realtimeSessionCreateRequest() + .orElseThrow() + .audio() + .orElseThrow() + .output() + .orElseThrow() + .voice() + .orElseThrow() + .unionMember1() + .orElseThrow(); + + assertThat(voice).isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + } + + @Test + void sessionConfigurationOutputVoiceIsEchoWhenDefault1Provided() { + final var voice = + build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_1) + .sessionConfiguration() + .session() + .realtimeSessionCreateRequest() + .orElseThrow() + .audio() + .orElseThrow() + .output() + .orElseThrow() + .voice() + .orElseThrow() + .unionMember1() + .orElseThrow(); + + assertThat(voice).isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); + } + + @Test + void sessionConfigurationInputConfigDelegatestoInputConfig() { + final var inputConfig = + build(outputConsumerMock, false) + .sessionConfiguration() + .session() + .realtimeSessionCreateRequest() + .orElseThrow() + .audio() + .orElseThrow() + .input() + .orElseThrow(); + + assertThat(inputConfig).isEqualTo(FIXED_INPUT_CONFIG); + } +} diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClientUnitTest.java new file mode 100644 index 000000000..42769523b --- /dev/null +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClientUnitTest.java @@ -0,0 +1,454 @@ +package com.sap.ai.sdk.foundationmodels.openai.realtime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.openai.models.realtime.RealtimeAudioConfig; +import com.openai.models.realtime.RealtimeAudioConfigInput; +import com.openai.models.realtime.RealtimeAudioConfigOutput; +import com.openai.models.realtime.RealtimeAudioFormats; +import com.openai.models.realtime.RealtimeAudioInputTurnDetection; +import com.openai.models.realtime.RealtimeSessionCreateRequest; +import com.openai.models.realtime.SessionUpdateEvent; +import com.openai.models.realtime.clientsecrets.ClientSecretCreateParams; +import com.sap.ai.sdk.core.common.ClientException; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.Timer; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class WSOpenAiRealtimeClientUnitTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private List onResponseEventTypes; + private List onResponseEvents; + private SessionUpdateEvent expectedSessionUpdateEvent; + private WebSocket webSocketMock; + + /** + * Concrete testable subclass. It also owns a {@link BufferedWebSocketListener} wired to the same + * text-routing logic (including the handleMessageTypes filter) so we can simulate incoming + * WebSocket frames without a real server. + */ + private class TestableClient extends WSOpenAiRealtimeClient { + + private final Set handledTypes; + private final String systemPrompt; + + /** + * The listener that mirrors what the production URL-based constructor wires up, letting tests + * inject incoming text frames directly. + */ + final BufferedWebSocketListener inboundListener; + + TestableClient( + final CompletableFuture ws, + final Timer timer, + final Set handleMessageTypes, + final String systemPrompt) { + super(mock(HttpClient.class), ws, timer, handleMessageTypes); + this.handledTypes = handleMessageTypes; + this.systemPrompt = systemPrompt; + this.inboundListener = + new BufferedWebSocketListener( + ignored -> {}, // onSocketOpen — not under test here + this::dispatchText); + } + + /** + * Replicates {@code WSOpenAiRealtimeClient.onText}: parse JSON, filter by type, call {@link + * #onResponse}. This is intentionally a thin copy so we can drive routing tests from the test + * package without coupling to the private method. + */ + private void dispatchText(final WebSocket ws, final CharSequence data) { + final JsonNode event; + try { + event = MAPPER.readTree(data.toString()); + } catch (final Exception e) { + throw new ClientException("Error parsing JSON response from speech API", e); + } + final var eventType = event.get("type").asText(); + if (handledTypes.contains(eventType)) { + onResponse(eventType, event); + } + ws.request(1); + } + + @Override + protected synchronized void onResponse( + @Nonnull final String eventType, @Nonnull final JsonNode event) { + onResponseEventTypes.add(eventType); + onResponseEvents.add(event); + } + + @Override + protected synchronized @Nonnull SessionUpdateEvent sessionConfiguration() { + return expectedSessionUpdateEvent; + } + + @Override + @Nonnull + protected Optional getSystemPrompt() { + return systemPrompt.isEmpty() ? Optional.empty() : Optional.of(systemPrompt); + } + + // Expose protected methods for direct invocation in tests + void invokeOnText(final WebSocket ws, final CharSequence data) { + onText(ws, data); + } + + void invokeSendPing(final WebSocket ws) { + sendPing(ws); + } + } + + @BeforeEach + void setUp() { + expectedSessionUpdateEvent = buildSessionUpdateEvent(); + onResponseEventTypes = new ArrayList<>(); + onResponseEvents = new ArrayList<>(); + + webSocketMock = mock(WebSocket.class); + when(webSocketMock.sendText(any(), anyBoolean())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendClose(anyInt(), anyString())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.sendPing(any())) + .thenReturn(CompletableFuture.completedFuture(webSocketMock)); + when(webSocketMock.isInputClosed()).thenReturn(false); + } + + private TestableClient buildInstance(final Set handleMessageTypes) { + return new TestableClient( + CompletableFuture.completedFuture(webSocketMock), + mock(Timer.class), + handleMessageTypes, + ""); + } + + private TestableClient buildInstance(final Timer timer, final String systemPrompt) { + return new TestableClient( + CompletableFuture.completedFuture(webSocketMock), timer, Set.of(), systemPrompt); + } + + private TestableClient buildBrokenInstance() { + final var failedFuture = new CompletableFuture(); + failedFuture.completeExceptionally(new RuntimeException("connection refused")); + return new TestableClient(failedFuture, mock(Timer.class), Set.of(), ""); + } + + @Test + void sendMessageSerializesObjectAsJsonAndSendsOverWebSocket() { + buildInstance(Set.of()).sendMessage(expectedSessionUpdateEvent); + + final var captor = ArgumentCaptor.forClass(CharSequence.class); + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), eq(true)); + final var lastJson = + parseJson(captor.getAllValues().get(captor.getAllValues().size() - 1).toString()); + assertThat(lastJson.get("type").asText()).isEqualTo("session.update"); + verify(webSocketMock, atLeastOnce()).request(anyLong()); + } + + @Test + void sendMessageThrowsClientExceptionWhenConnectionFailed() { + assertThatThrownBy(() -> buildBrokenInstance().sendMessage("anything")) + .isInstanceOf(ClientException.class) + .hasMessageContaining("Failed to establish web socket connection"); + } + + @Test + void askForResponseSendsResponseCreateEvent() { + buildInstance(Set.of()).askForResponse(); + + final var captor = ArgumentCaptor.forClass(CharSequence.class); + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), eq(true)); + final var sentResponseCreate = + captor.getAllValues().stream() + .map(cs -> parseJson(cs.toString())) + .anyMatch(n -> "response.create".equals(n.get("type").asText())); + assertThat(sentResponseCreate).isTrue(); + verify(webSocketMock, atLeastOnce()).request(anyLong()); + } + + @Test + void askForResponseThrowsClientExceptionWhenConnectionFailed() { + assertThatThrownBy(buildBrokenInstance()::askForResponse) + .isInstanceOf(ClientException.class) + .hasMessageContaining("Failed to establish web socket connection"); + } + + @Test + void closeSendsCloseFrameWithNormalClosureCode() { + buildInstance(Set.of()).close(); + + verify(webSocketMock).sendClose(eq(1000), anyString()); + } + + @Test + void closeThrowsClientExceptionWhenConnectionFailed() { + assertThatThrownBy(buildBrokenInstance()::close) + .isInstanceOf(ClientException.class) + .hasMessageContaining("Failed to establish web socket connection"); + } + + @Test + void incomingMessageWithRegisteredTypeIsRoutedToOnResponse() { + final var instance = buildInstance(Set.of("response.audio.delta", "session.created")); + + instance.inboundListener.onText( + webSocketMock, "{\"type\":\"response.audio.delta\",\"delta\":\"abc\"}", true); + + assertThat(onResponseEventTypes).containsExactly("response.audio.delta"); + assertThat(onResponseEvents).hasSize(1); + assertThat(onResponseEvents.get(0).get("delta").asText()).isEqualTo("abc"); + } + + @Test + void incomingMessageWithUnregisteredTypeIsIgnored() { + final var instance = buildInstance(Set.of("session.created")); + + instance.inboundListener.onText(webSocketMock, "{\"type\":\"some.other.event\"}", true); + + assertThat(onResponseEventTypes).isEmpty(); + assertThat(onResponseEvents).isEmpty(); + } + + @Test + void multipleMatchingIncomingMessagesAreAllDelivered() { + final var instance = buildInstance(Set.of("response.audio.delta")); + + instance.inboundListener.onText( + webSocketMock, "{\"type\":\"response.audio.delta\",\"delta\":\"chunk1\"}", true); + instance.inboundListener.onText( + webSocketMock, "{\"type\":\"response.audio.delta\",\"delta\":\"chunk2\"}", true); + + assertThat(onResponseEventTypes) + .containsExactly("response.audio.delta", "response.audio.delta"); + assertThat(onResponseEvents.get(0).get("delta").asText()).isEqualTo("chunk1"); + assertThat(onResponseEvents.get(1).get("delta").asText()).isEqualTo("chunk2"); + } + + @Test + void bufferedListenerAssemblesPartialFramesBeforeDelivery() { + final var instance = buildInstance(Set.of("session.created")); + + instance.inboundListener.onText(webSocketMock, "{\"type\":\"session", false); + assertThat(onResponseEventTypes).isEmpty(); + + instance.inboundListener.onText(webSocketMock, ".created\",\"id\":\"xyz\"}", true); + assertThat(onResponseEventTypes).containsExactly("session.created"); + } + + @Test + void incomingMalformedJsonThrowsClientException() { + final var instance = buildInstance(Set.of()); + + assertThatThrownBy( + () -> instance.inboundListener.onText(webSocketMock, "not-valid-json{{{", true)) + .isInstanceOf(ClientException.class) + .hasMessageContaining("Error parsing JSON response from speech API"); + } + + @Test + void onSocketOpenSendsSessionConfigurationToWebSocket() { + final var timerMock = mock(Timer.class); + final var instance = buildInstance(timerMock, ""); + + instance.onSocketOpen(webSocketMock); + + final var captor = ArgumentCaptor.forClass(CharSequence.class); + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), eq(true)); + final var sentTypes = + captor.getAllValues().stream() + .map(cs -> parseJson(cs.toString()).get("type").asText()) + .toList(); + assertThat(sentTypes).contains("session.update"); + } + + @Test + void onSocketOpenSkipsConversationItemWhenSystemPromptIsEmpty() { + final var timerMock = mock(Timer.class); + final var instance = buildInstance(timerMock, ""); + + instance.onSocketOpen(webSocketMock); + + final var captor = ArgumentCaptor.forClass(CharSequence.class); + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), eq(true)); + final var sentTypes = + captor.getAllValues().stream() + .map(cs -> parseJson(cs.toString()).get("type").asText()) + .toList(); + assertThat(sentTypes).doesNotContain("conversation.item.create"); + } + + @Test + void onSocketOpenSendsSystemPromptAsConversationItemWhenPresent() { + final var timerMock = mock(Timer.class); + final var systemPrompt = "You are a helpful assistant."; + final var instance = buildInstance(timerMock, systemPrompt); + + instance.onSocketOpen(webSocketMock); + + final var captor = ArgumentCaptor.forClass(CharSequence.class); + verify(webSocketMock, atLeastOnce()).sendText(captor.capture(), eq(true)); + final var conversationItemCreate = + captor.getAllValues().stream() + .map(cs -> parseJson(cs.toString())) + .filter(n -> "conversation.item.create".equals(n.get("type").asText())) + .findFirst(); + assertThat(conversationItemCreate).isPresent(); + final var contentText = conversationItemCreate.get().at("/item/content/0/text").asText(); + assertThat(contentText).isEqualTo(systemPrompt); + } + + @Test + void onSocketOpenSchedulesHeartbeatTimer() { + final var timerMock = mock(Timer.class); + final var instance = buildInstance(timerMock, ""); + + instance.onSocketOpen(webSocketMock); + + verify(timerMock).scheduleAtFixedRate(any(java.util.TimerTask.class), eq(0L), eq(4500L)); + } + + @Test + void onTextRoutesRegisteredEventTypeToOnResponse() { + final var instance = buildInstance(Set.of("session.created")); + + instance.invokeOnText(webSocketMock, "{\"type\":\"session.created\",\"id\":\"xyz\"}"); + + assertThat(onResponseEventTypes).containsExactly("session.created"); + assertThat(onResponseEvents.get(0).get("id").asText()).isEqualTo("xyz"); + } + + @Test + void onTextDoesNotCallOnResponseForUnregisteredType() { + final var instance = buildInstance(Set.of("session.created")); + + instance.invokeOnText(webSocketMock, "{\"type\":\"response.audio.delta\"}"); + + assertThat(onResponseEventTypes).isEmpty(); + } + + @Test + void onTextAlwaysRequestsNextMessageRegardlessOfType() { + final var instance = buildInstance(Set.of()); + + instance.invokeOnText(webSocketMock, "{\"type\":\"any.event\"}"); + + verify(webSocketMock).request(1L); + } + + @Test + void onTextThrowsClientExceptionForMalformedJson() { + final var instance = buildInstance(Set.of()); + + assertThatThrownBy(() -> instance.invokeOnText(webSocketMock, "not-json{{")) + .isInstanceOf(ClientException.class) + .hasMessageContaining("Error parsing JSON response from speech API"); + } + + @Test + void sendPingSendsPingBytesAndRequestsNextMessageWhenInputOpen() { + when(webSocketMock.isInputClosed()).thenReturn(false); + final var instance = buildInstance(Set.of()); + + instance.invokeSendPing(webSocketMock); + + final var captor = ArgumentCaptor.forClass(java.nio.ByteBuffer.class); + verify(webSocketMock).sendPing(captor.capture()); + assertThat(new String(captor.getValue().array(), java.nio.charset.StandardCharsets.UTF_8)) + .isEqualTo("ping"); + verify(webSocketMock).request(1L); + } + + @Test + void sendPingDoesNothingWhenInputIsClosed() { + when(webSocketMock.isInputClosed()).thenReturn(true); + final var instance = buildInstance(Set.of()); + + instance.invokeSendPing(webSocketMock); + + verify(webSocketMock, never()).sendPing(any()); + verify(webSocketMock, never()).request(anyLong()); + } + + @Test + void getSystemPromptReturnsEmptyByDefault() { + assertThat(buildInstance(Set.of()).getSystemPrompt()).isEqualTo(Optional.empty()); + } + + @Test + void sessionConfigurationReturnsTheConfiguredEvent() { + assertThat(buildInstance(Set.of()).sessionConfiguration()).isSameAs(expectedSessionUpdateEvent); + } + + private JsonNode parseJson(final String json) { + try { + return MAPPER.readTree(json); + } catch (final Exception e) { + throw new RuntimeException("Failed to parse JSON in test: " + json, e); + } + } + + private SessionUpdateEvent buildSessionUpdateEvent() { + final var input = + RealtimeAudioConfigInput.builder() + .turnDetection( + RealtimeAudioInputTurnDetection.ofSemanticVad( + RealtimeAudioInputTurnDetection.SemanticVad.builder().build())) + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .build(); + + final var output = + RealtimeAudioConfigOutput.builder() + .format( + RealtimeAudioFormats.AudioPcm.builder() + .type(RealtimeAudioFormats.AudioPcm.Type.AUDIO_PCM) + .rate(RealtimeAudioFormats.AudioPcm.Rate._24000) + .build()) + .voice(UUID.randomUUID().toString()) + .build(); + + return SessionUpdateEvent.builder() + .session( + ClientSecretCreateParams.Session.ofRealtime( + RealtimeSessionCreateRequest.builder() + .outputModalities( + Arrays.asList( + RealtimeSessionCreateRequest.OutputModality.AUDIO, + RealtimeSessionCreateRequest.OutputModality.TEXT)) + .audio(RealtimeAudioConfig.builder().input(input).output(output).build()) + .build()) + .asRealtime()) + .build(); + } +} From d9d482d3a267cdcf7ac028bbfe1d65b8fb83d38e Mon Sep 17 00:00:00 2001 From: vladimir-a-sap Date: Fri, 31 Jul 2026 10:31:53 +0200 Subject: [PATCH 64/67] Apply suggestions from code review Co-authored-by: Jonas-Isr --- .../openai/realtime/OpenAiRealtimeClient.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index 36e60c3e9..e86e9f25a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -31,7 +31,7 @@ public OpenAiRealtimeClient(@Nonnull final Destination destination) { } /** - * Creates realtime channel allowing to input text and voice it (receive audio output) + * Creates a realtime channel allowing to input text and voice it (receive audio output) * *

    The input channel should be used with a try-with-resources block to ensure that the * underlying connection is closed. @@ -46,10 +46,10 @@ public OpenAiRealtimeClient(@Nonnull final Destination destination) { * } * * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. + * logically synchronize their state and close the input channel when it is appropriate (e.g. the last + * part of the response has been received via the output channel and the application does not need to send + * any other input). When the input channel is closed, the output channel will be closed automatically and + * the output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth @@ -67,7 +67,7 @@ public TextInputChannel textToSpeech( } /** - * Creates realtime channel allowing for audio conversation with a model + * Creates a realtime channel allowing for audio conversation with a model. * *

    The input channel should be used with a try-with-resources block to ensure that the * underlying connection is closed. @@ -81,11 +81,11 @@ public TextInputChannel textToSpeech( * } * } * - * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close input channel when it is appropriate (e.g. last - * part of the response has been received via output channel and application does not need to send - * any other input). When input channel is closed, output channel will be closed automatically and - * output consumer will not be called anymore. + * This API implements full duplex (input + output) communication channels. An application should + * logically synchronize their state and close the input channel when it is appropriate (e.g. the last + * part of the response has been received via the output channel and the application does not need to send + * any other input). When the input channel is closed, the output channel will be closed automatically and + * the output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth From 7044a10902d2ca3b3743ea1a308b91fc5eb3a3a8 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Fri, 31 Jul 2026 08:32:42 +0000 Subject: [PATCH 65/67] Formatting --- .../openai/realtime/OpenAiRealtimeClient.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index e86e9f25a..a5ac809d3 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -46,10 +46,10 @@ public OpenAiRealtimeClient(@Nonnull final Destination destination) { * } * * This API implements full duplex (input + output) communication channels. Application should - * logically synchronize their state and close the input channel when it is appropriate (e.g. the last - * part of the response has been received via the output channel and the application does not need to send - * any other input). When the input channel is closed, the output channel will be closed automatically and - * the output consumer will not be called anymore. + * logically synchronize their state and close the input channel when it is appropriate (e.g. the + * last part of the response has been received via the output channel and the application does not + * need to send any other input). When the input channel is closed, the output channel will be + * closed automatically and the output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth @@ -82,10 +82,10 @@ public TextInputChannel textToSpeech( * } * * This API implements full duplex (input + output) communication channels. An application should - * logically synchronize their state and close the input channel when it is appropriate (e.g. the last - * part of the response has been received via the output channel and the application does not need to send - * any other input). When the input channel is closed, the output channel will be closed automatically and - * the output consumer will not be called anymore. + * logically synchronize their state and close the input channel when it is appropriate (e.g. the + * last part of the response has been received via the output channel and the application does not + * need to send any other input). When the input channel is closed, the output channel will be + * closed automatically and the output consumer will not be called anymore. * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth From 4e4f357295b266df73a3790b6373062ece4bd1d7 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 31 Jul 2026 10:32:54 +0200 Subject: [PATCH 66/67] polish code style --- .../openai/TextOutputChannel.java | 21 --- .../realtime/BufferedWebSocketListener.java | 4 +- .../openai/realtime/OpenAiRealtimeClient.java | 4 +- .../openai/realtime/RealtimeParam.java | 3 + .../realtime/RealtimeParamSystemPrompt.java | 8 ++ .../realtime/RealtimeParamTurnDetection.java | 11 +- .../openai/realtime/RealtimeParamVoice.java | 15 +- .../realtime/ToAudioRealtimeClient.java | 133 +++++++++--------- .../realtime/WSOpenAiRealtimeClient.java | 1 - .../OpenAiRealtimeClientUnitTest.java | 2 +- .../realtime/RealtimeParamVoiceUnitTest.java | 4 +- ...> TextToSpeechRealtimeClientUnitTest.java} | 2 +- .../ToAudioRealtimeClientUnitTest.java | 14 +- .../com/sap/ai/sdk/app/WebsocketConfig.java | 4 + .../sdk/app/controllers/OpenAiController.java | 80 +++++++++++ .../SpeechToSpeechWebsocketHandler.java | 4 + .../TextToSpeechWebsocketHandler.java | 4 + .../ai/sdk/app/services/OpenAiService.java | 8 +- .../src/main/resources/static/index.html | 88 +++++++++++- .../src/main/resources/static/question.pcm | Bin 0 -> 97200 bytes .../resources/static/realtime-api-scheme.svg | 3 + .../resources/static/show-realtime-diagram.js | 20 +++ .../main/resources/static/speech-to-speech.js | 8 +- 23 files changed, 325 insertions(+), 116 deletions(-) delete mode 100644 foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java rename foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/{TextToSpeechRealtimeClientTest.java => TextToSpeechRealtimeClientUnitTest.java} (99%) create mode 100644 sample-code/spring-app/src/main/resources/static/question.pcm create mode 100644 sample-code/spring-app/src/main/resources/static/realtime-api-scheme.svg create mode 100644 sample-code/spring-app/src/main/resources/static/show-realtime-diagram.js diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java deleted file mode 100644 index 973d9a151..000000000 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextOutputChannel.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.sap.ai.sdk.foundationmodels.openai; - -import com.google.common.annotations.Beta; -import javax.annotation.Nonnull; - -/** Functional interface representing text output channel (text data consumer) */ -@Beta -public interface TextOutputChannel { - - /** - * This method is sequentially invoked by text data provider to supply implementer (consumer) with - * the text data. - * - * @param textChunk chunk of the text (possibly partial content) - * @param isLast true if this call logically concludes previous and this passed text data into a - * single logical entity (e.g. gets called at the end when all parts of a single message get - * passed) - */ - @Beta - void outputText(@Nonnull final CharSequence textChunk, final boolean isLast); -} diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java index 82d59429a..390a1d63b 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java @@ -7,6 +7,8 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nonnull; + +import com.google.common.annotations.Beta; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -21,7 +23,7 @@ class BufferedWebSocketListener implements WebSocket.Listener { @Nonnull final BiConsumer onText) { this.onOpen = onOpen; this.onText = onText; - this.buffer = new StringBuilder(128 * 1024); + this.buffer = new StringBuilder(256 * 1024); } @Override diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java index 36e60c3e9..85f0fa962 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java @@ -14,6 +14,7 @@ * OpenAI client implementation of Realtime API. Abstracts technical implementation, transport and * threading and exposes business-level operations (high level interface) */ +@Beta public class OpenAiRealtimeClient { static final int PATH_BUFFER_SIZE = @@ -26,6 +27,7 @@ public class OpenAiRealtimeClient { * * @param destination - destination to use */ + @Beta public OpenAiRealtimeClient(@Nonnull final Destination destination) { this.destination = destination; } @@ -120,7 +122,7 @@ String getRealtimeEndpoint() { "Invalid destination URI: " + destination.asHttp().getUri()); } sb.append(pathParts[1].replaceFirst("^api\\.", "realtime.")); - sb.append("v1/realtime"); + sb.append("/v1/realtime"); return sb.toString(); } } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java index 76fe3e656..2909271ab 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java @@ -1,8 +1,11 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; +import com.google.common.annotations.Beta; + import javax.annotation.Nonnull; /** Represents possible configuration params of realtime client */ +@Beta public interface RealtimeParam { /** Represents configurable options */ enum ParamName { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java index a6d08b144..7b0fc910f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java @@ -1,10 +1,13 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; +import com.google.common.annotations.Beta; + import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** Allows to configure model system prompt */ +@Beta public final class RealtimeParamSystemPrompt implements RealtimeParam { private final String systemPrompt; @@ -14,21 +17,25 @@ public final class RealtimeParamSystemPrompt implements RealtimeParam { * * @param systemPrompt system prompt to use */ + @Beta public RealtimeParamSystemPrompt(@Nonnull final String systemPrompt) { this.systemPrompt = systemPrompt; } @Override + @Beta public @Nonnull ParamName getParamName() { return ParamName.SYSTEM_PROMPT; } @Override + @Beta public @Nonnull String getValueAsString() { return systemPrompt; } @Override + @Beta public boolean equals(@Nullable final Object o) { if (o == null || getClass() != o.getClass()) { return false; @@ -38,6 +45,7 @@ public boolean equals(@Nullable final Object o) { } @Override + @Beta public int hashCode() { return Objects.hashCode(systemPrompt); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java index c0dfddbfa..980d9479f 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java @@ -1,14 +1,17 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; +import com.google.common.annotations.Beta; + import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** Allows to configure turn detection (how model responds). */ +@Beta public final class RealtimeParamTurnDetection implements RealtimeParam { /** Model tries to recognize if/when it should respond automatically */ - public static final RealtimeParamTurnDetection BY_MODEL_AUTO = + @Beta public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); /** @@ -16,7 +19,7 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { * Less convenient than the automatic option but may give lower latency in some cases (model does * not need to perform additional turn detection analysis). */ - public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = + @Beta public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); private final String turnDetectionKind; @@ -26,16 +29,19 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { } @Override + @Beta public @Nonnull ParamName getParamName() { return ParamName.TURN_DETECTION; } @Override + @Beta public @Nonnull String getValueAsString() { return turnDetectionKind; } @Override + @Beta public boolean equals(@Nullable final Object o) { if (o == null || getClass() != o.getClass()) { return false; @@ -45,6 +51,7 @@ public boolean equals(@Nullable final Object o) { } @Override + @Beta public int hashCode() { return Objects.hashCode(turnDetectionKind); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java index 4b28fd561..bcc5030af 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java @@ -1,17 +1,20 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; +import com.google.common.annotations.Beta; + import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** Allows to configure model output voice */ +@Beta public final class RealtimeParamVoice implements RealtimeParam { /** Standard voice 1 */ - public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); + @Beta public static final RealtimeParamVoice DEFAULT_1 = new RealtimeParamVoice("DEFAULT_1"); /** Standard voice 2 */ - public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); + @Beta public static final RealtimeParamVoice DEFAULT_2 = new RealtimeParamVoice("DEFAULT_2"); private final String voice; @@ -22,26 +25,31 @@ public final class RealtimeParamVoice implements RealtimeParam { /** * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify * in advance if the provided voice name is correct and supported by the chosen model and use case + * NOTE: this method does not check voice name and incorrect input may produce runtime exceptions (unsafe) * * @param voiceName as named by model provider * @return typed voice client configuration param */ @Nonnull - public static RealtimeParamVoice unsafeWithExplicitVoice(@Nonnull final String voiceName) { + @Beta + public static RealtimeParamVoice withExplicitVoice(@Nonnull final String voiceName) { return new RealtimeParamVoice(voiceName); } @Override + @Beta public @Nonnull ParamName getParamName() { return ParamName.OUTPUT_VOICE; } @Override + @Beta public @Nonnull String getValueAsString() { return voice; } @Override + @Beta public boolean equals(@Nullable final Object o) { if (o == null || getClass() != o.getClass()) { return false; @@ -51,6 +59,7 @@ public boolean equals(@Nullable final Object o) { } @Override + @Beta public int hashCode() { return Objects.hashCode(voice); } diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index e5da6dc7f..3416030d5 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -1,6 +1,7 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.Beta; import com.openai.models.realtime.RealtimeAudioConfig; import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioConfigOutput; @@ -12,6 +13,7 @@ import java.net.http.HttpClient; import java.net.http.WebSocket; import java.util.Base64; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,13 +33,16 @@ abstract class ToAudioRealtimeClient extends WSOpenAiRealtimeClient { List.of(RealtimeSessionCreateRequest.OutputModality.AUDIO); private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private static final Map FALLBACK_DEFAULT_PARAMS = + Map.of( + RealtimeParam.ParamName.OUTPUT_VOICE, RealtimeParamVoice.DEFAULT_1, + RealtimeParam.ParamName.TURN_DETECTION, RealtimeParamTurnDetection.BY_MODEL_AUTO, + RealtimeParam.ParamName.SYSTEM_PROMPT, new RealtimeParamSystemPrompt("")); + final AudioOutputChannel outputConsumer; final RealtimeAudioConfigOutput.Voice.UnionMember1 voice; - /** - * defines default eagerness (if EACH_CALL_IS_A_TURN) behavior if specific turn detection config - * not provided - */ + /** defines if every call to the client should be considered conversation turn */ protected final boolean eagerTurnDetection; final String systemPrompt; @@ -59,39 +64,22 @@ public ToAudioRealtimeClient( final boolean defaultTurnDetectionEager, @Nonnull final RealtimeParam... params) { super(url, httpHeaders, HANDLED_RESPONSE_TYPES); - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.OUTPUT_VOICE) { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - } - - var turnDetectionEager = defaultTurnDetectionEager; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - - var systemPrompt = ""; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.SYSTEM_PROMPT) { - systemPrompt = param.getValueAsString(); - } - } + final var defaults = new HashMap<>(FALLBACK_DEFAULT_PARAMS); + defaults.put( + RealtimeParam.ParamName.TURN_DETECTION, + defaultTurnDetectionEager + ? RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN + : RealtimeParamTurnDetection.BY_MODEL_AUTO); + final var resolvedParams = resolveParams(defaults, params); this.outputConsumer = outputConsumer; - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; - this.systemPrompt = systemPrompt; + this.voice = + mapVoice((RealtimeParamVoice) resolvedParams.get(RealtimeParam.ParamName.OUTPUT_VOICE)); + this.eagerTurnDetection = + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals( + resolvedParams.get(RealtimeParam.ParamName.TURN_DETECTION)); + this.systemPrompt = + resolvedParams.get(RealtimeParam.ParamName.SYSTEM_PROMPT).getValueAsString(); } ToAudioRealtimeClient( @@ -102,36 +90,21 @@ public ToAudioRealtimeClient( final boolean eagerTurnDetection, @Nonnull final RealtimeParam... params) { super(httpClient, ws, timer, HANDLED_RESPONSE_TYPES); - var voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.OUTPUT_VOICE) { - if (RealtimeParamVoice.DEFAULT_2.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; - } else if (RealtimeParamVoice.DEFAULT_1.equals(param)) { - voice = RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; - } - } - } - var turnDetectionEager = eagerTurnDetection; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.TURN_DETECTION) { - if (RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals(param)) { - turnDetectionEager = true; - } else if (RealtimeParamTurnDetection.BY_MODEL_AUTO.equals(param)) { - turnDetectionEager = false; - } - } - } - var systemPrompt = ""; - for (final RealtimeParam param : params) { - if (param.getParamName() == RealtimeParam.ParamName.SYSTEM_PROMPT) { - systemPrompt = param.getValueAsString(); - } - } + final var defaults = new HashMap<>(FALLBACK_DEFAULT_PARAMS); + defaults.put( + RealtimeParam.ParamName.TURN_DETECTION, + eagerTurnDetection + ? RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN + : RealtimeParamTurnDetection.BY_MODEL_AUTO); + final var resolvedParams = resolveParams(defaults, params); this.outputConsumer = outputConsumer; - this.voice = voice; - this.eagerTurnDetection = turnDetectionEager; - this.systemPrompt = systemPrompt; + this.voice = + mapVoice((RealtimeParamVoice) resolvedParams.get(RealtimeParam.ParamName.OUTPUT_VOICE)); + this.eagerTurnDetection = + RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN.equals( + resolvedParams.get(RealtimeParam.ParamName.TURN_DETECTION)); + this.systemPrompt = + resolvedParams.get(RealtimeParam.ParamName.SYSTEM_PROMPT).getValueAsString(); } @Nonnull @@ -148,14 +121,42 @@ protected void onResponse(@Nonnull final String eventType, @Nonnull final JsonNo if ("response.output_audio.delta".equals(eventType)) { final var base64Audio = event.get("delta").asText(); final byte[] audio = Base64.getDecoder().decode(base64Audio); - this.outputConsumer.outputAudio(audio, Boolean.FALSE); + this.outputConsumer.outputAudio(audio, false); } else if ("response.output_audio.done".equals(eventType)) { - this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, Boolean.TRUE); + this.outputConsumer.outputAudio(EMPTY_BYTE_ARRAY, true); } else { log.warn("skipping message type: {}", eventType); } } + /* + side effect: modifies defaults + */ + @Nonnull + protected Map resolveParams( + final @Nonnull Map defaults, + final @Nonnull RealtimeParam... params) { + for (final RealtimeParam param : params) { + if (param == null) { + log.warn("skipping null param for realtime client"); + continue; + } + defaults.put(param.getParamName(), param); + } + return defaults; + } + + @Nonnull + private RealtimeAudioConfigOutput.Voice.UnionMember1 mapVoice( + @Nonnull final RealtimeParamVoice voice) { + if (voice.equals(RealtimeParamVoice.DEFAULT_1)) { + return RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN; + } else if (voice.equals(RealtimeParamVoice.DEFAULT_2)) { + return RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO; + } + return RealtimeAudioConfigOutput.Voice.UnionMember1.of(voice.getValueAsString()); + } + @Override @Nonnull protected SessionUpdateEvent sessionConfiguration() { diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java index 30b75eca4..a5c44e4f4 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/WSOpenAiRealtimeClient.java @@ -114,7 +114,6 @@ public void close() { protected Optional getSystemPrompt() { return Optional.empty(); } - ; protected abstract void onResponse( @Nonnull final String eventType, @Nonnull final JsonNode event); diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java index d1201f40e..f6bd39376 100644 --- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClientUnitTest.java @@ -67,7 +67,7 @@ void getRealtimeEndpointBuildsWssUrlAndAppendsSuffix() { .build(); final var client = new OpenAiRealtimeClient(destination); - assertThat(client.getRealtimeEndpoint()).startsWith("wss://").endsWith("v1/realtime"); + assertThat(client.getRealtimeEndpoint()).startsWith("wss://").endsWith("/v1/realtime"); } @Test diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java index 99164fcea..a441328a6 100644 --- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoiceUnitTest.java @@ -18,8 +18,8 @@ void setUp() { } @Test - void unsafeWithExplicitVoice() { - assertThat(random).isEqualTo(RealtimeParamVoice.unsafeWithExplicitVoice(expectedRandomValue)); + void withExplicitVoice() { + assertThat(random).isEqualTo(RealtimeParamVoice.withExplicitVoice(expectedRandomValue)); } @Test diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientUnitTest.java similarity index 99% rename from foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java rename to foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientUnitTest.java index bec50d848..5addcf54b 100644 --- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/TextToSpeechRealtimeClientUnitTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -class TextToSpeechRealtimeClientTest { +class TextToSpeechRealtimeClientUnitTest { private static final ObjectMapper MAPPER = new ObjectMapper(); diff --git a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java index b6fb22154..34b36f59a 100644 --- a/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java +++ b/foundation-models/openai/src/test/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClientUnitTest.java @@ -78,15 +78,15 @@ void defaultVoiceIsMarinWhenNoVoiceParamProvided() { } @Test - void voiceIsMarinWhenDefault2Provided() { + void voiceIsEchoWhenDefault2Provided() { assertThat(build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_2).voice) - .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); } @Test - void voiceIsEchoWhenDefault1Provided() { + void voiceIsMarinWhenDefault1Provided() { assertThat(build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_1).voice) - .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); } @Test @@ -98,7 +98,7 @@ void lastVoiceParamWinsWhenMultipleProvided() { RealtimeParamVoice.DEFAULT_1, RealtimeParamVoice.DEFAULT_2) .voice) - .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); + .isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); } @Test @@ -267,7 +267,7 @@ void sessionConfigurationOutputVoiceIsMarinByDefault() { } @Test - void sessionConfigurationOutputVoiceIsEchoWhenDefault1Provided() { + void sessionConfigurationOutputVoiceIsMarinWhenDefault1Provided() { final var voice = build(outputConsumerMock, false, RealtimeParamVoice.DEFAULT_1) .sessionConfiguration() @@ -283,7 +283,7 @@ void sessionConfigurationOutputVoiceIsEchoWhenDefault1Provided() { .unionMember1() .orElseThrow(); - assertThat(voice).isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.ECHO); + assertThat(voice).isEqualTo(RealtimeAudioConfigOutput.Voice.UnionMember1.MARIN); } @Test diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index 0b8a56d22..a93004d48 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app; +import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; import javax.annotation.Nonnull; @@ -11,6 +12,7 @@ /** Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API */ @Configuration @EnableWebSocket +@Beta public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; @@ -22,6 +24,7 @@ public class WebsocketConfig implements WebSocketConfigurer { * @param textToSpeech - text to speech realtime api handler * @param speechToSpeech - speech to speech realtime api handler */ + @Beta public WebsocketConfig( @Nonnull final TextToSpeechWebsocketHandler textToSpeech, @Nonnull final SpeechToSpeechWebsocketHandler speechToSpeech) { @@ -35,6 +38,7 @@ public WebsocketConfig( * @param registry - registry where to register handlers */ @Override + @Beta public void registerWebSocketHandlers(@Nonnull final WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech"); registry.addHandler(speechToSpeech, "/speech-to-speech"); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java index a15031ddb..446ad5061 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java @@ -3,15 +3,26 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; +import com.sap.ai.sdk.core.common.ClientException; +import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel; +import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import com.sap.ai.sdk.foundationmodels.openai.generated.model.CompletionUsage; +import com.sap.ai.sdk.foundationmodels.openai.realtime.RealtimeParamTurnDetection; import com.sap.cloud.sdk.cloudplatform.thread.ThreadContextExecutors; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -30,6 +41,8 @@ public class OpenAiController { private static final ObjectMapper MAPPER = new ObjectMapper().setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + private Resource sampleQuestionPcm = new ClassPathResource("static/question.pcm"); + @GetMapping("/chatCompletion") @Nonnull Object chatCompletion( @@ -41,6 +54,73 @@ Object chatCompletion( return response.getContent(); } + @GetMapping(value = "/realtime/smokeTestTextToSpeech", produces = "audio/pcm") + @Nonnull + @Beta + ResponseEntity smokeTestTextToSpeech() throws TimeoutException { + final var respBody = ByteBuffer.allocate(300000); + final var lock = new AtomicBoolean(false); + final AudioOutputChannel audioOutput = + (final byte[] pcmBytes, final boolean isLast) -> { + respBody.put(pcmBytes); + if (isLast) { + lock.set(true); + } + }; + + try (TextInputChannel channel = service.textToSpeech(audioOutput)) { + channel.sendText("Hello, how are you today?"); + final var started = System.currentTimeMillis(); + while (!lock.get() && System.currentTimeMillis() - started < 15000) { + if (lock.get()) { + return ResponseEntity.ok(respBody.array()); + } + Thread.sleep(1000); + } + } catch (final Exception e) { + throw new ClientException("Failure occurred during communication with the server", e); + } + + if (lock.get()) { + return ResponseEntity.ok(respBody.array()); + } + throw new TimeoutException("Timeout waiting for text to speech"); + } + + @GetMapping(value = "/realtime/smokeTestSpeechToSpeech", produces = "audio/pcm") + @Nonnull + @Beta + ResponseEntity smokeTestSpeechToSpeech() throws TimeoutException { + final var respBody = ByteBuffer.allocate(800000); + final var lock = new AtomicBoolean(false); + final AudioOutputChannel audioOutput = + (final byte[] pcmBytes, final boolean isLast) -> { + respBody.put(pcmBytes); + if (isLast) { + lock.set(true); + } + }; + + try (AudioInputChannel channel = + service.speechToSpeech(audioOutput, RealtimeParamTurnDetection.EACH_CALL_IS_A_TURN)) { + channel.inputAudio(sampleQuestionPcm.getContentAsByteArray()); + final var started = System.currentTimeMillis(); + while (!lock.get() && System.currentTimeMillis() - started < 30000) { + if (lock.get()) { + return ResponseEntity.ok(respBody.array()); + } + Thread.sleep(1000); + } + } catch (final Exception e) { + throw new ClientException("Failure occurred during communication with the server", e); + } + + if (lock.get()) { + return ResponseEntity.ok(respBody.array()); + } + throw new TimeoutException("Timeout waiting for speech to speech"); + } + @SuppressWarnings("unused") // The end-to-end test doesn't use this method @GetMapping("/streamChatCompletionDeltas") @Nonnull diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 3a43b191d..52b7938d9 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app.realtime; +import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import java.io.IOException; @@ -18,6 +19,7 @@ /** Implements handler (Web Socket messages handling) for speech to speech realtime api operation */ @Component @Slf4j +@Beta public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; @@ -29,6 +31,7 @@ public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired + @Beta public SpeechToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); @@ -58,6 +61,7 @@ protected void handleBinaryMessage( } @Override + @Beta public void afterConnectionClosed( @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent( diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index c15f13a03..7dba3ca8c 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app.realtime; +import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.IOException; @@ -19,6 +20,7 @@ /** Implements handler (Web Socket messages handling) for text to speech realtime api operation */ @Component @Slf4j +@Beta public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; @@ -30,6 +32,7 @@ public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired + @Beta public TextToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); @@ -59,6 +62,7 @@ protected void handleBinaryMessage( } @Override + @Beta public void afterConnectionClosed( @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent( diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java index dbe16d8db..a3a9bf508 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/services/OpenAiService.java @@ -17,6 +17,7 @@ import com.sap.ai.sdk.foundationmodels.openai.OpenAiMessage; import com.sap.ai.sdk.foundationmodels.openai.OpenAiTool; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; +import com.sap.ai.sdk.foundationmodels.openai.realtime.RealtimeParam; import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; @@ -129,12 +130,15 @@ public TextInputChannel textToSpeech(@Nonnull final AudioOutputChannel audioOutp * * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16 * bit depth + * @param realtimeParams - optional additional configuration params * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16 * bit) */ @Nonnull - public AudioInputChannel speechToSpeech(@Nonnull final AudioOutputChannel audioOutputConsumer) { - return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer); + public AudioInputChannel speechToSpeech( + @Nonnull final AudioOutputChannel audioOutputConsumer, + @Nonnull final RealtimeParam... realtimeParams) { + return OpenAiClient.realtimeClient().speechToSpeech(audioOutputConsumer, realtimeParams); } /** diff --git a/sample-code/spring-app/src/main/resources/static/index.html b/sample-code/spring-app/src/main/resources/static/index.html index 8c9c7f2b3..6c1e668f8 100644 --- a/sample-code/spring-app/src/main/resources/static/index.html +++ b/sample-code/spring-app/src/main/resources/static/index.html @@ -82,6 +82,78 @@ max-height: 500px; overflow-y: auto; } + + .modal { + display: none; /* Hidden by default */ + position: fixed; /* Stay in place */ + z-index: 1; /* Sit on top */ + padding-top: 100px; /* Location of the box */ + left: 0; + top: 0; + width: 100%; /* Full width */ + height: 100%; /* Full height */ + overflow: auto; /* Enable scroll if needed */ + background-color: rgb(0,0,0); /* Fallback color */ + background-color: rgba(0,0,0,0.9); /* Black w/ opacity */ + } + + .modal-content { + margin: auto; + display: block; + width: 80%; + max-width: 700px; + } + + #modal-caption { + margin: auto; + display: block; + width: 80%; + max-width: 700px; + text-align: center; + color: #ccc; + padding: 10px 0; + height: 150px; + } + + .modal-content, #modal-caption { + -webkit-animation-name: zoom; + -webkit-animation-duration: 0.6s; + animation-name: zoom; + animation-duration: 0.6s; + } + + @-webkit-keyframes zoom { + from {-webkit-transform:scale(0)} + to {-webkit-transform:scale(1)} + } + + @keyframes zoom { + from {transform:scale(0)} + to {transform:scale(1)} + } + + .modal-close-button { + position: absolute; + top: 15px; + right: 35px; + color: #f1f1f1; + font-size: 40px; + font-weight: bold; + transition: 0.3s; + } + + .modal-close-button:hover, + .modal-close-button:focus { + color: #bbb; + text-decoration: none; + cursor: pointer; + } + + @media only screen and (max-width: 700px){ + .modal-content { + width: 100%; + } + } @@ -1494,12 +1566,20 @@

    ⏰ Realtime API


    In these examples, web socket sessions are created and used to send text/audio to the local server and receive audio back +
    + + +
    • -

      Text to speech

      +

      Text to speech (smoke-test)

      disabled
      @@ -1514,9 +1594,9 @@

      Text to speech

    • -

      Speech to speech

      -
      disconnected
      -
      disabled
      +

      Speech to speech (smoke-test)

      +
      socket: disconnected
      +
      microphone: disabled
      diff --git a/sample-code/spring-app/src/main/resources/static/question.pcm b/sample-code/spring-app/src/main/resources/static/question.pcm new file mode 100644 index 0000000000000000000000000000000000000000..d0707295bf8098bee8e97e78b77d3ac79a22629b GIT binary patch literal 97200 zcmYhj1$Z1eur8|RVIH$%JC2h$%*@OT8|G}7nVE6J%&=kRWW&tNhL}A}&FZH~&bjA} zzHY(vNL8g$Nv*#$0N^M9pePu?Pz=!hPpl}i0+sORPyY9pzi$#M3;!bgZ8#3!|9t+W z|F0zc%Rk?SQ_}BWgzw>+!&mqo{v}i-l#)>TPuZW8jLO1PgntpPC){qhEZpv&(s1g3 zYx`5Li1Oju!uS6_UEx;#Q1ho8L-mFE28eqMwMqj5r73jB0EuD@l@sXtQxg8=pDQe# zaC!I}3f=!a_otpeyrO3RpFG0l;ryRZn42(%;nHv{|CVgH_3$@}BA_@dr7(8@{E-j; zn-@v>-#EE6n*HIl=$QhvgNf;s06%zd8LwYncAwkZvM^96zRQ3Cp#Ba*NwDAgyI1egb#r-?vD>+K8lBHxS+33nZ;ivLL zNk`!aN~fVXL-~c`U+9?}#i`^gF7!Lk|J(zpCjK{9f4Cq}n;2?Y^_vPcqD_m^24F&Y z6EFe`y3D|a(q>>qIUBm0P_UtNIB!QG0_DRw!|!K|=$A&6HlmsgD6Rd?vj){lA-;8p zUlnREtTzP8<0$^qH{5SI3krvNS5iWVN=^x&E2Q|41pFu+zPr&C4j~lS-yt0HD951? z9w%Yl`ZE?OB-d~sgo6sT748A%|K-DchkH2OUJ%tDZpVY9>_8zO$@UkL<9Fq=@=$j0KayOt2WN0BgV|upR6JN5Bbi0bB&v!CmkeJOKAm zxQ)V1RCXL3M9=&KmVy~zH0X`$sRK%&Uci7``KsJe&M14770L`{l+sISt29>1DG5rf zVp2#Y1Ou=T=D-a24gNsaXOzl>SuhtCKrakJ2{MXViBgi3@=6`0xza`XTbZUTQ8p+? zl^clHOvDid;z32w1oQ&q!9uVB(Q*kq1@Ay6a03Y#FdLSDrC`Oe(pWjHGFBd4<*+hX z1uPkbL@X9FV1?8}^ zNm;CnR|Y9Plx9i|B?Zx_RyaiU7x)z3gs0#^xE*eQtKlLz7tV$=;4C;FE{4nCdbkb# z3(vy4@ICwmS!h7K)m6GGVocII#`FK{sd$I4pf!$yg zSb%zWFzAdVRsnHkL0n}ct-6C+-;Q`1hG?m$#3_v8gfHMlMC}4N7WRhCU=^4Gqo5X< zM@e?expKCgA^(zp%HQNP6z8DelKpZ}hDhacXoX3z46F`Y!hvuq+z8LXSI`Z$sPDTg zGnIqN3#8d`pcT@Xqu?zdu&US)Y%SvG7pB20<2~?M_%8ek{t@7z4^!Js)PiL8(Vjij^6W@Vl-P-&u+P;^Qme1J4~8T=bI zhY1jadGd4ljJ#J~EKib0$X(?Ya(%guTnk;zP-r7}k^9QS<>~S=c^!J{uzXg&fx>P1 zKl#3VSH33SLh)@BUzhKpvb*vV`KA0`PM1AMTjF3tB&Ac(32P!6vXshTH5!Y}u|1fK zwZ*sLId}rbm{4J9$YN~-<3LwX1!#H>faZL z%k%O9d852k9xZp5>&iuBwOk-Qkq$@;rIAuQsg6`eij|BKC5fV6grZi8mdZAO?|$*=-E3`?MK*$G*)>6nB~!v%ae@sLO$r;^u6jx0-cq9#+TseRNj>NItYI!^7O z{-I`5BT;!xDw<-+m*iG*3|WuVlAnqF!~~)?K@iXJEl4XW;34cWwi4@x6~%JFRb(~C zfo5nfU_jn)3wIXT*PTcz^OO-vU!^^=*|n8qWa0AQeWa;Nkwogj25Zh3C^AVry^??}}tmkvu{QWKC*3 zbs0^pQFL{>DGCGVfpkkcg=VQQ)M07@Rf8%Zk0QNEMjCR67(x^y{P+ue2R<3EkE`$> z*m-1m2B8^20BPVWxCf3QJGmWM-+5>T7zPG`4rqoX!87HMGGFPfR8SzAtFFNva3LHA z`ysIBumnJ>8f;0Iw);H91WBjOGPDC z%n&b$OU0pLC6N@L2s?y@LN}qIP*Er=lo6^49fX;}R^gfu6v~N{#M@#;>5NnY+1rb7 zyRr}r#`@z8h$PZQuBD38`{^poM&<7oW2Ci`8{8B z0Bt}`5Dh%YTAV~P@NlJ*(gtaTMbRj^@E*K`tm6zMkCteJBtafojEl%#%t6}GTqflw z(lcqFv>tI^7x8Y8tP+ski5JC{;t;W>s1`p8`-GuFtdPxL;kWQ>_CO7xm07+$JM{oMom-A zD2-WjLETG@tBb^wzc5?)OMv_vel_2pPvtYX z3tU?+g8R&#Vb8Hk*rV)jc02o;EzRxW(z)*Z13p4{B6JXAB#(4XJ_5HYm%w$*OdKUQ zQtxP%`Ju{E@6i0Em9=Aa%XH;*RdfluOWM)e|1>|;)zxcNX^ewzOi!hzlaXW>Vja%l zqp=(`Lv{qWm2OH?^V+1XM`>o(~I=puC; zw7Hs_>g6gwvxiL z;*ceFz_w_Gl&MSyKF|;IVnfl&q%Cn8jlfx?j_OG*qkd8hJ(ylbPo;a%3KdUBAUVG$ zGm(~^M9S59Cyfw$35WT7t{(TC?ZWy(+e1A=rqIRUlHk~2yI`fDHTWVh zD=;-MI?yW6E>I>A9hex{9yk#w3``6D6^dkO?iM$aKOtm_ljMrZGcX)~MNFqQGyBvQ zU5;LD%rZ_f-83Jt+_E~XO)dYJ=bC)R6UGhtm)fgpH#3FaMAjr?ux?6gXpv>{s-PF< z^AWt8JIy8XrT9pGHh+iTAdC}hN&iVNqzUpuxLg?kqOf;Zd%ObCn5<4k)7R<7jFTCs zx~J-|ey!%!d1|#LNBxibih7{>u_{H?oVh@|sT1Ttq7-h!oT&G;a%J&7|B>TZnw=3e z1-kkjzW2U^z6L(Rm*uVLJLT)-|LHFh7!=qLs1$q@936TRS{phSl0x;_nQUve3|k+) zV9nt6aS6PYf6X@*&q%U-UO9;k!%veVn1gDYzJsxmDZ(7I6t~^7KaX_UKij_9Zdh|I z5#}62aa}ddXjL;Rnpg~W$Rnlu!W8ZclKc0-Q-7L&4|<8a6nGYh2_6mdp~GBBp@Fzj z%#)5lQ9-M2yfT54PpPgDcGk;F#yw;dtm6;9TkY?r!Tn?R(*W z8psN2*x}p_o)xBx)6o1dK>8((kS8Jwnk)}S^JqJ{47@3?fd+6_!ST(+C$cn?rmC)u z(+@L`vL3WV+h$mw*v3Uxj*PZdG2Jyh(Jj-SRF5K;gZc7aVKNsVdgcG&v3YK}S?44d z^Vu{N+SwbZu!GCwh0GuGC(*63ATsA2dqNQ&Lq zasJ~T(z!kVL(b%^LBIB7dNL|!PR;!MOPA9suSj7Hr|1s(e8D*0E)I}a!`8|>;Kn38 z8?TMm!JeSE`GLw;_#d(hb40!PO{guL6LyI$kd1F6Tb0&$KHXn+NPS*g&R8sJN!-`y zZ+3%ybHu5La#0qmQ(c*|V62qHp9y?%!@@Ip)~xB7s_anCmfV^7&kH8F5`!!F?$S{u zl^UU)Xnba>Yb;^rY&9c$MRl{UF;6okn_HT97=P&7>uzgV%|`WO^)HR6P1H5iw$v2W ztkX2nY*1%0|4}aq^gbecxCx=Hfrs8HuJr{{?%-S|@1MK}c}w%_7QS?@^i~db=F-Ft zFc)kg&Qexp0JDTPQ~5+!yaV_mZxp7o3Bkv{GoJJAsxHbo+BpX;nLB!BdQ`r@gXQ^t zat*u#5y{ll-O)d>!`RIDj)~Wc42kX)u_>-(T%Cwg=JRB0_)Yfks{*Ax0}CEyS+d(_ zoz6U&{Vk6y$afU?{Nu0Ay^}sE(}{~rSmQH(- zzU9}=oN)z}Jd*-{aUJDb$}HSP4WVz-ewwGt(X+{XObr)HL&Z@-jIe}#61e4i?928q z38sY}2lobgvhh5|+oh)P7CcTaVG1aRW|m>Laed^a=(e`Q=5EF`=3lZeZ&bH5BaEG`NfCAJea!7l@60bu_w_lt zmHM7W-n7WjPCwPu+}1H_WbDA$#<6o_Z^UemdK?+FS2ADLmLW;m#SZb`alJ3No!vCE zdFGQ$LpI23TCm9Z)ZdTKln!F)z=B~{-jg6N9%hA+8^p|w7n=ajdtXM~&d zI2|$03~x36bKktc3ZK#4(BIondK-k61Vqm`wm_^Ro2W&aPuhji)r)7P3`^xpI1=|2 zJ5eH`$Rb-^ZBye9>fj4EokI!G>Jm8+i?gb+>qH(#gn9*)s z9^tfJ)|F(MszzxX+ArDz>fVMsw(^m8<7cN#D}AJ*wo>nM8_V`9U$Jce;v?g_8zr)m zG}<#JEA_kjjstvy#DCWFaI20JNBZ`Te>e77a9j|20jHN{M9|U`|m<)L9X+H>z(JM;E`7o zZ%s8L=12Y&w=b5BQ>Sc99u}Qw-b8K{nh5U$Kk|;JKmSzat?To-uOGkk{t46X6_n*B zz-3r1X0-mMrCAIfJ1L@}y{2WKv7z~hVWUQ)+hn+Bd}#IBheypyD3(&XxGOcX)UxE> zNq1rwoBC;*5SfAc`76`jzK(p^>-CA}(VyraFTUK&p6l|2?rF+Ic8D#XxHdtE-V%E> zk&Lfz&thtGz5V&V&#p}ct8zbO-OhdFIO1C6E5{!bi^z|$1Y(UcRm=!>4{T=7aa3S~ zdy;FUe;#blRMwp`wY7}0{T*XSIGx~%>tX$m=*=DU>hnuxFrUi4Z20W9xj2_EiacO6FCVTccgA zA=SrM%t>_@T@X#`zr#1qnD5`8*ZQya<%lb-uYbRH_|3jgru1gc``kEv#nhsehLo>b zPF<{C5o^lzxO{UvZI9r*+}+uavkUXDxSF`9J9n_#iIemf)pw>cIe{6k>M5stlk+NM zq-2iF*^@Ugk0^ZWO%b)mjR|7$ywYt-uT33Q+Fa^PkLQf?vA^& z?cR@PjX%B0Fu9hpV?e5&wtceE*0Yu+wxjl&rW|G-wio}vOfw&kZV_WG5>;$q>ZGc5 z>({SUyK1|#FO#;!EHK+ZbLY7X`K5B(bv@(UhzpkgG9MKA(6X?P&m>Pa`;yz%nb)p- z(+)K*SJ_d%WQ9v9_tXYP=DbNXDNIB?T>K59mw%mxjx--g= z;9f5ktQ{=ro#^=P7#!*jzUc4RC)*C#cSH=aiiX|HE^(EoTUPF;d9T~NTJU1Pi&yVA ze4m^5I^Y(*!}WA6;~v|68>7__?Iki#Q+k8-AgGCDm zr$6~#>_gt0pCspX}1C6BZARKJx~-VLq=&OwDY z-HDzpzGlLI@D6cVV>DeiO*Pcjo*{ZEW2G2&eW1YC!F|&`(tl9cMNcuAqTa->N?4Xu zyI4We-|-Lat<{l2rNVkYroLM8r0avpw|m`=zkB`8oco23n|@s7DvIqOdx1&NB&Cyk zv=5kE^(1wada&`G%^1Ix(uhwV~pO>ffu~DPJTp!q}7eE;eH8g)VTP1ET_`#cCL>ZD47uKdoPG_?Ov2 zG*uFVG2Xg9pJTFv^BwXG;~eA)=CtKv-LIy195JP`qGQ z+L;fl?+v{&>Uz@UA(wvMxptp=tID2IFw@goIz{c${cW>HHnHD~YnpO1b!4>;)jL<( zP}Nj!U&@Ds52n*{f9LnKRj*n+aXq~7VB(t!KMrIbcHb8-F$bcSC+;r#EU7_4*=V!< zm8qFdukJ$Mr0dfesz%yt`Xn=HYig@w`lvZZpTj$WmCBURg1`ZP7ynS94j&5nq^j~6 z;MDiGbdOjZv&?qTUL=-`Efv$vuF}jW=7*@P3!f{!8TMk#;|Y&@J$mx^&D&C6p8R}U zkXbm#QOvctV7IG5@Q1jHaO$b3hY3+-n$~DocVN9ywQ|aADst8^L7G!gJ^imQ!{2@V zSo?F)ACCMkp-y0@KEhnqSjDnXzge|SeT{60FIN{;uV)BDl%bR5Yy6w|gVC2GzZmc8 z8tdce8uEO}$}bG+*$&=h_gUX0Uq62h@g-4Cb6?NtIvXoRSB|L`{UFL1)z?lKD&YD+ zUf#v*B0pPxtoe4(%bU+TK7aSV!-zjEh_ z?MpJnk4MGn7UM5MD(9oZ!v$;e{EmUHuAU!3Nw^}ORf;R~MT1xhtb|{&H0?xn7rQE9 zUexs16*0GM*P`o2p%sFzi-x3g!AC$TgNd>X$*uF=cTUJo%PZ?H>F5-4$fc!RVz92I z>8G_+#1J!XHtQNdPk{86$@`jfEcawuY+Cb-L+LxSXBFW2Gjg6}reti%+2`!$4+zWX zSNcOy^Aa8>?<(E6)XOqkN)(MvvGB}TsV5J4jI(k1!E-?$TTcAKXA3hz69eg?D7I*@ z3ZIL&qsQt-TCdp(W7{W8j2IVjJTlu}FY=jjt2U^qXYgv*;N$3n_(Nd;f5TZnzmcb( zBJ3CO3n(o0>?wW zd`aHjz89WJ{vWV|B9dcF0}PL?10%ke7h5LUmRK6vsz!+R?h#Y$r7ZW%y-Yz>QSz%$ z)mOf-Aa8}EmLtMHncd8nBOtLKZ%ZfRZ$KybEtKZ(;Gf{DRTwNt&99qfOW*hH`9-p|^|xKZC6 zUn)>yW&a}AS8p@ND5ouOKj7gm;L-GUHDw&5e`EM!Jz^{WiSHR|5X$7f2a5z>b2@KDUtwU8OY80K);i*IVshJM z-2SXjFZ{CSL*|c1pRx2e!5y%VVVCWONo6tF%g5}FRK+cemaTJQMn|8weMRf-W+u$M zUApS9Q3XY{iTK|@Ph)kxXaDTa!%#!ihJ z5mPc^is_TK3fUNIE#KzOvGJTPcreh)KgIpfUBy?*>2tR840d#Pe{&BgOm*tqW%8AXn4N*=7qk9Z!RQ>1$QE?a~R#Qch`n_M?}chan+%(zdn z)uSueE1J}*t@1{$h;O~Shp&p)9S~TJ;Fa!54dhMmmE2Rw0y6$wou*%FZyr?;)iquh zebQdZI#An<8VLG}=^=*wH&o#76#VM_U>wQEO&DLzu9AQ$oy8h!MquHDY@C% z=Y9n;#-x8j@35Ql-@5k%J`im*`y(499w~Az(GxSk-pqC^`g2^5WJjtuX-ndS=n-}? zs)OaZ@ip;H6uF(=;=yM@Q|OxblK;iuM>`)mTy6dz-YRUDc=(eTt39R9FrT(5X3jF) z@Jg4Y3o#~U1$G`*g3X0X-09#2x65&>ph!-woO?MXa}2r5v!imAoRs{id{<77+=T2h z*@^l43$8mx`-X>B@#U$V`o{Kc2_@rVA{*M(hJJ=3Q3Ioz#up@Pig^%QE_zS&@CdEt zvf&JsOCHAE$|md|w4;3)U`lI*VBM7@wDYiwo5J1XKZ_UTPgqH!HhzGZue_1VK@p}R z9N-ohs0>F)Weny*D8O^Mr?{2x$`uKT>TClF12F0)1T7RJ}}N*Ys6ks@wDwdJnaN+(`_?UMdq|UAe6kC)E=< zw1d}%{S_LLKNi-a-Jsjz9cie%L~e@^q$g5Sxd=kv z9>ZD63ZTQ@AfJT(i0+~Y#cYjL$1ftJFpg9sN=H$tRAp*3MbdWKNH?T8Dwldk4X1We zwW-eJabh0612bUT(Z2UFgn4HpJmauZ1{8oUSX-5yF^)JQyx0286qp0(XE78-R2zk91Iz=?(N8I)RzWBrqkINAy>^J2Qv5Nq3?D zqHd5WWIKd;13VIo2R9MU{2A($^9YG(jJ#TS1ygj8kh73HQ{@J7Q-sJimxoE0B%L%> z_{3*(YJNGtiZ=`Q`67HdVLZ|Yvou5QgAloQ@@1F?8=$@ay|5he6j_D5AF6^|ia}|G z_9-4BSG#(+XRS+h94x#g#VKIc2&qaPEB=9Rmz*#_J zO|hQHKVmY%a`Hhewg>b@`8L=DEE{<{oB$f^C-wt>fG;MyQ&;H=%ujkN!_n)=+0+Q4 z6&^u_2os|16}W|OBxfS5tgG@CdWp5*4zwXZgnUJYHh7tzDBqFha7+1xVzg-CdkZ?D zoFGX-xvk=oN(z<5?&4!fEBBZF5yy%jq=^WtD~4*AC-0Vm(hR6VUNuJSJo2*9fGgM$ zgg?DTb)Uy4;g`s0sti#Ye+&wdoq3F{!UZfIzl(Lp^N>_YVk~|NZ-BSMdSQpLRK!Oi zS%FL>HsaR-hONf8W5bmRa-{T2w2Fs?MZ#RZBR7=&J0x?PgcPv}LJkuV@-!Iv7$nN$ zrDWu3@HaxSCWBn?1|%V^+JhCt$ANr=el9|^wFPSAhx9k{Gr5QG!HvWU!c8nfDB>G( zKXD47$a}GgL}z*sRutTkV`M$4!6;A-`3Kxnm!Kh;z$_6DN*h35Wk1#sycPS)6;Xd} zkrm;!uwVWFb(mi1Ej|(t%Pol3u%wVJ-4nWV@6ir#6Cnv+0Lj=rG~%*A6uF+%;Emuz zj3HW4b~=Z26EBJNBe7K6VBM%-%t<2}h=R6S-d<6~TOAvK@s zMR|!@WHaI|b%vQl)67%q7TJ&LLCIteehX_&rXxG`0k4l$P>v&9v>CE5-K5^K37(es z$wL(f*aPaw{|OduO{g<_jQ8{RSxe|dsDPcoCbAW{Wqe~+W@qs0rC)Ff+znpJC4{2# zZsbeU0$jr=as`!2)}Z$>u}oL>Jyk_!5wnl!p|NRisTML;dLucPUd2qKZqYbhj2=Z! z0!>H~VW36uPsqPP07Jnp`~kKFp^7Txixdk({Bdy}!Y$RxQE4Sti*3iog|dR%f^&mw zf;$3-{5$;zgNbYjc4DX$J2*5%xGxnV6kjLxM_wc(yoR*hk6ovZkw2+*^a*;gX0+z6 zdbz$py-_<#r%|ufw9zH#Pv}o*(0;zYv2L*T3tfrk$iDbDoJUTYR^+!a1`I`X%#t69 zK)A=Iq4~AF*i_088}ru#CAn_GZ2k$io(qKbvA@`;AQ@~GY8E;g{ORxPU%(~^m4vQx z7x9Wv8+jW z>;KmKwEt+t7i2Z6>LP8x zSAZ0GyzG)5^3OSpO$xF6a;_ssgzorf1?T%}_@@Ml1&#+ysMi*9Z`pIflm6>&ue+7! zn1}S&@YV>lW4nfw;CwE`_m(=t8e&U%r7|18M$X0BBTtjF6r+8l9<8Eut&M9<+ssSM zRV^CpRO@N0*|Nhn-Tu{Tv7{SX7)I#x45jriRXfO?_&wqP_C?t%mE_9`ix8GSosDA? zxF+Z=YNGF*XT5K4pt!e0;2-}9@AW{rV1?jA?{sfv=cL^6&e5Lb?&#n!e};ElC_m&3 zw&q&$Npcaa9_Gh*hSVOxcc1_lTB_^)}|dLR29g--iZygfX%-CMo){ZDr?0fD5q)xwe5W zq5AwlXVnI^PxnSQ%kbKeqVI0)ZqrANjE=R>v<x%O1b~W-$ z^nP}(b*Fg#^2E9}IoG&WI|?183f|>B$?Nal9D2aL=4+u<`ZpHis=-RiSNtn=i2g;d zWM*iKY2ytV;|Jp`Q*Fy3yE~$5bnj@+-YsH_ov~H7uZf%$K}M~#-7`fQC+mmlW3>%b z?TLzD0sbCtQkqI5c_%+7u*LtA7?362>OLxp@G3AVziV{Q6^}cmkL`+O_q|2_e)r$~p zvC&(jszelvI$^0~DrKsrFQZFU)lxO59Qb$gDHw$L(EHpTP71yYz~B?s8#wBl@0;k2 z_FQzIbp3F5@s{&Qjy{e=XQjeT`4NsQSvztDJH{85_00FpX2%5D`Q4$$p4Dt|a8`~- zYq`^8Kl&Wgmr0{L>uwm{nu?m5nm^e!QK`{wqvzSJk+mY@qol|ok-MVvqXyVyv%xe* z@6k@u?PT&vnsh50@Qql!+(J6a7Z1e;L1=dn52Ocn1$KB6ytUjP-N!r`D7CndceHa{ z&g<*=J9mHH#5`^OSm#T}O8=u^ivS38^>p$(`B}mod5SU-w4#2Q6LdJc1+{7mj8o#2!b zB{vlgvsZ)p!9_tcw=LMuzs+C8Tf=?PEqgzEQamNxO&yC0Ul&fytDILSzf4Z^9B1C$ z{J&iR-)#Rj&pFS>K$LGus0aU%|AZZ&XzX8tVdhf1R3Fg1THUC(ZnRyCm~E#c`$Zj$ z#bT5QA+m1dfVdj*r6Yw1w{?nTrg^gNF&Yia)VG*<>BMJSx_hcd_n8H_IWRIUuSpDH9KlKU%TG%&0p$8x|U|GS!XF>pA&T|dQ8;%sF9J=Bj3i>j9ncQ zh`b-wIl8%Jg1L>+Xy|FWr|rlH)OqGPr6Z2udE!ZFkhqR7%0CHS<0i6=L$3l|ywm;l z5%Rdxy~S13ZE&hw9UbNJ`{XswZ<3dm9hV>J%rAWE9_c>raJa8}qk?LF58s`8&grH1 z2nqDl{n6Xj9r~-Pi~6j_X8zkU$8z3ovqjn}MW@BAjh-DfBGMc&E~-`}Zf_8I)Yit* z$kbQ!T|HXYU01+#Bxd1%BmeAD$~MU=ZV{>qbA!gvlb||Ojw=jg_-?uzIV-pqyXv?W zIF1xjjv%^kDcDe$=&0&ySy;Yck7K4Y*{cg33ziB+2(qNR zOXyEzNvuEC7~6#Wm!65c_}#)T;aTWmC^^(Ka5nG?&2!b=n>dJI&bhq(84L0UVp#1^@`P|i)ugI^!3t}&tfme~u8G_!gD%6E^ zCv@|*Jw4?IwvOh}hQ7uG-2|;mvr_$wUV)dx+faWI zYvBv|rtA>!@{ib`Y;lg~E{DRWWwA zOt#*!PB%3)?Y2%g&o^ALd@+yEM_XQ-cWYnkM;H%lTB&N%$#i9MIdPGCM6?7Ol?V7R zcw87DPZG7rJ97iyhMUPYM{Dn!Au+JpzcWzM@AiH5clGD{5Bq}N65ge*zuft5*^PM& zz81j=$g5~eupiriYrxLsONxD^{!nmB zpslZ0a3gz@`-@u?>LjGdcZAN8O<9cmYMbJXiEC6Xq7peq(?nZJwU_FzxofznyMw%L za;@j}PIWKid98t7po=jbQ*Tr~Q{7`OtJaexsoG=+EXEvAfRn&*vfB=&e{O~@E>vJ1J% zp?zrQa35bAof6@tA#xC%C~2Z>2KBHaI8P3zTM#$s6zyzP8LdwCn%A1j%trM_x))uaY(}f7&*=QhM|Aq7qx4nmB1elc>*li1zKT6g%NU7tQjmT zb;G-0$N2BULbT&iA9-fJ!CR9TvDV-8W}}e<7cfFPUnp6QG8&hd4xT7Uzq1pjrA$^z%Q2?m}^PN^plTgBudK>O17~ zh6V)3`5UtXg4cpyxznMK>?*XnKPj%317a3`UeOZGF*DlX97dD_la}6i- zrbz*4)hR>|xeI{w9eIpYk(7Z}=*f2^Ieb+pEwoEGgkxNRZwh!S*7jcGEO0%-H|hnR z!^V=%ZD1pn){33G?71eBm^GB^F92_$JMrXV+H&7;1gES3^XJ`lNFEW!HPG#ZE z2?M5w<;ZPx78gmDBj&RhIfCfJFBKT#D_9N32~(sBV3AUkjhFUdCAjlalDq=#iM{8L zMZ(K-J=hU=YiUiWwqlnSa26#=atNPbI=@XwBrda7b{~}{1J7Z)AN?h`iGHip<5Q^h zU@||42@)S+0nq?kPu#$l(l?-%*rD1l9S65nPVxobfYdQYA`$yrQ=NPU-Vl?BI`jm3 zvQz`q)(l5`I1S`&bP$Dvm`%)OG&DLW-+peyrTx+ByImLZzqOwfuu#L8kZ=#gWj zeK4MHD&3&B@kV(gv5RYuMHBz>1MxS=`{^S-mU_ibCCh48@scpvbeb6Gf1nz#o{sf| z-IOlcMUWE%*l6lKgTu-2qZU)DO6#;5X&<>0GYM`@PbObjC)HOKSKam-30n8Vds=E> z_6l3ryF>%kJ?;}culW*eCHU00#0y*}VgtOyzCgQgxt>pO5xF{O#x6)aI}_W5R}f^R z$rR>f^M#T43AjMg@-2vqc($+zZb6>uC14rhAJ|XxOlU10Rj=R!$~gQyeO=5F8|hb| zXBX>g!_I_}D61S%FXo*Li=W08l6mY(rXE%iTSl(uyBbsZZtm&{g3#7`PPbf%myTl9 zz%tq;eS$TZ)7&}buxehY3$UvSgr@Q{v=>%H?1C?a=g~Q>rJ<_uy`gBZaiFznH@qLH zLCuiO(oMEG)d{l&zmW@pMXE(Eg&zbHfhniaIglGfyeJ1g+l#`)JclWTY~)*|-i;UY zrfAO+h5iWDC-r#u4@_-4;XDZkYS#!qz!l9F{*$=L$_eebo4RB2Po!rA@isI^ZJ}EQ zEE*?XOmX0Kgf!|4xn4L&;4)44guTjQ-K*SF$Q!|H&u8g{t|oVa`#?v?YtZ?_r_v_;9SmS^r7E(GJ`1x%2Qd;` zC6r*!1bBSEYD4INdYUqitw+C8t@Fl^v(ytXO$*24Ubc$lCG- zA(Qq=Cxi7EfgIr(L+{YoJ^R&m{9Z5%uZmB^=lC<3E^>YLr>d2>13ydu6&!{E(-|K@ zE+89=Dl#6QB7PN4Y3g8@Cr?$5De6C?ey#Kq>oe7nN9;YKHf9(1ke|>@6^+fnv_KHg z5J&l)x~{>&;xvO>m=gOauM0jaq z|5jo!L-FH)iaEBL){Cl!*$Hs&=wh}N-C7N51&s3`B@=O z(-!aT8;vI#Q+<8mbGoZi4Bq448lDLo_&cga!VYoK097f=e0M$fKsh=CTJgp@Y7+)GR75kf}Zs`alks(}kZ*syvpPsTRZ! zVtcw+@UAvL1V}Tr*w;!sN#WQAnlB2$-chXwSNNescWNQm41YnF3GUTY6-EAxb)IuH znXO&!&88gcjb53KCKmBtbi{qK@C^i2t^6L^j9ur3YK93VILi21{EJ(mrTGkHJicE# z2piA|LM|9a9to|XQ?b*&4AiFJ>#VWysVcx96tC*--jZZK^(J%(AEQZjI_O;8a>sa0 zbEPqytmF`7WjwMe8imNjC=JY}7h|t>b&ToXgQu=CPFKpqV#Vce$n99y^b;Xc=~m ze(m3_(n$BPJcU*UkTW=eKI`wUTCZFTHZV^JCG%hPccn9d!TJXB7JeFaQXYWSR_cKY z=v?Did=VT5o(Vg2Nx{2ttnQ?52=!j>ihaQoxh1N9u|)QOCRzpQ1mA7MnA7v85#al8Weg1Qg7$K1Gsf-;Nl{mv^;H$Hy25ATJ4`^U$HU48bSiooF3VfsBHSd0pcm9p zUP+bI%K~cE3VDFM7oC7Eg;kU)keT2ra>|*9PAKtMS8R%yMNGr=VkFg$i&g)}Uj-k? zkHR#~=3tC^gftwT7wgPD*4zjjCV5aPNN6^5U6eueC24_@Nm%4Ws<>Pqn?+3JuTu?# zY+|%H6W4>|up)j^NT!n{lRTY#4mV?)#D7(r+3z@xNAn^e6b7u4Rg@i3rNdS#OQdw- zIy@v%*eR?$+^oz+C#EkV}ERpcK{`o%b&bvN4r7pI8ao;v0lyY787M1H2#H4pz!0G+u2?4fNhNhB^-p z3Imx_aIhp%F7Y7V1nwf&NL$bw-b`t*>Lb4#;d=9>NPLI%9A5*!07@=Oxwvl)!H%C!|xno9sd7`i=5L9E$4+we%88S9Zw@u^H%; z;&qsS&4qi=|6Pm+gl;;cS~yrvhyS>Q5s05gyqXhHNc$`JST8brEr zPwz&r*G>^zjXJ~=?Yyzr=n8Mje}oO$ctbokE)$RS(qsdymT6!xT!O!O4|&oUP1J zmG@7gFkG~?2p=&Eb@V5CNAh3d6VX$D4%Pw-SEzs!cT?ge_#Rt~Ys4brGtej1jK)wU zeT1f=HRulyhynUPB8gb?gfSGApE*Q(;<-_q$k4|Se;AL66}V5hOJsy`o~Q_o!b4*Y zMr)8>fc(#&$YCCw08>K3{rqN3G3KLke2z@j&w)15j#y*7CsOdLf8be8M64{vYx5Ir zjRf4&6CkuSCyqg#6F~%U{|lf#oMnu_&x$AB;!3+h4-`#&#(io$LB_1PE5kK=EVh)G{ zKM}Q^jQs=yPRlT2yHN}$kQYW-xUIj!ZM+cgqb_m7=!{W#0-}rxr&vLvKXKofibxqu zv<744A)a9^p@wQ>G=OSn6j1*{4KPD4{kI(OT5?pn6*y4R{~F|Qp9<9&bK3KdXOA=FMWx(xFfHi6&iuN^%|P2|42*C zvGbaWQJ9GL1r`$4ote;0><7=G6|`XM@g6$h^XrY9x?wCrjn!!sCJK`8@xNC!CKG9T zdtw(NC!Y8Xk==}ViYqORdw3ZmRtvu3xCeTBiSd9dHl8 z&%la|(R3m@esC37=(-C@; zKZqvATVx?KjPqDIqDUI|bri1tGG@*aeX!95EBOxS(k^J5phIf}YL9MQ)B9mY_CWN< zfmF~KtH=jLh(TtPM?nEfLUc93bMKN-&R7c0PFLbzj9)Fp&wb+-)HtoE=U^NxL}Uci zpT;KQfUy>9&||C>vy4Ny|CK>z$QA1FjbrHKW$qktQRWQSEBGN7+wmxD$vQ$OtHIarVW@kjjeO&b|s78n2 zcY|m_+#s#c^vOga{iQw=Yexl)P#>d#(UWMRKi4yj=XxFJ^zyWY;HZ4jn?W_y+1LYR zRSsU$29fT;Zs)XdS?>!aBWD~3ujmWr#%IL(CR{^X{Dj6>N2*|@eF56Q9;|C$5kIGm z%|;5|Uklhae zM9vAsaYdt}F%7dn)`$c1;3Rf~Rj^*982ynO-)h{}mqO)I7HZcT&=?9tBVrsP??18= zh4Iz#f3_z+KrQxx*o@DX!IdloF(KQqU_=fP4EYFi@)XgYyaKgbJ8)-8l7~qKF8C5q z8jT?nC)-2@vMw6$`~<-=Wa%R z&{;wt_QV?>^v~LC=mvw@S}m+v)v(-1sUeS%M}eWS6^w}WU@PRvK`0)Ngp0_jVM^W) z{o_(aRM%;(jL&*lkAU(pOcaK8@d5kG4#~i6>lNzALW|XMyJMSPF$} z$z|mH@*Bx4^^~B2k#a&srQYFg()D04xF=X9kQ2xXczwzK(*8rfXy0~!Rln-L=wIe* zLi8zLLmlka4)+9uk!ThhhlmM69m5$33zh==wn zQR%T#!qfO3iN_OdiP3R)?2Ap;%vRe6TU&dkx#R%C(paWpU*!)UPy%)Yt+}+DUr_;opBL1$vQ9L zQ9_ZVnnk!oS3FfDsqo@L1B?8dv@s@MOj~;yaS}hpyv1_bHqH8jS!}$~dBdunmQp-v zITsx#9XYuT-4(q%ec8bhffIS3omKKmxl0B+_n7l!UPN9XuQePGH9d}C3;zb`0=OWW zI4;H=-#*{dLg~p#F=vxD$Ip%VQrK1WRgtD8mKF==pIT5Y(XZ(DGZ|)|6B;O9lq|DQqMRF)dV^6@9;I83Ib{5I`mUSvK?38>1U|l{RPD; zpQ483aSX_9kYmjXWy%?*oK4=+Qb=y2eN!uGQ;Z@|ou{*p_#?tL%RxJwIk6vNx5oYx zUD@uo3^jEVcba>ccA3VB-G%a^kDjB8N<=W%*T`Ks?~e1=y!oh!7ja#26w3By*)j^G zmrk#n-Y|p9I_X&FYU&QS>$;qI&b*ww4W8N1vwZOP2ws#rhDT~Wj56AHraV87X>30k zbt-O1VY)!GcrtE2W~G|^un1WwE&u8w1M;^mP(S%~Qd+^pq?m-J5r3GL*j_~Wqk=YB zXvMUp50HWI1K;34LFroXxNlTmZte}w20!O5;JB9gPv*R=tvS_mJ~(FkV?*_%zogfJ zt-(^^6s41P0P48?+(_ZJCE0$)=CfV3yf7^mQiK_TMI2!%V6I~R!^8>I_(se;Vvtr% z)}=n7EdML00^9h%_=dQvHtT0*<*f4A<8voC&gR9q>f~+9>*4qn)%TaK zhrUGrsz83QX^u+$L*Jp<{agD?*J2n}F;$Mdo!BUUWODa-Dq?qZt=Mj{T7hZ>>KEx; zqI|(sNofg%^Zi z=|J15qO4iiE>NP*cMu*6pM)$>DOxbi=nv#LBcB#jcWTu^(%V*{%Ha^QgfmiM?<_Z~7H*wr#U2;&cSKsusOUzvMV1bh;kL%MGv-^S8RoUN z=T=eJP5r>`c@w-A*e~fTK#qHZYF#t=B`7~Pq|>M?o%Qpf%HcWTRH>xY0Dg`?Ll=W< zLmQqL`b~9Ctx**rF zTCVCt?LJdg)R3yH9Z@f};%smYPwM|TfFtyYAYVFgb{b2@fG{?ST1_pXr!g}bk{LzE z!`;#j4u@A5Phj~W0%s1up@f*O;naEybb~D?VQ_MEz zJUtqeyV~?mu=IIibg*PNF`rmXY%Y!$yNE@_okCT? ziC1jnck>JQn*3944>uTEZJpf=s+*r#%p@}V=^8W-7om;nMII+w;uO%^uo}yB2d>1W zj)#w;sQLg;;W$vmj)ALn2nvjepe&)R3O~iGuo5=OdEs~AyWzLtec^3z2{jG-r6ffh8B-NfE!Ke1K0Ra`bViND8de5_y-iU{?D5yE(3 zAfjNe@J6U1t`i@E-St{LEDjPIinYWTQ5QZ6mxXykE#VWi>_z!W+zYl6+n2cx=3yo^ z6=cJq5Q9`N;L z%WI+7@`O{uGs3@zi-vQg^KgOvA_YSGLK8xzLhj(J;CB3q2Z3r#Xc0bp7v!Zz(lqIf z)H?hg-j6GCb>*fq3Ugr?$ZzGqKl>dg;IUFxpR44iZZ3efnBQu5ZF%v-R z+r@5Yx3LGl_C+}I-!E(6IKj3v}!j0ipa@V+rTpDMFui!T4dgH?9LR!sPv*0Dur4~b)@2R5+9!JR0m7EC?k| z*Ay=q2qD7L*ho@j36-0b3mk-O26yEf`cH3nWd4ur9q@RdXn)Bn%D z+zi&1K{mmCd_e`M9`tEip$9VKm_guProd-ckR5}8C|bhaPc7-YnfY(cg>Mn+(tKs(-& zDFFZ6M|unW8!gb+sNs}>y0{-MyBDCtJpdKX3&LlTwnNRYZdZyax3R(o;nM1k`4g7z zO3S5QQhAA%euQp;T!&{BX3gc$jnM5-DwJs^sS4JTzoflbgYDst;dvMfM*dY^EuWGz zF*{q}-1bI^R)1BOVKpxeQsffwLcQ=|_JOy@sn}Q=Y$|IXI8#!zZ*|rrp(=B#_Utpd&u!^O!H`ssJnTW|U>{n(bc3W|9X`P~{)1~OA)O0Ez^@>~scb*A(fhEY% zMI*a<7|*PN##3;2yMll#=$YCbJZT4OwKNW6cST(fHs}borP>CX!C&!bWt{S5C9HTc zYjTw|B=WOgKjHFm95G$kTe%4OO$`X58bEyqpSg0axeIr=auJ5mSO{Sah$qM zy`;L-Sgk&O&OGpFMZFmauPORvf@khv0d~g8DobS^nc-LgyjB@*nN+ zJef(xQ!S~vaP8UY#`NFxB|4XmW12G45l`uG>sDi%vO~aRJqPmWYxW8v=pk02TZp4Q z7@<}8S_`%i>tr5cSJRV;W?s@e=_!b&65tuXLoD~F%2O)l{wmaBKt5j;(PM-C_x znYMvO5-KHgkje6Dr{RXJtYxW()G6vOs!zFqRkx?oNGT73m?1yEk>)P^~W4M z38L)>WY0>0`Z@&8%LlrM)nF=ee+;S;XDsI!uKOWlM>Xs1?Cli-;y2w!ALwKzz;1@Ld18mUI%&&t>d_EVRF z*Lp`y$7uMm@0Bnj@mdi$Y-@oG*%qY8_F7l18=e&-vFn;y|8YqcENU>S9e?${e$M9xemii6HQ2|mt;q)e8ino~oO-Ath_Q1_`EilvKyaa|WY z=Mjjn1$ahHLxjwrC*#lQ*ip^HXkb^3e-DA)u>qY7?Z%bx1j9)A9 zu>z-$rPLy7J~fA$P0gg{frdSn8isYR3ubUl>KCdwWv6uPR2}3;@)>yt&y%fSJdDPe z)kj>#L#>fU+yifT6)_E&*xwN8h2Yxtqr1UV)bMr~D~%bbO8m!vua3;5Xvk0~zXl)q z0y4fU!Frzt7H%uODX0d8|4)n#V3mFb_Vsztfm5)4|Eul5nPQW+9MQ87-%cZ<6k1{Z8ypq4N*^?hikb8JtBcTaed^c z7ZSU1pWpwV8eak)fF9&Pay&Au>p_S=NS-FIk&m%2d=IYpPx2f29o2&jjKELwD?axP ze|t{8A)kWmdJD7p4EW<)$!+-FWw;Ad@dW7$W^{9Kw9DZm7Av{|PWfx%F6xFUsQb-F zJ*+2k!?i)tjYYosBVzvm$o795T`^7-jFQMVV+&#^m>W6ptbRe9-31H(lD-vg)_I7x zF`&&)1~I-rJi~v1Cp-av9|0C^XT1kvvAP})0==%$V3G@n%b%JAih5Ce*FI$1({wlf z&p(i@FAE~?ed9P*_r|C_o+4x6q^^x>%M~#DcJpP})#~bCT-d3QB95 z>_j`GGIqcLXzI_Qx|OWA(l+QORP;*F&7pr8uf$QSm`n6hdIPaT>7-sX=3&qN${3-K z&`KLsP-BRvCs8tLhmS~u8cY|a=8%6<6Ueg0F}Q5olDQxyc7?j8FVylW;QVJB6!Zfh z(2XM3s7TB=BDKdxDe70O1WqCkGi51iw|xvN@m;H`zbAHJS0j+u^`=Ui)*7+9!WfI% zd^&VNUqEA@N0cWo5(~ARid|QU$(Y}5jA2SItrPhPHRTc1H)E}OP`^oZB+FBsjC7@{ z)`wnCAJ+;QgQ!YIo_yFC!p@*esQcAV=mHT#l%=LYsT(jl)!YQo5198ggOWq~NARDrc zZfJ~AN)eA)m9Bz_s;W;Qi;>HTJ9IfJ7_O<_68_-U=noW0y^M8iG1-+?jBDx&R5?ZC zAT6?zzDY?!mxBr68dP8InI5nFPWEJj`avz*IHT7hD^OL4Hrg}21+-HsL_E)s4Z?}~ zPpnA|K?ZIN@BL#k436_Gq7wY;l~4<+jU4tseF#kvoSu$5M?n$Zly;I+^c6VC652_! z0CYgb^n2tJ%+?X8+)gEQZ4Hz%N#q5+1QAa)*BYt>-G$kpuhUkN_o2O*O8WJk+A=uY zdmFPf+8`+wquGsqLQVunaU1c9vas1;Xm=s*s%N!6Oi{Kr>hv3luv&w_*@haSelQ4V zqN4T6Acpov_1{JYlw_likVhNp2yh}M5HINt{ft%A-F)ann>PYWNv~Th7## zH~C9zlUad3Nzi!5WCYL$cUq+3)MZWjBD8m*6)#)e9h&xEq{4RyFN4|D#mI)E(C zRWZ)W-LPh_QxDM#`Bg+W(6JCLTryjUYNVA=*XRed&rCQ!qN18%(bm+D`ioF9@^Kr5^(}tLbUB_OqI&^9awUxe0&X;GZWr($-b9^!ArQ)7xJU#mH~jqImQ!~Wtb)r9Pe{D=-d zNFDu*(T8>s9;ul!hutl1(8v1MYKz!vxXS882KENI>UH`cUsOLJM;XQShQ=l6K#MTF zhj9*24nCB2-pui@-Z<(G+aZ{# za(pW-6dXpBVRXH-u>xHfVw5MLM5QeuP_(HFb2)fescmdgWA*RhD;0d&$4c zd+1NXHlnY#LY=Ef$kW!-x-$_%f1pwT{_WR>nf}8@oy&}pI@1T}E$rXSYHFgkDYQY0XGc?sL|3qnO6j@AX6V+c3kl?7 zZ=CKhrPCecGN871_IrJE*{RV@xIRjRIy$f?u$%p8>B9B~t$e=pi1e}h`91Vn*)P>2 zE>eZa1<-^&U~#%otFRtEN1Yxztr+A>E`=FNy48WQhE5n}qpL8CE2J)$is)a7;pzjj zIvXMeDHTJN$UiJsv;~sN)-Z1Ra+Hwe8<&fY2WyGOOqzNr^qq|}zf)=jDjRRvf>aWd zMK@AsO04>e{zXeQ_lZcxn{q5*JVHlEzAdaMs#|y>2VoO(ldZ(jzucm_|u@5p9CrQm(HJWMk=KOe-Od zdg^`}xXo^5h7%U%4EM=+Eq#$ysD2EiF*Nfn;ZM1xj+4gf7uoYPlvPv{%}|C@1&HRz z$Nym7vW1Bv(i-_LBMGbJXMwWxUGX%1OwUlAS{3y*p)lh#R-R8kw-ixLp#thwbqtkc z=}ZMeUzGc}zKcdk??zrj+-+tYMiIRzR3dfdWuO7(P}`*6iR#Q^occPDCAryb*vK#U z3?3x?I9Y`Q&nFcW3q1qb~G6hBlH?yrH>X~BL5AsC! zY3Z7Ng;s=J=IeB1$QAfAlp&8{ms$VjdP1S`G~6xJR?VSj84=8F+Z=kOXMXTG6dspU zEAf<{$yL!VVI_R(y&H^Y62u$K1TKks;`jSSy^(O959pjSj7+1e8WCz6W1PMQs>}`4 z-_&nRAFWZanBIvgLX4m&a|x;=a+A}IAY(xnge>X;W^Px7mHt#l&~1<}@6TM&M@SLj zWFrRE1BHz>cT37ROJ_HbD|c_ z;oQB6br9nN<3g9AHJ&BxBX5W2s~1s;8ERA~8t8-7WaAcT}$ORJW+za1HOlv^krl`B>IDPRk{cz zY-M_>QHOd?&!v`Y*HkZgm=eh-7S@Vz5p4<6f{NFy^b5A3T1#$1HR1{r54D9vA?g>b zuRb+IwW3}cW60TT72{Z7O=v9jTv%oN1HVjJx*6(ILr5Dpfix*jWe9nISgi~q_Oo6# z!w{7?b*eI5O(zaf1=+q-CeB>t6;YOr0_gk`Nq3=#8U4dUp#LZZjekvYJ-wc(K+Kn# z1m=aEL}hL!|A_2NtkazG6!jIko~f=EC7QD@^(gtIRs&sWXs(ATNG2$RLTRzkKYd|l z3Ejy}sFj@6%w$!1jo{?0DkU`~4O-T`dLgDAcFU1em|TXK=|vBq4$`;jlg2^ys$PxU zZ9E|x3mbTiZbV&G@5rAGJ6Vc2K<9ApvDcfX{ZwgkBDBTRiOEbnaay&b(iN!b!5O8u6M%ezJanrdZ6X%DaHy)A-1cdl#$wYrH#ztZg*xMgZcST->19`EyL_9 zMt`Fh7=>~2V&qz(9r9}IT9V-xYi@j(YiP^W#h@@v)Hg$OKUA#_r$93KS=&KG(c9H5 zc`kV$T&LoCKDsG$gIq^{W{=aEL=k!++mro++Cj<8Gpu_L>05##+@yQZ>)8nA5cz_M z6&nb4E{R=7{RA;AhB-kWq+4(-`yN`W-=OMpkY&-is~JuW44&>~^hx9e>N0X8t&FzX zpXx%jHwaPNafV!|y+RkHOWIVuF*s)J^-;LqhUy)KSBfZhWUophQ*{C?fR+*it;#If z7hVaKPaynC8X5fQALD)CI^=B;yaku!0%R!&c|ec~9t@5PRgvl`rJ+Zh10B^_qMLrq zC`Tu-NvN%?M8BL$pe8*w?XX;cmwuc5x^0u?g!#RBo%xYTHeI!(o70 zO3_`FT%)O{f z5p(Pxtg9`TEE%?>h-LPDHrCQy$j4PrV8wVzq~ubK}Cp!*itrq4A+N!Rf(m zp*rDd;Z>-BWJ=S*!{s5$CH1^k9Gz2sk|(J4%pO)^JMhcI8s>r)+0k7$j5{8uM{%Yn(7SsM32OPaM+DU{5+Byg4KW$v}dys&~0(py#27K)rsT_jk`8 z_aXOnchJ?=#keTfN>`4nv|Df;aGc5w^HUxG4V|d^5sRC;44rI|csAZKmp?zS@5GvkIf|<6B0UKjM}i*C-4u$Ow^}M%lWl{ne_KV%J-FUa zvXkIaJRvj`T62G}McFu}6PVtgl)b1&Zj%Qo1LQgANwZFxBPB{tgYQ6v33`wCYWUOr z{R2Y-j|0DeYjDF?%NOY#?oM!>amG2P=8VZYoAD^UO2+W4#*VU{=fRKab!rsfR?Ohq z(VAXLGh-LrfgL3th{%XvlDw>N)xsAF%u1>i)z3Z+9ZsIuM@O8B-WiveR50JS#A`9h z)@uB9@|;#g=`5|1>PW4^2hb3-uAYRxE)MBi@J8@saA2@P;Dx_Us9$)EnyU{X@<7P@ z!qjG;GgnaiI8Ba3o@pi5SnOyTW+`c1WNmIWOrM2V?kIJi%q7cG>&YfWDzbR3puN1V zT~hLc(%D(=DbwLYp|Qcw!GA)drQ*`*(78}f=%@54yaDz5r0|?j#~>H%4Yf+1uZ=gK zr?Y#mtDSR7&WDT#>2K2OWeC|3j((nB!|_BW>k)qA`>}_}{_x_r)jAn5%w+RX`}Twz z1?gfJi>U?7Nqb}GN41KmW8H5#VmlPoDqyoc6ucz9!QxPO@cyZ=q7wVW#VR?q5}$uPr$1vrGA%?v;dU_JFKh(d4K z^87KOu=$OpsP&oYFfXundK=jtlmeT2TuD{)A-A7vn^?{jsmAj=%w11NH20S-;}s*5izD1Jf+K- zmfLPbO^H91^emxmOq6{Tzk_J6tn^iO{h2r2-7HvNj>7quC)ZL_p;%q2a#Dxjw?Lm@ z&(J8juvQHon0rhouD)RAQ`qtJBjncl5k*L-R*k>l)~W~n=xNk|;us3+$xb3d|0dUk zo}#2esuQ#=TB_av^~51it*_VLsY8|Sa)a<_xrZz&oSLGhX$8@nv7h!1bltC^UjjXR zbv?8z&e_k=H0N2?{H(n{CV#K~?b?@_->?2`kyFF-BfNr~Y?>S~#&*T_R_w|t%oSn@ zS&9_DR@0@gT0cFi)VWhQ(8>5?T;KCeItsF=xcE@pVWu` z-&~s=7aVt8Z@hy-ht$c$dwLnVUFByFQ?m_9`=Vy6ca>VGf)pZxbR<`Wf6r;0lXbF# znYzg7SkneDTzO3YjeW)* z_9ab`BaJ=ka%HP5$bIBRa;CBx^{Tn*UrLv7LMY8&#xvhpC+B+RxS#PqYJY3>CGa`o z>w<65>3y>uPCmF;pU*zC?2jxTofw&FZ6rSAHnMeD^sX}JMGefS7F%5=x?I&Vl}iQ- ztxMQs-^Pt5MxvMF0OAbuhj`E&ja9jou#Ks1%nUvE{F3);UT4oY|4~U%D^VUUSF8#} zT`|teOa`@c9G>d8jWJN;ry;A+ky$|(q$P40928?UtDFb!p36TcP&Du~a3a`SDy^K> zJ|bT?hA5A7{ScxP`inZzDe3GyZLDz+6_>%vTe*F>O*kiX zG1NKK0t6S*`_6UCSuFSWEbCA2H~mZK^PTPv6PS8YRVVgKOvCxIDS-g zEBg%7D82;4(|+!QwPZYie8r>6c*_c9Dir%OUvA_uVKAXcze3SqQmzvH7!$uw)J^Th zuWVi8mGmVb`%e0v_{)ZhDiqnB8!q-WvzFTCOQwcGTQ-g^L(VY1YNu7N;#cZIdE-O% z`ijyH)yE&`IrVQ~v_IQ-!sqeGzN&$Sp(miI^fNZnv)L1DWqvSMhv~%}Azo{9j9hh` zz8|^#-^k*0FQzJppU24J#9c5hvqEej#&^YA&wJjR>AmMY?>X*fTtghz?989m@3+6? zeeU*U-}h75d7dgtkXvED6@Rawz3{w(6Ozr*f~7E@M&*(VnS$a2`>Mn~MXHsmQEoz+ zQN^0%D<3_JziZqM4f34z^!FE&?-A44!Tf5|Aya$d9`jDSBdreJ3>*)33LjT%kR>N1gKk6;v zJMLxuDZ#|>YCV-&!tE4OOs&kdOpQSd`3?Mzy2@kOC;y4Hb0E2luE&mNUeklfm)c!< zROpl+1M3kygFJUUIi8!I>+Yg?g>q|TP5&ACKJxq3A5AjeJ12(v6Z_3wVt*ucEnKDO zi~M7gjz{vsV!DYjMw>z|;d8C?6H*I(DRI2afl|)GbrP%C!;C{2>A{|7QRI| z*gB^DmYLQZa|YjwvTNhQG19M6XStmAl^Do%G{xDbL`;v|9nr@2hdG9SPHoZeNo9R^ zT?3ubj#S4G=Pu_~&waeZ{^2X2`lLxY!REfo-UaTF?lfV8-^0YE6u-PT$+OmxFf$61wuKqbNXRb@~K9MRyFI4=$+Pr*P za{8~hj<^Q8Cwa1irPS8sM6r|oLR41F)u_d`qNdw)f-zej5sC;l3&qKuz>ocl9!L)* zd+Fokmw{QXjX5o{%49jSw}OTF+IK6IB!5xnsVC%jU@N8sOG(pk(%(;xWxtAkn>lh_ zlpb9+x_DIn$hY>^wj=g@5##K9RJSO1Wc8S|D674?t)O`%*1Rgz5U^Xy8GmW#;6y7= zJf<5m%g|4ECUcCb#YJ#E*$?atUKMtNgSyacvDAeJ;u|}PEUQ%V#Gjy=Kv7Z$ zxx_!nWa73KsmxST<*Dj)YAR>3oQ*mfYf5aATsF}Z?}%PyTW#vVC$Yzv_S{FlpsBZI zm~Ex~oB4`4lOKofzeV+ON?W}Js)C8^MsA<**kZD{%{MK(OvrSJAwCM4!xX*;n@;$Z zDE|QmlRo}q-FH)77kvBgr@a|pJ@eEG^aK7cQ(18#TUHPG2Rb9sQ(~5*j&FeSl(}h} z8UH37m9i)uc3u{=|8>Ia#jB4ghilN!G$F(UCwY>CJh zmL|d#aEyMC)j=1YFO;&jwa<&>$zWMXAXC{v4bT|AAaav5< zq}&Mo^uO_(bZ6$Raz;4U?>WclSrLT$NdNFQhDLX-3OeMBY*A?M<3S(-#2NkHihcR)fF$8o0+Sd_VCp~+p0@k z!*jhJd6?cWOtEP89#M(WYopdjHnoS$gTTaV4L0&Bp^&M8WvngHo@xJXduzFDqJ=27 zD!q){Pu8L`sfWx?b_!orI4SPMvsbW9wAQp$uoX6cH}5i4BM(Zm^LBrq^l|e0QmJKM ze|Vq$^+E0~X`nus;`wqItAg?o&whuIRVB-vbK84baWHdizsEF5uqQj?W20)Bn~>#| z^uRmMJ$DCBd0&@MVby}_aXZqcyOk;7)&A|C%6UI?8s#2wOm7?1=`ayjBR%lwFok$ce@rEo+sD{0vytmpp`Itdro z_7^pS-7Sm}AES?QD*Gp$LA@d05(Q}|Q=iXg{@3b&_iAiJwEbWEE6XR-U{jXxLhNIn zZ&_;{X4mYe?aS@|T1%U!3FE+QQpiNggzVQ&GK*f#Z4!E$qRg`Z1M3!fFJh_fgy}15 zH_C)o%$=6EEHBsBUK*kuA)lfie}wF96jrJQfAKtam~s#04tDl+ zkMpxqeQiAX3mY$-7EHoU{u7&z?ZY%hRiq@do~2vV4^L4$~{`Wh76v|2We3Ospf6&p?S^0G!Hg8{cQg(&x+m2}e zP<1rdCqj*Tl~gn7TkHn=Oa3`gK<54ZJ-yw%y&FP1mH+Dd-H=P13O2xwaEkw++v_Op zc;kr3OZM&wZjvV%PP!7_$$H@v*Oz<6G=h%m2$Ze&*cQSX@tP%UO#totu)Vtdv2CGk zoOP77j9s&4*nWvfw5Qo7pxe+obaG{QCv%@Z1Xa)!Xorf^zp`Vv!$JbO3oo)Qvi)g4 zXui*#rKZcZy-_)7pNY5K-dNr|`%w7Dnau0%rNMRTIHQ*y4vz>#xVGk=%{`iX##2nH zW>CV=h(WQH5|+mgiL7W|${1>4Db~NwQ`Kt*8LhoukS@(W1hH=y*-aY|F6Zy%`s^Hq zZkcDi=3r6zux^lDQLXO5Kj7|w{PmXE4K3y>?kV3_sBhY89%uexo^6iDudawii+x$d z&B%>WTGa8VMUe^ijuwaTo&B5cNnV1VKM9`Rb7WB_oz?hJrc35emT_>WG`D;e?=cDb zrm*0vpK~U?>*osZy&ugAx4dVg=M8kwNyxr@a|d#xoG){~8>Q`$xL2 zW;gyh{@dCwXTIM3F)p)&W1y>wXTGn1&+YxwSIvvgwcfkF2cg-W3!jDE^KVe7GT=}cq1+CY9Bx)ZpGe9&5do1iVUQmUZ1 z)JOUY@^_{KzrxhoGSb@Hy2x_Gv_Qya8`G7^D0p{0WCvCgqOAKP`bTe#oe_5~mWVkS zS;aQbd`@7vI5wGyh4QZ+9YT+j2%3h1!3M?X5~?UwgB)wL(q4xL1*ds$=WWStkTo-- z#?MCS?SGz!FR)SGY|kDLN%sefg>u3b(Q&&0oIWdzOxnZ#;HQhTglfVap(@{ld(Nz( z9>BSfLE7j{bUS`YSD^o)){xK*=>3%vVLdbmn*XceI&vemjeg504{Zq#4XcH@MVDa| zFsw7!Ezsg0gVsFAnpuH)1C8AYY70F8wc|L`WlM7~P6tHIjyeJc?-g8c-MEv zn~3ahKJP@I8kiYQR3nIj>|kN8Ni+91Efa?bC((WO7itS}7vBE-M02tyWl-no*R-EL z#Y~1qw*}~n*STN#uILTl9@X`Y^hL@=PAB_A`L>+0GsjttD=V19>Y^+x5b7bHGlOXe z>MttB)K2;|vlX4N8}sMk)#=Yq=PI&E3`?&kn#XAbv3OPOXH)s zI90TnM&o(IS-x7HSpT&>vrV(fRs)^kE1~AQzew3Wn}O&p$iVIM^yM2HnaJ`LFrM z1?B|bg%ZMqyhPqC-;$5W8S)3@Odsm+h<~Uu48?O%PmJ_Wpqmgsb% zGn2W?d|{!vKnNrG0dV}h;g_I3uvYA3dT9D$DsFCK?qwcj?r)xHW-N0og{`}-@2n}- zXlrFlhH0Sq86IVgxlP}MI{yuM74(Uw`X2S2Tn|-~X92(Ox~Ckn#EtUaJ7e>RJk41i z6y$E6SKgLB&3n?j#q-u(8qDn_uI8?~?%zEjU+>^S^lTdqzJ3oQhbVx^7=%vu!{Eg@ z&D4R9g5&OUllf!(E50EZ_fLeO;!x8R(_E9qv|j9rs9`X_N(feAl2BTlBJLLVi49Co zOzEbM<}T)6%w5d~&27v<({WRpcu>qQzT_)l_p=Eq`Utin6GNY%qNq5s5^+>Fq0W*Z zR|SwKhty4U0W#k1Bex!mp?_ZrV> zUx{ER>9yPuz1&lEhHOu@VuFazcZmms<8(o7(o+ zF4*#+gF-{4cQhIbY&@GsE+eU;EwtZa%ioe-CZq}V}-6>9S@ zSc3gVpQd_Jm&kEoDSO~HWV8k99HpfEQJN461&;ai`($?q^Z;&~t7cEirn3LaewkC< zA>=J|pZ2!!JN@PTpS_xAhNq>wzq_Glou{oYe{iqVT4|=&BleS}>8Xs3s|CvQU^p&m z(@Xq*Ps|f~3iaV6yD3&S&#>&bPOxR#ber9N-?q=%(h@e^5?{d&n=BmQX?`5nl}q6o z@h|x{nAxj@`$7{q&aMiAVB({2V*U+zh`*R8bV)iJ>fw#>m_0W7>GicfDl%ZQTN)m^ zg<9L+-hbUyT#93P&cN(BS!!lZmOZC$Zhz-0S3OUvHw8JL1Gp3UK(lTQ+V(ropT6$F zf#J#OFhhpcFhoyhOYrexinvK^g)?AC9BEo@ibVI3H71LBy=ATf1HvDCbI_gF*iS-yTl~*8*qP-1^xwvKnQPS-)hL&jpD; zFU_6dnd@DFj@{*5_w%Cjdgb|C@6a3de4v%|tI|+!3gzoT%Enw`H}l2Bb|Q<_roOmH zENuEF))X&^S4;yelWn){LnH1-yv9zsP{eurTw4`u6=WCUOc%vr;&~wiPuq7c3K8WJ zT8L}Js^T8pjXJ_&eg(IXeZ-7rHo`N|60W3P^bx8qHG`ZEcE@*pmo`Nmt5lKGrHP>> zfhE3Uo>#7PXK6>joU2*2vPxtv$f}sr($Ow&ncMHF?-jhWJk#A5Tpe9;ps^>oIj_^d z47K6wDr+1dc2EIk3fBhL@LJd>zBTPPcQmgBoBOWmi@CD(i}g2KmTf-nM6rk;cAqWD zw#3R>r&%H`HO(IJlkk)8#rMS%WFSrnhxjQ%RdFohj1nyTAk3+~oDaRoiZV55i8?@C zfl_iZXeo_RlmA`MMAc(wczS43@Th-_cbsQ}E8F4E?UHNCd6Atzr$A2C+*s$3yn(K_ zt^uw=u4H$4cOUlzcR9D>eu3-15UeKk51)}QtMFowZRk`+X6y1(g+1aWQz!FWd>l8A zu+*_$vWBc*t#7S$Y!7T*?9c4C?ZfQ@?B(rOZIy8^KAZo=lX9=nK)AyX;cfgroJGs> zo3Wn#E`AYH#d&xysa!tR&io0s-6YiE_iH*du6BK{_EG5}uMA%eH3{7EHS$h#KXvJO z-JQc6eRGTCyw2{K(a4NnEHsj_^Rv~`aZeNXs<7aW2T9AN*kp0 zQ&Z$TX;!EUC>)=CYrLP`hh1@~okck|r?3Eo35rbSYVaBcaGoS@#<{!_kk%vUIb{ua-eI#_DhcH2^|>6U^Pv*iV@zm8cq z)ij69lPrGA9P1jKGh(gr)|*zJb%ix-zKzV%2YB?t@CZ%k7V-_mdFG(yrPXG=XugQE z)e>epy#Obx!_d(mH6o1GdM~Y@nkG}>Q6VGn%13*7cTZ=l+!{HXvX^Gx$vT(yFza;I zn`~G1pq%Tu#~l-0%e@QzS%F!>-NDPjP;dsA^p(P;;R&AhKaS1u>Cv9|7j|2$k4Ua?nGLXped)*=_eR>nN!Rq&3_yB=)3Ei>P~3- zs%LPQnT5y)`hk4H9C{scg64CV_(kfFdI(<*)np%b7CeRB>8EgK{6!WcJ*xJh6TxZz zJ)R{(FK0%9KX*m;m8_>(s_e_zPqWKsj|AOdclHj@98Tqk`Gn9xya}dcBjzr3hzgS& z+-Zx+b#SqcCk~O97)pIm^M~fXcAY-SkYV~^_^R2%RiN*av#CLF&kTga?-sjMov6F6 zEmT+Mi!;yZemrQq^!vVmLjLrMb;qF8h*Q%yguD7`Vt0@?w7WhE?J$<6=P44M`@8g z%6qh|wyw61c9mu~9|valBC$W1{eEeZN}zVp`Ah@aM-C(Vf?%f(9}F*5#nAWo1Nz67 zT9JdI4n-cdjWmU{4fuh~bSR{!vcvf1npiENKdh^&+o5^E-{2}SG&wa?Jvb{+Q+lIx z2@%R$PnZ17>C2OwB|rLU_}wXUbJp=}Hvd}g$DFK;Ng1)(ck^bt&x_r{E%>9_a>lc! zUV59J)25&XJi<(3h9kjXuzHQ{avPtbGqRd-f-ir9`a(Q_ z7iz6)V0e6Jy=oGjq%LQv7xg>(&zK+9bp34g1^N)q3&nmC`*? zTe(LhlK+s%wiSx7R`dtbqjU`96lA1#|9LLyQ*wu2$J0;YGtbX!nbSVIQLdQV7b(E= zo!$Ip!oT=8`fK_Zy2aXUx~}^Ay7%e;6CyW~YEG|B*UvN;Gqu(?Qa7cahU_31{1yi) zkBJ#{Cw3D%2CBxpn5iEI(`u%?P9~_D46V~xyW7r2ca2h+FKY|wSfaA(G_jHDMjoVo zQ6{#Cb|aD-{?@NyHiX{!Ke*lg6-t8YAN;9_@DeQZ6y#~rE2cC{R{w6C5zH`WW#!Dw zev^ACZ(z=mTsq(8d>B{~?#Ncsgw+o;HMA-EJ%&n#%G!Ngb$%avN4-(EMt8*Y$~xP; zL)Vs7;J9cNnn_$F9ueQrpN*qeviZCkZ|5PfI1`jCr8~0i70pG{X6vh{2T@Hd6LgtO zKJ)=IXfRyEiO3zT!Hv}|(tk4S(~h8a$yWbC@0LJq~d%BujAM&XD3y$Yffn!o-I7tga zSEQTXdW8eC$EC0OUHoT4vYedsvu;L8ZvX7U^wqy-r=7{WD0~$^1P=!GNRi<#Oc@?CFZL*Wi@ z4o8LND`8a$PShPV^|D>IZL>|asmu@cahhrLHR8zs{03A8(jMv=+8DaWhUz0$jyG5TG)XZmLP9$KTWvv!7Ni(129hIag<@|T<>sY9LV zznC2OsC&V)*)^OP`l6~tFXg9cPZ(yKMwsFvq9WpKCoE?*>$&bsH!_1N%Z*c;bwv%| zbTj!>?gn#8l^s%0cW9s#=tm4vZBr@-wt8PWj~5WROEVgz4^Equ-aO;a)C#|x$y%Wtq;KscZ=9hjbVnWbM-gOJ0pHZe6+nXb=M4G)?r?+ zV`Mggf35av*6I%E+GsAp%^`sf=9MZaPob@B3QE@GU<;qh+rjU$e$OJ740n?=l5Iww`Q7t(iNdLLRgIxm(XXf+Vj=vL4Ty=%4{e-z ztL=sDv@uHaj@2=vxCuCe_U8^UZdOpg($&{*)I6e1VK4MEDRPu*2M7lF;9-I%fSg{F z|A0HjHL_63?VZ~s`+N4e{2}(~g5oYO>~Xv4EHh zW&(8?d7|r8VJekZ(&3L54o<+}@3*^4=UI4ipBNVaNX&$d?S~{PsM?gvIkE5jpSqM3q6y4$E6|h_B{OPAK3z) z*N#QjoTN2qy7E5m4&O;_(il-O-vF0pm-@ZBuI47rVzJzCW;8vB)-(5*KNx}PMLNMw ztsDAF*&(eKUqScZ&{Ni(CRp8V+#%t%>#}pGlW@*={&Lc;*RD!}S19M6?f&ig?j7bI z8OQ}U&xf`-B?|J-CNxN{TB3o&gjSKll3h2w`MvGPGk$!ZP5vs%-_bfCa`vT z7qtRuNngSPl^)W`z%Ji+uhHAqv&o$yJaA2PK6GT*E%w5~dxeh*$J)Q!L-uoytInRT zk?0AT+*RC9+|@kgy{~;)eqUgDZ~}5*o$%Y3Kzca@p8yA1;YRe}uG4*);b8l8LoetF zUrKXXJ3&9j;4+LdJ~iGo4l@=vN`^~@uwjVtKjR)_*jU+=Xsl)M>CU2}nxd_*y|4L8 zGYZP-q8x!fY#zJ{<%rA>A#W3V1eW`^`p$TNcm}yE3I|=SoRlNSzQ=x-Ss86{&k(@)c0vuHk#q))Y_f~C0SEKUVm%$>}QOdAZx^o{jf zbpv!obW1g})D^(kdWs$dMb{>;t3HKF$yI}{zy-g>|IAw#s=VFkeq41mUA2#16PLYDB#ZS&H;JH82iD)2MF1y6t{Gd4IocqTYo z`X-kN?GOJ$%pk|%DIUR;U~{;I=qqqe|BJi*eLj?)JI;d)CjKw2l@l?lAH+?bg#0kBN;e)OW`Xn7S*VL`}Gi)1XE;SGi)h(fO@(JlX zl%dQ03!p8k<5}z;E);R)BfYSg)!&uqdg>}8n1n)MqUWe* zo@cQ)**o2L#XrtJ!)Nvt@vZPp@%I9srbw_rvMG`9{5K4LR`n+9(H9wpbMm9L5A{va zgF0=>HFdNEFuA^LJ!xHIy=Q%5wOM;xDqAwmU(5~6g7LUvuima}r0c2Op*f*m%1`9p zF*%fs1gtb@;9JRugPp{dfqwp@zU$tho_6jY!VA|h*9Yfu=SrvG{MWS;|5o2M2wAXe zT^e-654v^>@$M{ltoMU=jjtl8Lq~jQ|Q{fpHq54TQ zrv}g#_6Gkzvs%~QFxvRi#97u_vT%>bTW4AOS?gH4Se}?&<{Wba%WHGcG!=PkZ4A5g z**a3!4|%Cs>P37f>@Lmdw$w2~3f~D8Rfb59p|#BmIQ^%6TaaYg*fYZY02=H8LM`EM zAbI)7c-wC9x znBYa}jl33=^76zkvN_$2eZ%KzX6tl@HOAAX<>p?NOO{)fbjx+iHcJ=FJ+s@~%`(c; z5BECVBpI6+&l+roqI#>Y1Nu~xu*3A=8e)Gm)7!`((!Rcd=~gK12(}UH1X}yA!uO!@ zHSj8)rs(x`cPHY>*y|Ep>8@%*31N$CvMW(2hO24e9^gLV4!Wb<2|^dwpRUU;tMC#U zbGQ4aXRSBNx6k*;-#~m7+#;6%8-G1P(a~%QH$wdonwu|rm*KNcaa z!g3J?sj%g##b{|^{*LaSXk2Xc8@d<@_5WbL=+#_TU*NxUpK;QSpsP_%A`4X8^PzZD z_?}=rBylGO_WNJ?=VJBqdNVx@-9e$T`;0J9=qpqZ8VjQYtFYCj6_yIa@NUKkznvAF zL-72!bnbR4&R)2-Qv&0D>3-+Q^zQN(29m^qk`{jT)MC$eyk`#3 zm#7Qm0pg~LR5d_9FHspL)ACwL34RT>!_z(+ddhsP+_9b(p1z)&?*H6IcQ+wXsE2iN zimQ!ln=|BC?)YF|ZjW$mM|$8q=XLb<&bcNDz1{!eO}*#SgBP2QT)uxpH&ri5JH3xR zjikLN>Mt5mtHYj`X?S4VVEV)S&D_Vb+j0`KwRp>5bH3@Ysj4Z}IM%qxa7%C0m(xu^ z`l3qnng7l0XIn6}=y9OO>Bxgf3|^s9gWtCWzRFQbo;*%gq?zz6Y!Ukf4*4qMJ$>z7 z>~88#7d8o9h1Eh0VVmoMYqZPf+~wq)^Blb#=NyADWoVCQpA|j`d)yN}y}cuSz5Fu* zYsJYyWPzhwXe2(6J?ImR$adqKs%vQmX`ARotb~n?Z;UHVt;{#gW=j)`%e>m$%KX@* zHf=ZlHe80nvORK1b2KrUzp$F9z^-h|u4dlQ)#*J{5o#iQ%#*>OSPQRr^KcJ%^(rcQ zSqTmZ#*5Sa@xED}wSwSkATUC2cqSr*R_Htyb^Szs>3ru%=MiU`qnqQ9Lv+-0w#90B zQ`iju##HZR@HF%M&#?mykhaMWFh`CgzL5v%vg{tNg}SV!l(xREvc9WfnDJkeVA^D^ z0Il6*OMgq2`ET<_(=X#7V-I6r!xw#Z{abC8CJucvBfkO+zcg^9e6)(bLUp2&$eSP` zkHnmq_MJvIrUv{b^^pRcsc@KgRw3f3g~-j@#-COH(45n4 z(2q0xGJZ4NH($5ZwtcXjwY87<&$hwxw>iRS(%00Pv>!ANHCD}R^-rGQ|Kv8Yv!M+9 zo9V`EqMZ~?{|n~KS?V{qKC9rftVuLgT?s!$KR7};DSrfS=+9s=v6a7<_otBUs9e}G z?@M+<_8-}2vZcJ|_L{DS?ggHXU>j@PLxoeAvF&x%bJlhaadj2ic#M8C(q^;ejF6&g zNd00|yhFV~yH+<$-^SoE%rG*h!KNLid!{O8gGI0$wzjoRvmLPp%=Jv9;fnUQ`Uj_F z=h7Cc7@moBAeY<&2b{%OcY*2-6iJVXv*a6)_|}r8NaQi%sVPR(2Tf)-curl3y2y#h z2+da3$-MM1@VC$5UhK-ZUxRljEw^g!*IaLI(}LsnSl3qfNzX9v8qZwMa_dt6Y)eE<^s3C;8_IyLJHXO3HX#I@(1ZBtKmcy z0Ut$k%;@`p!tyt93)Gc$Nc(sh`mPAloM1!I1TWG|Omprzwn4dexnDArZGFHXE2V8o@t=TdFO559$(s>2-ROKSS#^A|S`F=d$! zn~$MVX>Zl8J5w92Evk91o`IFH2ltvC!EB^m)DfKW-oeYqg^MCH z1PK<+;7I6c*IA4|N`Jt#l9dHtd1D17rwLJ&Yq*fE!P=mOUGlor!ceNS-~Rck}Q}wKezXCMhnN?v%T~D z7XmlLcG7uiuiOnhm%6IfgiNd;3*oDr&#Yw+^EK2>H8XXC^lHOLqZZFlO>;-{H&X@k za?@<%Qo~k#qApo8Ruiwj#VrF9a}Zd5v*;TjL{$XedmMDH>%m=*0;j(ar`ruk1-XIm zBnyn08sUb>eGrg$P!mj#1h9I3f%kL-&YQ8yelXVZf_uapfj$29zMciSbx*O;cH8h+ztj7$hwXvyjlVOeio9>17IQZ)Md<%Xm-21~(56B?- z6{0thN*zQ$e=tr%qw)6@BTo_cLA9z$5TL5K!&AaK^boFsgp?RA3{?%gLoGu#Fj2}Y zujKx6HaKWg(bakca&8U(EZ;is1y7_W(LDxLLpj$LXNh4JR{AQGnEpc4_yLLX|(D%!H`ABLu64Hvu-ou{$ zo+IuWsEP&(3Bo(q23JGZYbTHI%In$ynEF4R5JYYlY_I;c>a7zP>6=|}56L8I^zJ&e`7 zmz&6qJNpiNK91Q- zHv|Xs3)O;(0$Xi2*&E8gzsWzyCeUN71r2kRN~hWf@|OV5d4teX*T@7|9}s)JkQz9wuI&I>C9r?B1K#xu&>!B^K` zHSjR-LVN_3&}w-eSh9JcqhL3VSFywbVmNwu8PqwtBe;}9SO>cX?Brd15$Jk;s1vaY zjMkjh#A(ZE`$H8oLEAuEQmex^_EE!V^E6vDU7;><^CS6#c)L2Y_m~#U1G*828msUO zhDo^R$qMkZy1_b|tcpc{I2joe%fhw77eL->7<#NsR+=jTTwhVyCryz)f+uVVCW7~8rag{C52fPNC&wlr2_Zzp~)6+wES9mM=6yH04_kdPR6{iMsgD0g9 z@;*6Md7@Mg?E$+gHQWz*8cD=8atH-(By*h^#}?-vaLxGle1G)=b+l#~^fIX$32vF% z+G5&?@y_Mfmw% z;W|%;8h{$NOjR4)!bxNVNVoOrOnMUIWR^li)qy(%{ZJIvzuEi^{CtFxXbJex4WW%V z$4%fGaZb$SN3+%0Oy(>zj%mWMOfFprO0I{_qkn=WxtbnBccNwL0yU57M#WHCN&vT^ zfcy;V-4L+u9K<$aEK#WX2aLp<;8MA9#qpt|VEN_9i)9hdZ82#A-pAH(ZF~G9{P%qI zd^z4l-p1ZK-l5*5-Y?!#zA3&7J{kR_nf`Qt%fMogfBzNFh_W~qglQxE?gyk_QcZa~ z*x(bDGNEf=OL)UOP<{0Ui-#avko!muD|!^Yk(TKR%rB6STd-T%FKh%?0j_%j6Y#6- z5Vjg?WWRwhYK3!lEFFcXGzAl^C)96r1SvWe`!u3m=>)KW%yc4MiH--Gv@tyrM9JIW zlU{||?99)j*{9Obt6gvCUHUgQR*KM1f?fsq{2B zEtoDY7wd~=F%87tXMwbUMro=XCMo{K*CWuRU_nrC4;Tk5d6Kvs0|j9Nn|7HD)!XFbUt01`N_~& z<&s!07||c`Yd1RsTE?aL-4O7PIX0Quz?5VBxGxFxDXJm$8BESf;6nBw^1y?QLIPWz z@K?}0O`#l63GXQfKxMpwyK(}V@);lw`jkRk%YLOlNSDPGBiNu0ISZ7$J7Dv6k!xZN zx5#SQC~Gj4TLA9rLg}D%9K_W_(ln4r`+^R>UP_i)%G2aEAlH6|bM=z^T)r=_kf(#c zIT1|TMA<680*m`ka7$Gnmi~_$-3N@@yULkRb=-v|sbh+edvKiil z14K9E}^C?_S2sEbKVEvwz-^okl2hi9(k>}$sJ;oI-kk^7YdIMA1yP#~V zutRvI4L`8cT`b z`<^8yk`1vp2eAU}Kn3*#S6x=+LsFFny^H_lG_6u?L()|Nm}@NY6r}K(|IfSBq0V|8 zJ^|K#OVp3S5RbFcHBc2Nfzmh@?B2m}R%yT~o3B(s_h23<#Y^OW zkNklm?@c376`SYB^Gs;!zWy0F$yI7`Ba3;Wr7-#+&CvcGfB+BdNi$ z?Fw4v7Vs|LLJR!@gv?vSc98u4BUHFE-Bf9KueX3j=~G@Pua!HX%N_$W?=~o&r@?5v ziTCI}eqT{Z0o(f+II|hj6KS4glm-PwF%sW+zW+M)PP_gAZ zPrDfR98Yy$nLyQGQ~7r2l-POoCe^UsjPbRV^7U;Ss@5&a+fO}Z_*2(4RFS~Ev& zR$oJXP>ov*^0~~c#V0JsK)*+K#HXAF_0M3^NmM4fVuf>r>WA7Z_vOCw3u);8Gm%YL zwKw2t>;#(cNo5~u{LMIQd@ud`E5&&bD>w%G7(z ze}^E^Z=upxzAU+d2ZOtU<%0wgLB5GKgX&;x@Qk=ryelpQ=e=Lxn*S-jw7OR9kaeo2Z#14W31*wMm-V^j zAL~7{V7g=aY}l>uq%&$iqyC=5Nno1yXSRW&Jd}PxC1XduK*gg@Jxn$rvxt0rKfjTK zd|VzS{S~Y#E(z2MjQ7X;OX6p%AM`VEN3gye2j(MB{-)N!!Ih|fsj05B>96QLP&)pt zpQj_Vcho)kJ8UCnDD@WZs5zloO1#`n+7m2>%I~x2M1n-F=n+q0_b(9liC2*wF*1;f zEu*CGu4k6Jfe>_7cV4wGE8K$9JC%PgZ*Shly#4vD3SZdoxN& zcN)i=%iEZUN|BQyM@IM~thNT$j^9cZ9HG}%Xu>25B zuC{|-Hk371hqtM%rQA9=?*EZ9*mCZ>QGz1e4;b!XeB` z?7rP%8+n7$L3NKjPEY54@-doP+Gt&}ZlX@DYoQ&XNm0M%2XQ6XTFeys9BRcluv$kG zors^vhD!|(fVS2RLi&AhCk~3K{`_IZVU>_wazj}xw3xjUkJ?4PmsVg|*o zkDea2EJ7W5$+{lO;5G)AUeZ6;rlX^zXd0obu}0mJTYy@o484e07Df(0aA9D(FTpLj zhPza_+f5w~`)21f!RxNz>xt_nLV98z6{bgU>HIu3L?)Vm=3h-K4JO&#WUeGz6lc`e z)IG|Cn$bl2s75#e)uB&QKTr`}Mq1@wWr4I`tmNP3S&07o>a44Hj(rd&jyh_%-B(B7BQ_0pZz#p-kQ zjSQnK=dHD^e@29?<*n_lPmE1;wbkdD1;iHleIV8oa&~bPK!N*L{)hb6h25OrggozK zv5itO+=TFwdyu3OPya!!#M9bE;?1@D(+eG%d6rHTR)i zD8^Q%BZ>FQN@-DGi!aZUC5(pGroB+!UD6W)4o8bXNPHbkfp7U1&MoJqH^J&6m#xvuSO4vo*6AiRg2skQQC4ruhJZ6_mYRe5&h;X z>Y3)5>|&kooPC_tTzWSJ=fhQhGh`Q9aB}Pkcl1tlrsqm!g1^KP;&ria@Ox0Nya=VJ zYEoYM5oY(-)Nb_`HK`fN2iW1vIkGe|ska5U1~&SdfsE7KqwyAc|MUL}v_qG-fqWRd z|8ubHyI@B3L#zVryxkk}jPm4zFR?`!>nd;#7IL7I%5c@QXJBfdQLx*-*D=Y}-K7s~ zm5+xq*hAcSZ86I$+uF#eXkW~mn7i>m;>W}^jjI*IL=TMVXp4%tV*6#O8qvji-}1=z z*80M--CV`^kM61Z3EPQKg$kte{s>>J=YNyZIhaphaLp2`dFpzT{H4VY;ui2kx5!yI zQCWgx11|yxe2;xe{)PSzVtKi4s0hJRqnTaob?yScfs5eIG4XU!l2WNdb0ll}J{JvO`%d3Z}ZY29^Zp&?~?|8yovi?7p~^;-!j-3BM9X#P5zR z9a|8&-B!zb$|9O~S*lo?ShbOhA}2=wjA{|pGb&_jV{WNi!%ZT~ggS_oJ=a~k94#F6 z9Mv3`oSZPiljsZiUIto-k-;YTE-s5r@wQofxt{%=rRbmR@?7;!3A}|q>Ak9e)Uc^+ zU#>o`SWP>K9pTZ+aCjfWfjNO9;;%rLU`?qrXw8ApAoRliK~Hf%lz_cM|H)r41HS2Z z_!{^V{Kx!yaf>)xDyuZd`Q?iI6P1uQaM9gTs8l#Jzim!bTHo|OX{$45=h_N;dM8SD zhSK-9436m@U!~ZTVtW&^i>`_%;%dg!K%(8S$dxuFLK|s}OpD$hQ!RE*Y))*e*po3Y zqW-kjHa*cyVBV-U23g-g;kDz3UF*;|>$o!91AWZ`!-9*0rGkxu(P9^WYfM{i3ipJ$ zt`DvPOyU0WV73oWzyL`VYN1LYGRP1r+2!O<)$mY5&?Dml=X_bd=DwhRzgS#KQx=7L z6F15E)N=GZ%9Dl2Z9N8h=7V5Vuu8C-*ew_iUXesODnugns5((nwFanSPwktYl)XMT))^7}rdrMYV~nBVmfS zZs2!dw{!-{V(a9o!Q+8rKAUHmu+llx@zZezTBF;-JGa|g75x-@a5Pk|i;y;PM>+uB zQJp|}^kVLNUEa06v;L*x-?%T|!Y#>mNRl$M>zKw279+AD{N2|=$;b~d;AB=3>D*Nb z4Nki~Nh>*)C{1n-cR{WGT_RAYjrCq~x`p=nU$gSEys4wpzNa5aublO-vx~P{$j0^8 zOt;;MnH1ltNM!uqan{)C(W@dn<~&PN+e+Id_-uE?vPEVU)g-hi5?ORrT&!|d(!4*=iM$W@Zasx9Ty;2<$jYN=gs@lk+m@a<@88IPX7ZTkb z`>C9vIp5NcrI${7mD(k1WnNeSL7Hfz>0zoJO~$6je~CX9+bLEZ$=h6(mFC@+PS$(2 zebLq9Y8Gvg@H`S-3|?xeuyms0pAs0SO09kD_Bu(smux= z1z)ffw79(F5480g-RoR;9Tgon9qEq#u17AP$LF2kGY1ACQL84N#A?DK*Ko&GM^{u= zdtEo(d7j1KEDe_Lg>>lrT&68J;kt0b8^ySg6?2AcM`n_RR3oSkYeMnY96j9xhDExH zfoiWx2<=5*@0(;*T8kagalPW7=PfO)DLk8-mYth^GnG#3k&%{j&vDV$BNSp+7}Fv~ z#mtXi5uX&-CboLyP)mgAtg)QYZv16#Z%d7K#hZ&ZOc-CRR02`7MQoD@pD9{@lYdMl zs&2}mV8!68fD)Jl56?3BwxWcpg|8ypYhY-noD?h>_~L8qo$FpIqzLPUlAg72L!b6G z^p5l%^EgmT>YVHBF@+5Zhhs9(k>39{T5Zvd<9RIJ?2P+E8})CsSUV)mN)M7hBo zJx7ma6?pd#a1`&t)KTJFbI-vn$iuX&8etB=I>WQO_#f&T>f-Pb4b{eJkE& zD%b~U8x1k9iY3OA4l;qd94;3gD8CN22?o8VyxV+_y^RBZiE{%!WWf03A@W%9vA>09 zh-*;(@Z1xbzKk`Q2QvQ4X_2?wK2ezODdC@mKC4-DUM}U>8}!%pTXb4&cddvF!0vo3YIz;o6O`0zbS4>1O`+>kO-T>=hiY2*yV5)m=gD^r z$uF6`Br_>>TH4v{2Kki*Li(h7$_Gr#EY~A`+ge-78h2=1+-eHP z{8zkMNv>4ak{gSDjy+<_*QfCv>EX!c9;xg_7DHE6TkN-Vx`WZdDSj8uP-`4F3-=Xvwx1K~`O~BwK5$3bP{_Q~Cz`!@GW-fH_!*C+Sx;9Q~z)d23~K3orNh_^Az7%lmpc*ZP-a{s-dxn*(m zqPVOghJ^P;ql=u2SsHQPv|QJoAIA2h4^j%IV^x^$Yy~z0D{nq>@CK=7DsQDjAQ*N) z4c;YK7MjDL>KT|LT=%S>_KTAapeg~i9buHKhGXpc--~TdsN&D z-|#%&c;Se1RzX^Bi@Y~EZ|ozzZ^Rx%Gxd4>TWgJ|2R3cQ42w&@OLK!;K@Wu5bswF; z`7~E_Zwz%!8;zF?J9L-%kznN|$bX2(y{84yHPSi9@e<_Q0geT(-7d3W@qY3x5_9By zc*4~&5$#A{Ktk?8?kgX!U27O)t{CwsDi){kDY27c#z&>u%369Dzu`Pesei+9 zUz3~1CP0V0hRLAIP5qPmO4*wnPHOkFL-y{RL}9d#@g0*HiH1j`s)NW+&n<<7;p<4{PWJCtP35}j z$68BAHjiSXOy+BbC+b@4ztnl+4^S>1@(nZ}^|j3V&66w|b0z&=?mYEM=@MAvS?F{X zj<)Xujq{+ufkm(k-IxW)T3i(<6R3_H!jfX5q?bp8suM@3YHVrsY%OhQZklVpY3XOJ zW{a_DZLM)Sv05gZHD(zav|omL;J@_Nximl2YQ83$Ne@6zBvMrd>Xine&B&Wqkjyv+ zN}NGZ$OP#jOcvXi3$c~ienfxJw~u;~3cqAe{+*OOqHXBuVxgb=y-(hf7@iy?^L+W)X*a%7oPq2Pq49)|s z{Q%AIskC*d!zb3Z*`x)Bnyr%bu8DHT!<%x~#={Yn@!cs~V-= zVDd(Eif$Mkh^T8Btl!0br}B|L_6y39#_S>WdfgCHd-E8hRo{v4PVETU13QJ94hI+@ z=k4WCH&*gy_+JKhOY`J=n7s9u`Uj5rZ~2D%t#IW!q|u?n)L}Nrdv#9?DW-Rpf#yQf z0yAZrXPjsp4jRCp<~rtnmLJwA>%Z2r)=Q?5#zDFrYCC(D-b*zkNMfE!Lp&qqL32`- zoQ4T`HBeA8$u`t;>VGcL7NB^RH5?0uXk^BJhJB%?z4Y(Cb$Rq){9xt4iClDZIx6qcrUQa z&mgh4Z}2k`kY*4aX@wi3DWPj&7--~8cB9|8*H9lMhj@KE<62`q(`ZX$YY|)Xh@-Zv zmYL>ShK9PXd}UTcH9(f$t#Bz~m#Q3f8(9J4>2$Iwy^s8gSx6RUF>%y(ss_CP^S5-$ z85$fM=1p=8%AJz>Df#GkZF2LUtyAA-l`Ht_9uRPg!^G~s>28bTdhVmlxf!0!v$@4x zm;8f>8QLUMg{WY3Qq<%~(fm`pj_r&5p&*g~R$`srr7dj?n-WZG^d5C>wxcRDc*&dO zjCGnEwH#ZVvxQ^ceE}NN*3t4F=m(}qesPI7GO$jp44&*HkVEa{9p(xfr?u#c8D1K@ zqoWuwjMER&y0lHS=e4CxS;i>Sp$MO~Pt?Q6s@4zY5r$~(F?K2a6;t6|&^(MG&x5*I zot2Yw>AIX3gAEG#a@Me3twK5iE3NUCa3wmAI6gpGTh6n^=Llqghy6+FuGGe)y-G06 ze;x|y80n6@B~+DML;vRXawD{BwVeL2ewlWn?xn`5sRQ@UZp}CCNxjQxwKTO>L?Zhe z+iojsE@5b|`M?!`^mPGqk_O})>IGezeFeA7FSZ|>OSLDL5L%>h1>7C-cE-cJHm2J=XoYpL>O2IbQ zr(k_XtN&^9$CQlyH)e5U8?#YYiqA#1p@#0vR8;?=KW^-48D(y1IHGILohG8?KYWda zKHyl-aqM>{x*vPxz?WbRyyt~V`_O5mRUVaE!++BZJtAjFMd+yF>|qf5dEFA-MO_bc z%-d=*`BB_zb`APeSJiX0M-27NMQlwYzC^GQ6)Z1J>AJq^tE`urM{Xm#Vp2MX+svQi zduy)p)3}Gse`HV9zEE`~Bv(cnOqtM5)n(Od=p`lwJ9#fVUgoY$J(6tF%RC(ht*xxS(>fz2@Wz( zwk)vJH$OI4HB~pv)Ll^bhtErbO5i3E*oX36)f4%1Tp~B0DMpD{S_y z^E%l7@%1MDW1HzI>x77QQHLTrSneB4x@b*9%{$E{-BYL-i`#bDHe0uvFX*zlm83(i zCtmkT!a_JHx(dZT&EQq&D7Fl~6lW=tyjrOlJ{js9KB)2#H;5I;<*`r|nK9s*WKt1G zZyZYvWH#~bHQn_~jAh_ExMHepXbo*=Kdu?O4D)fjrn6CPZeraOan-7rCmM6q71%e_ z9#s%obaz479f13_nUB&e~L!Ecl591 zq*p1se~tg$KC?y6+uUh|j>1XyyAGdotuxv6#Qm@DozzwJlitkDfLGUQxoQ2&`pmM) z(%)*Zw6QETKQj$CH!y8BR@Hyg)aENO*Km?F%l`$Y_*|ZJ_(Ki~Q{2Zr)qMQ}cg3U9 zFlBmpE%Kl5QHwE|Od)+_PogX|!5-;>T%ha=Kcz*ky#9shT7(oC8`;gW!7x=5#T92N zk$tFRtU*7+8WHm-P91+O{#{h5i0Zm|$OwBc^$Mkhju8FW-D-z!tf7y}n%DIA(DlG@PjP33!mR~k3qA19jdVpi{e|=Gm2x^{)=yoN z6q_RbXp_A5r~E5DBPM4`!5(`Fm*3UW&47Sa$NSh@(QlJ>DznJ{xEtCi((0$)B==2%dXIyV4A0)qP!Fm4M-lm?%9-DWxZ@OO;_Xb-i z*Fx2iGCY|m3Cf^@ybM3{2c;|A0H*^-0-MBMax;23m!n;1X=~1~bTmKGT;)zt-;qN) znOH$@(-d0%js6`!zv!O$TCwk}Jq-`Jmt;qH3ka$^r_~KJUN!eK~G>TY<9DmPRKe$rdwnLvC%jn1t{ z8blxoUj2vRm*KRbmi90^pNI?3fYxGwsxGyMpJ{kw{TcN(YJTKN%P4($^?b|;+Y=1P znu(fvhDjzB+=_&5FPobb8N> zMVZ=cCUS2D z&!_8DrgU?AQfS|Zo3xFpuM_1-rmGf&87A1^fAF` z8)HYEU=WalCq@s(_iEKISyC1pr zoSK`+G(nQbL1^A=$joj|-3D{)3213qj6ye}{vj@hvz1wL19<~xQWwJc#B#7-S#}~b zlujf);VYQu-;gTGHA7oe6{sYp33r~W$bDsgBCWNq>M0VIzlSWU5?}^BXPUBA*a}QM zRg-uga-%|sk=i09oPa9jFqHdsLcH>)gnSx*J>OdIXm7UnyD!@>2TqHZ#1rE002$~7 zN8%#iQQu(y(Lk=)N~!=2+6w62Dk`a{HQ&ey%4g+wC{`7RQ)vKM^UaXcvyLoFz9u{b zW}oCysvn(9cZCvqGcwTIFh&07^Z^a>G}8k*g?${ucjB9XvHFc$%(dc*K)I8}USe0W zwV~cQ2%5)ldInt^s_+}Au}4zT(C@QkS)w_n(w9SR!9|Ql0&0ECIo~RO;}eZQqWlje z3>CwjivgXYWC;0Tp^l*-G)IcO8cFF@<;u`tEku=k1^TmYZ~_jLo&}2rw~JLo0q)wY zfK6;Fju&@`=b_!$Ev^(ti@n7W;yUp=m`0a_e<2mB4HSDrl<_J7cU`ZN#xqeobc@`BkxZ?L%31?t0W>2GNG{y_TjFQhfq#y$Nv zG!?1DN1y@Z!#$wQte{$meBVl7!R|n^>u2O+Im1oGEpefVKvQ&VCg}Kn95Yi>MrP~F|1kJ%X zOuniF%R!kG8O#;$i3gFy)L$GUZWmvQdL+E<4892Zf)U6Xse)YiaZqg9@m*hn9(y7b z*#t7C+dvaK3HspOpR4pary_x~3KFHeKrgb8*awUqimow`}tCtblE zJSCl!E=w2j`yWyW9{K;k=uDNPlrFd<8q-(h z8^r%k5m+zslu!8YxAD(ZFjv(f3w~sD_dbA-wj23A`|$T2g1+$!{v?5QsC1}a=%3IC z@dWs(I?(V@Y z5F}`D4H6sz1Pcy<;1E2xOK^90hr^xYIJdQt?(gm8_kTX-*=28cXLh=}y1Kght+x&U zDczMm2JW&gokNz0JNzbU&X>k0JVvguj#@eUD|3qmum& z>p%Wf>>yK68{J8MCR1N3q~zDKM;6lAFq4OD45kg|V(`S3hjsI*sWsiU+6 zWYgxPvw`6*0=Y~?9qOqpT?thSDxI(|(SR^&rvu0mau|D(eo__uHP3;U+`>Pha{D?| zCBF?E?gixM3?L~)p*iR+{6b~q(Uqe6MH^I$9tmXmJdDV$7@d@7fwm_Y|5(GoP4gOA zSMj_CnT{RcnIbdvVK?xZkNJ6e&=54RzaX7+L3#A@EXt2p+!*{0ezODE(wB z^NOdwxD*mKuw2Nv?B5j?vVU^B(g~gU$qsvl+o z^0FkVIQ76>mD_k}Mj3{=-g;rAnoE#_c~RepEY_!}dYcU>g~n!Q<1t?>a#{`f0pQU8 z0+(+!9&L>=Q_KMJzFLDhWFeo*H;4${kHip?yqCJDKudb-Gl{VT4bs?DOduXq*CImMTqEp$lrgku# zE43Asbq8PGR7AlVsh8Bp>=3dVTPPcmr?8AYCd*k{r5QcKDxm)I7}lNc(@ zxIO|A-UMSiH_ctJaZ`*FW{NQy`%NzcHOy!|uQ`vdu;ew@85_-#JPP>Uwq`we1}m&u zn6>x9a{X)7K;^+L!bvBRLa64N!ng6+sJ;DxRl+!}rJ%wiSwL5ai>S#u3Mlz#L=tnt zXRWOY@Eq$(M^q`iLUxfNG#y+rp}=G$C=b-F>^OE`PvJVtNqOK-h5~so0Qt4)A{toh zwb+$UAq9m(e^CAtlYqnVfJJwnIE>hWPG-^n#91yHlPP+ z{g+lRWY8T%<+)F=RmrHrG>512Z)PdfuY0dIMD4CpbO*VN8k3v28|c6C{1j&U>3jwD zuUw=IYsfcQeMm)eo`fS;0u^C-Q?i_wpqs5HxRc4e1gS+b)3#&;#%@7ijJA^Ps8aZW zbRsQiRp7}-();`&M!bnC?7wjv_G0EE!|x6%x*rph!~G;P*0GW;Pj!8VVbyyT*td$M$SP>qJ~}rvYC`4$zmTWbvC1MB81KXrhlI}NBWcZ z#xmr=mm(*`OCC)x0ju4IHo!c3ffl3tU?GN!2;@Coh76C0AtXvvBppOE@e_T8tgjT> z9HaDSuyl;zgRK?xExV4k?y#7^;*E-?dSoBG-sO6p@) zsLq?C7dYxjQB(9sr7v<{G%^5rvH37x z#gdUc9KQBY)SnHbV?=*?Li|W`(I3Q8T9$WWMM)&2Tb2AyX90l}N^8&?{0qH}%B}U- zOT>*wi%4!4Gck(tQwL%d+xRKEm@IyV@DW?xWoG!Zd8VI`5Z7&X!4G7zbuR0m3KhB!zE@gb;$ z8wWjlKzfonOt-=?^Q?sYXVSsQfT=-i0tb8$Sb{yg1@32-2t%G@HY;A~YPQwRi1GRf zRIy93GDI>kqzA=x zMS{q06(wDfJ^44OCHjbm$mG3%TCn%|K@w~JOLmAV*m+tYGWbsAuK7u{An$o3RzpuQ zmKww7j<#wbBDg`kvJ9F}EE0RgW>mNL!lOBYIbbEML2mLI^HE)rm&{a7A%CYR8IIXL zFY7CYTFuBTvJbzhfjcXjgs^wKzF3JJub+_(Qk{iUSrPe-*ukq{UKy^<7OOF)JCGT` z^z2q%^W(^%>xta<$>ccf>T+b#?I9Ynl}2FBO$8IhWRX*IThB>*QOoMiGMiDf1ew4G zSm)p?{RE%w0{ump$nd`^{^gG_C;Wr#Y%kp|;&^BBkF}f56t6`R^@=q(fHLKq$tbmo zF^7#37sV*i548>QLXumtF275+AY!(i3=;?Ohg?U}D57UGWDsoOCjNluWF^f9^fY}d{s13Y4r1{ym_iY+ z!HqLjGzC}MQD9*%iw>d!`DCGFm}o<1@ol7&7=_VYk<1s_k+*XR+EG;$A>-KJ)&zEr zjW=2-fAg|*FrCbk;G>KZ2|(&x2Dkk)@XoElbw86uGz+r4CeSCMlhp}XL1%e)__KBC zpZu9v4qmSe@(@)d3LwkBhgE_w*kn6$%_sB0WDU6pe-!+BVk`ekxx=d>=GKglrF)S- zwoe>`G>(v&;wt}&G1UzBR)#EMi+OHZ9M<{Mttk+qU1%LXQK^iXXflZt@UGE!pG0qH#Bbstd?+8XSyu3Pu+6m$~eTNL2#&o6FPi{b$y5kzx#Vg(bvbYRJ-GIVXfD1r6KH_L#;{>6A-B4SaWvMl)< z`uGdtSWRdRD>pRqFEI^%{wdPJT7y+=CD9gpjQcP$U%?{($&11Y6vLd|3VEN&WH#(q z4jRj=iE7{-JSJY#_53jAw@+B-O`-pQ@%aMl3XX;akXkKNmgtNtoxzHs4!MB&WddZh z7@qzQkmOgg5_4}D3B|l|2DAG>k&_l85kjyt<`v>(&0!TkBjag2qBsho8OtE0i~KjX z0hx~D;hj`Mp7eIGFm(Z{tsnW1|Dx_hE$(@kInF^sjcFq4x*kM-7KX0;O#1OE^bdI8 z9q0#9h_u8W>N=d&No*tKF(L=jQdr-7#;Xm$5Ix{`L@JriOTlZZ1`llxRWVa^BX9U! zWYmGU7#1iF7%L63^P&Fde$gB@v?glewtzR31OAqS)Dy2@x!f4J)#*!~jP++nXiXt{ zggmyEir>jpI^U{F--#Yz6qUAQ3we%t_7eE8j*H=FXEAbueBghP#>g!lNeHsTmJtg1 zwZXb`7IqV7@H~XWO0*((_%!+&E4gd%HZs9OX@T0unn_56E)$u=2HIP+qph*34X2-B+vf?Bj477$LuGUEW z3a|SPaxOR14wz$2%LcznK^{Gl{>+ z8PbAhXYCN1*ht1%ud#lQL2X=)J{wD#ppHm(^g=v7%?EFCy0|UgQw6;VATv-k#ndt2 zAz4em@uTqM60Nnui#Ss$zJm6{Oj4J2z>Jq$6on?d;xoxfaF|5W(&Bes3;WZjL=@dX zZt;`w{S;W2OS}N)`(}JFo6CQsFZe2;;6K7n&V-LWS|o|~KqxoTmT;Sx!>;rHFpmv{ zO?-;EZH{P9;l&e`))fE1Ix<=cBfc0j+aywp`bBNJl3YT6426cSfcLC`@xwu8V8wMA zna0n_RWaL4hOa*wmefrTi)Ew|^u9PNWDTKpz^42ww0)L1VV&d&#I&}e6b^yctSW z>nS_V3*#Q%iM^=F&;%S3^}r>zl@%Ag(2vc@6p;>(d=)&t?U3v(^vfpL=!N2%63Km7 z5mW)MeMii33(RvQ40}SWNeFV%f5)E2a?JRZA=%0pjYGi0@|V?w9<$<6C$25D={@Y- z1o03f;}Y>&(HKKVEk;L@Vq%k6fcTV8lozA<0Cgz#v+Kg{JmK#!D_5bFNK^RGHxNhK zjsDICy*U9%^`s6_4OS#4*^OGf;aG2-q-U(Qbey>z`d7hfgfUkD7QH@vse-H%zelTC z%P@-2B$cpN?7wKB$?&YV; z`CC|&8saV8!jdtX+_X9$O;(u>nuhhQgS_C!t?A%RxWK2Qo?0@WN7iEwF9YwnI$evn zbONISc7)AFMbDj(ug5Awx+)sxCPt%hb|1*&3SwKOX>PESUc*e1gBG*4(s802c~9Hv z%Yd_5Vk*Gi)E5be<@_S9h}Wh^?8aTZ7E$1osv&BDrFbdELP0AhJ1cGl2Eya9cm~)5 zvXgeS5l~VOX%4Y~tP|tJ9Px4tK?)wV3X`*1&PPXxgg?T zbI46&5>P2I$V5-EX3=}1B zpP$BBDw6jV9f1j4%FDt}8w;PVIA)^Om`#VnI+muDQ32Y8SkoOc3Ra@87=_QGMJhdv zm0~E?uvf`lcApgD)gbd~q=A^tf5z_gA6QRhBfr5%vqU}E^$39a$cn4Zndu;Tl~<{=s_q zUqm4ai)q+Fm@mG-r~MD>2pgFSCY1lMMwjdTHzXAG&)$)&tRbxee>)%Th8SK8jKfg6 z0~TZjqIe@jZg{2d#ZAN%n*yPBpS;5SehJs@30qJG>$VtC271#BEN?f!7*UPv1b_4l z%v=}2?LQGL*jo|p)X6aV8S$pOh*{?*_b@lPzyNUq9>@c}T9m*p!%7i`7<7HiS9h^u zstjJ3{8-<$7KIUa`9mBh6=)P<24}$*-<-xmes9rF55)no9Dbz=&o&2bjZtwBYy%Tu zcej&F(6JTp=XyZXcEctQB581+%%BPQwSMq(hhWv+gcKlKz-99o^H&mPzdoWEY||)F6tmDv%nS>$#x}9O zAmTo62bP0PVhQeI2wFD{{`Ml|_dbH(+>bv%47@P*QbzG_ydr%5lEgs#c@B1Yj$pRh zg>_Ol%u>amnPbUwG8ftr#(rlOn9qu^Rdg3BSw5wHnw!;NCDmf~zH%?FQa2v*y#p&_kA1+keg<0la7 z-)gorFM?rZ0rKMN2ezT|@j%~l?;cMz7P+wJ0u<4-s#2DkECBkF6Z2gdEjUs z?pfjK?rr01;9n9rrr$6R@%La9@1uRN=XdROpAXp?dL3ES#oT3_2kqBUpXw5O3TF4V zY#6FYu25fT+;-ny+)>fd%TWOJ@b23p!7h-bRk5XNhirZ9x9qzdQK;#4(HTI^DX(Kc z7{037FQUpxoO(@duM}5iu*qPCTLS)w$6$_V1k}zu)&{%94HQi&4(7orSb5YIlemT* zh#6?*bVRp{n7<;58ww0rvS)UNCw+SQ7vPy+WJGwU`D*zM-)>)X&#JW4Bg*|uM93kojnuo_4%k}Zdw%o2yFn%`;>j~!GA>cZR=w?sR za%t~TscX~Xh(7${ z{hJ>?DzOI=hh#L@ABa$({5OJS^CayIe{}=DNYYWMuaz@L=%w(dnJZ@Lm3e*Ue32!? zcDVA|JF4SY7OV_!BX6S)_BSF~9CkJHAaaKp$pX_4nD85b0jiGwgMVHimjTucs}H#h zWR^#(VxQ~qp~i9+_f&UTcSSIjHg$36$5dyUV?AmSg`>`x7kGypT3@w@GMOd74&Q~f zD2mw519*Ao=_KrrRZ;&|h3Zk0)plS!3R9b)-pvirXBl?m9XEl z&#KK!^Lb)Ft)=v~{pV=q?h%?de0_L)c)jqiV3IrRvN`wLm)QmbBSFC#`5t3OGW0xB z)-jbu!29hZ7W1KC%*urMrZ(M-`XT+m2=;?rx4RsNP}}Ku@IVx|-2zAAC0l*R80U4@ z2=^EFV0Tm3O~-l7!?0UyrUXX&274=bkE1U2`@jg}mU#`e9QGSqeJ9e2Czpu*@-g%+ zdvW^N+2|S3hh9$l(D`%0#0u#Jyxq+=R%L#cuQbo5H;demMhtc zMLo@4H(S5Rd7%fK^EIlTr2kK;5X~WYuqe@rN zz(HfE^#gWdN?}$3&l2{+I%8L{1R@UQ)Nl4tz@{|`cSRJ9SQ;J(9Tig4RoxL`pRW0| zcHs2MZ|h|1X4{Y6|3h7(00#-5eKN1e$M9TO8xNsfl?JM*9k-=|7igjFv6ik5P^T&j zz!cR+%>!<+e9nBX|J;+^PULr7aNM=kP#40hs*T*>IP*5LQvb8|TdCL`_F~WJqQ9*t zGQB|ZFA2%t#(nJ>bLvZhFXg_B|1#-|FXnE{xtNb%#n%PjrYDw9UX#{6z>FR&QCn@h z;9THr>}X(DY_W8&+J5RPhc0V|?E9CZ6ryOnL(=g0y2p)ce2*7uSSx0vwRFfch+oDz|$a zqg^2(5jH)2M|wEwc@OEjp&uc6X*;iNZd z?Yt`jMG=d62L7&&^eruonBop`0y{Uo?H`;YL(4{nXZ}5Fi!8roz8StC^uDXRql0ad zdIlV6zt~4QHaq4zR@%dC50r8w-n<_u=s)J|ajjS?ZO8oZXdIZSYauh`A}gwFP<{e)#!0YPR#MkuKXfYl8#s+T zKqGF#3=$2rkPbN=GHV*4`eXk_@U?$Qk5Ai~Iw9q7@;KC4sGe9Mp=W&AxclEO$MV?e zU{z=mS2tcy%$oWxJ;|HXXkr#Xe3+uo%3wWq80)>8?2UTQmd*V~$bj(s5j65)M9uIM z;K|74MA>)8EcQDMD}3!tj*t!%toh4$M_x z_FN_yb^~8wjwyqh^Ib_ZF%^i%NHdF36+X@y@L@goPxQz6`uaw}&-p83fBIAKZKtFb zNDE6_ou;RqNL!qqExi!fBYnsfY>nCu%YD&-GUjTmEt{|v;3l=Jdz3%fCpMbRRZeT4 zZI2yUUF%&6*b+B5pP?qFSGxlA34s<2vPav0aICQ(u{G5OE4f))vKcIhrLb#rocsi= zUR{=s>bJMB!9??teTF$`!;6;9q=fM6~0c2;-z^d@2SS!M@0^5Z5iy;n^ zj(wG@*u}et*lc#h#$zyhx8enWJjub!z&l9;8s~yF31czLdTNdY&c^^##C)TrQQOF5 zXhuPxp*kD=j2=MxGy~sLvi?{6k!?m?p4fdyy_ zMB^*qer5sJG6)Eu)@o%nhnfUWcL2~YfSFHs6n;@K^>4t{bc_`!9- zD_V!uV^!ca`@vPc2;3%idK-JD!?5lsi1kt`X2P>rK`a&15%m~_8L}<>$buqN7(4;H zesSP^5IhQToN|bJ)IiLmJk|lFurjKQI+HEnS+@}lMQP#2uKz8}tP8QvJpy~=O(9c- zzXI-S8&D)ut+C)StqbTlT8s(~z3XZKM-CMsBF8!EX_ zt$07!bBnRJKui4$hR@50T(?GiCIR}o1emp=h{j#z>##f79IedDvw|7eg~PzTe+91b zF*pw&;=rzgMSzDYfbm<8x8WVZXg8Gq&S&!F*sa^iPeL<4K{sruu~QV1tqK`egk($L zC&o=86)sfY<0SQonRc|okQt7A3X8LY~GVlBLz+yDm84}4T9is&fq zM@NDkb_V?$a$Q1~1B0~~|Nlj2(bVw`tDOra&Pmd&yEQyuzKNtA^S6%QU|fkAAFm|%BA0R|%C^9L{tQ*q1( z9&RJ{U$0`%{}W=YK4Bwn;F<_1L|N0*3PTZ+680+SQ>jGlJOI+QUPnxY$Ochfn#<45yx|^ zz8_)V=Q<=TcT}(9@0Ylb6vPX&qm>0o*&qXPRbUxvLyk2OPiTO6Vn@ic6aH^W8bU`) zBWj!#cTKSVC5S7gVFmsPagN*K0`?7dg1LA-*3WBTwYFj2md6%EGdJPyg=qf_U_B;7 z8dJdZIUDQnMey&J;CZq58)q%Y^UC0{1<~1^sNZ=Saf|D?(+l|j5}wcDozvj_Jch?r zd~y?#yaRo?j##ez#wC1u5yut$eK+{YQ$%B4<9LkYF1~*Y-@PO*2hYBbk@pmmd5hhl zbo2%z03xG5GC|V{V)aoTc*>uGeK83=un02V1VryPpaC|L4X}Gla4bUn?{Dq>Heh z)Gy(lu0e8-AjN3xA%4TyI$5BvQe!E!3q_ZpUvl?a1V^`qcLjb^zK4Q+ z_{Y#!Ahm=pI`QoQ?&%Zm?lxq10UCZ5cYYand=*kWk9W?Z6<5JiDy8xTxQi6@crxS= zgR?$i9K3)QybHdc0ckPBx*2{$$M@pU-sjMw+c^I+t|T89aph}}$>pF_uHpY1xb{O_ zQ3n4PhL+_+%X7oNMPamN$8%Olz>Z(% zIKz)KdGK0(T!{tSZQvar-jllH!F4mx4p2wpm6Q#GEL_mu?6^)IT)hA!UKr2P!sNs| zS@3>H@H>hWVbGFh;QWl>kp^u{!HAZBdGNoEXE`o>XpOYDCVs=jYl=I>h9In$6Tc{N zlU|(h6+Q44l8lD5UP1rl2)qla+<@d{zetUIiqBp{-ibl258#)-_o%erHdy;yI0{1_ ze!x*1mar~HPjx(MKwHXSJeGz1DTSjn{wj$>{#6V|L9{Io&X*R?4$O%iBbh>8^1jp1 z%Srgfm|#rx3-0p^eqHu|AlOS5TAhZ!5?~3wKnLC+cKQze_zwO42G1XH$1l(>V3W{- zCl~|wF=8IzSz4Y)c)p3&J%ENh#5Z1`?H_{L^yz=+zQ zN~?SuXFtI42%kR1@e1F66+Ghu?j;U)rw8R9f;Q$1&K9L0gJO_K<)DnJLShXunrh>y zfpJv>uch4N2&;|f+E~xk!gs6Sj8d>h`61^V(6=Z^Eju3JkeV%M|3F@VUXi1ZV;s2A zGj@F9z_SH?kXkL#Owy`I+x7kbM7*a%%CUGyO7bgSXW;*D_+NfYS|TZTInKTX z$7lRXEUa51`qTrx#AZG;fuRLb!!$gH;8}&{M&h+>Ul@r%+ohKwM@SKj%c2-D#h_co zA){hJO)DIn4T?fL^I=34fV48>+nI0$d43qK=EPr_AtU))0>7LbZ1*SJO(KquXmcDS zAlvvF(s+%w#X?f?I1+-NNZpe&(MNnsqBNvjzU!=y%G(ZC}ecgLY4I&>Z-lb{#UBMU>T!h&8v zBrIkWdRp3MIYZ^cqaY5cDY=5S)D`@;9nYB|jV$OImf+0Z@)&%Ws5(db;)Q& z23mx@DYRZnI3ALSN81zdjt6a)BTmjt?}Ad3)=i!#tzsIk>qG0M-T!{xr=X6=(Hb9= zqa4Mt_`bA)Qn%jWO7DWYBW<^|6j-34x1|QjIZlNXooIpd)1_|6hqNDd{O!i$`(cN4 zvg45BsVK%+DM+Vea3+*8iNYsQ_&+}+QwX0*nHR%1bKsD+N{+OgIC2H0oddrhWhQ4N zsl(s>1{;3aiBDudO3NniQqBn9?^)`yoZa-Gbr5KSJWI~m>5xi1WF@sqjv48#$vM#% z?1MDCmSa|qN2zb}h{YKhkYExdD5WSri9s(t$Kxsb>J=VOf|CFL5|lFh0;zt%m13Zy zaX3#}O3-uQkRy^}R+Xbrw&=Trkuvzdai!sZ>8nZ|QE=BL&XFF18PsEWj*h=%i{$4j zWTW8W4)%iV7bj%nhE(K3N=Wt*P*C_kA}HOE;A|j24GA6*_*?d&4cGg>f8Xte^akWR z|M$L}Z>6OE&mWOi1clUarkp?Iok;sD`(FAzB`_Pw`KbYE1SCXt%SkFv2 z`0uWRM`D#yKxqN&WoPU+t_4zbr8&(=1H*M6^wz_aLI!N z4t0!DWcw@tKIWcz%_s|$j~}s^=fJ#VL0(Qipeu{era+|Hk%!V3XuVM^5o{Wpffs5C zMC32ZYw8Blqk>Wym2xVvCg7QWNZSB!XsVsGt;m27ybyL`UtUrF(geHJ(;cAdm2x$N2n;})o{_kJOKR3cvy#KY$6b8orTAI zNdu0@tf*C91b81H=a#AoN-aXIuf{0;RVl(MSiAW`?ABEQf~5yynN_U_^A+$DaoA}e zOc$yrm94ytVH(>h({_v72C|5ZAB2b9K}2G_RV#3mjzrvkB_h|SiE7ScZJjq+ezPgJ zSRFN!Xo`qoKl6*YicGV?w7j}XD8_Q@ckFB}VkzW{6=LK;K22@Xm|Y9edAX!){tj#u zDs((&v4{x$Y4)@_Ba3V|?qdpFXEZUEkQDF&?F9}hgu0MbyO!8(SGDrA7(X3YWE^0v zZF%TZy(f^)hmd8p6wEjm8_#cNF6b!;5=Y%NLOGf&#MKEAFSufW#C>5nca-u zq^H%3iMdU#L$-KLNfbu(I4dwy5RlA^OLyhuHd{V$A zKDj5z9r^8lAsSfL%I+QJukLJ_xfZJz z=!eonF(IRdq; zx@t>aRz0d&dLB<6@>V-(7Bw>|H?jYhLm8`7<(t4{*xTC5DmjWN)2x=pT|FCC&3`E4 z>~q;gE>P}ivHdsf#tlbcGNb7&9?s3CkIYgXI$$Y+srn;GWuh>OL2fnVZV9WD#hZEu{fQE zo7JskEeFfSS0kRc-K=01(|&d|=5D=^l?F>|Lr(f+YZC9RR28NDaeSWkP`d?$VjfG& z;Iu<{w7Sz<-^j+sVHA{7kJ6s{E~BK?nuI$`+ecb$GivH7_H$Z(>k`RtPY<+6RF#fd zu2AmpLsP7kdQmz~+XFrKQ%l_&$ZEZ3|MER%5&LA={^0Y$nREPHY{Wh-HII8?7R$i%+4!~L`nbMz;5hEHbXS* z6rbo*d?_|h=v$f#737ZdlV&laxOUKx-2+%9+4+ew&`Rw)pb8>W7@R-UqVyZkq2{7qMQO>&T=J@l~

      PIZGsY`9%r%UBoXCRX5X3h%~8m!y5ehTw9t|qE8KT&%~G1AtrvN;yLL;-4QAQXzE?iZHd2og zWiLpEcuSdW>^lF^_%-y2D?a0P+9K-@c1D?D>(57<<5*`pmKQLJSv!<0+ET^Dd(YHu91U(aedKl-ox>-ySz zqiJWyy714A%$~A|_4INfSDj;!!P3RLR zV~kS|S!#qQ^HQyc|QcSOvpl3W( z+lJn9zr&1lz#Ep_Rea9e(>}l)AS$N);+q#fD=dT8NX@Jlhn;K4KG>(R_udn}#jLRV zkiDT*DdTJ)MzguMi*r^*M;_~7Mg@A@G2J?96j8eCSM2HTKm6s>I{J^=&t@v=9_N3W zJRvQ^?CqMKv#v``Y4s&NwX1zxmU@{CH8rDET-EqC&V@xwx?SK`PWP3xW@^u!b?6D~ zXY5lx5Z(CK_+~_?heZV|(NG;EuOfLg1TG;&J-Dh<~X3#>%a=yxQioA68HT(F}sAg}W z#E?v8r3|k=hgPt44SAsr3e2;%ic=Y3fk;+R6Q!oZ((aP7WsQ1ziI-q zpSsXQCQ#qGx0JhXlafqwj9PrufnKvsukm3ZoRUXS9_W51GCJh zN*?VOGuf!k2N*fDT_JglM+t|-BS#KeflgG%xnhmx>3NW`lU-e|tP?Yc;@Cq<`>OEL zwrN@iATx{k`&q}}Uk(R@>-Dsr2?fcMYz^HPtVGYgj9tLpY*eObKZ{1*6?8&qIA54{ z*{r4vbCz+P(0}(1(MPi!u3Fj$WNFz0lf2=2?~q2B-ZEce4&N|$RcoZ@E2|c=Mp@)P z?^}y~h2hpPeoJ#}y04txL7QeKCyldx$QS1*^|e?00QE21OC<*Ar3dEHj48fzVzn)Y z^C4;A>BtRRT{A3@pH)X4p;(SJFW(CE<5tV>%IwH%%s?H&GOp9%XUV@tQJz_U4c@u+ z?7btW*3sCR{DJSX1+a;C;4ZRkYkRspFeM)oS5y5;CT|iS@RcCk*3PqPeg46F!h3_iQvP$jb8RqRr9JV2v%wafskvG!wY@LZHe7R9Kbte@ zKz`6zZ(Br$nu{!_Z9CU}5g}fCB{MeA!8bSEsdULVUYnfK6LVTOI^DLCX;y9JGp}nj zpjFjX$|`Z+ybGLYUeu>L>Ua~@#dLa#>X!s_WOr<$EA=)a9C_5gD5>PE)mG$l{Hav& z&H?XIa$uu9Hd8$wl2#i05UGJG%E6Fc+I-!Xwwu>=jKQiEmB{R`_*b(!Kc$xwozxuU z9lI2gES4txYkbQ3P8sl}skcPLuuK<7Ay2QgeWt_Z4#}ylwP5v9n}5B`zGQD0elERB z@(B^I7SR(DBh~f=XN9^`ZhDgV3I9m{IeV6zd%`w*c7HpOa$b3Ae`51HzR;h%Zg`%9 zNjdGTM42kD-Kw=K*|PE|{mCOob(f1b_ub4m>Rm=Uh1PMGU>B_E8ErBS@aC=y+8|m_ zsV8oFXQbUQ4o0Y%Z9 zou!3!Q^t6TXY@9XsV&{%_98}x&!$K6T1pewpY%q?8hRwGhHr7ApH0dTEVcp5C-f4Kr2Nuf!Zoh$SJRW$nAQr>-#n(}WPCPKe!+pF9C(vVz#84HG93 z1emD(N@`mP$ekWcyyOg@seN@`Q40s&rA7zZJI013+2@m%=@A~?YA7ZNo}f!*J=@P%WB!e--${Bu{uFB_t7+91A+|17e}ky|k-1o3>0pGXwqg~s{AK%^@yXYfFQX$| zn;l2>r>Rr*xoo_>VrVH>thdJ30lxfUpR$ZlZ7Gqy1!9h|m6gyAv%-oU{9vPun*McG zK{iUur`A`ti<h75Y=u-W@pe$;fW* zVa3s0;2>?teW*52lpfKK_^$A};LMIxsDG@tC|%@u;CRpcMr}_&ucj7v=Cr2*)q2Iu zW%VO%l{~h?#*gXGmEz$|*-3K)@~AJFQd!5xxN?&6M8BqLYLYj#Dk2ro&%yx zXaz?%{?v0J?W@`Y~QuYdDFuc^iz+N;)6Z+?r2jaQAxNE$zEWL&Ci!zWeFmAG*?)CO>zC+1J{AWpJ z+e=3-dr2$9zXmKVku1qR)4oF#(NnCfzHh!#S}kWT*D%}9=Gv4IX>R}TbW-+_`PUoc zJ`G6Y<^WpI#EYDq`H*fP~u0YZy(eYxKRy zs-4oRkgj@&zJgy+z4k<<3F~C?v=IL$+W}{F7Ht;64(JD~GgzZukss|vwOu4P9cLBx z7Bf$3W7X=&f;bCK^2z#by^)qnJuWEf^SlEaM^<&5`ofAgpOC`ZPOLfC>qA8a1mk~K zitwaBH~8yaolUbBVlgS*%xua~)Eu8{ucbxtzx00mg#w%tvK|Vujkwk-SmjgZEb>iz zsruNjRv84DCy}3B{oLtR0pIU|4Bn46wO4fyQuCQN{3rBh{>xT9XF_N{*4hk3g}W!b zpf$|gD*gjgZ%cL+tMw7SI{M#^n{Lf`;+=0EGTxKN?6h)^)fL0^6Z#M0sI`|52I^w8 z)>s>7wn(pKIvnTihqb-RS~5)6GA5ZP?5ADBoJYtq@4&R`)=T?K`yF)xi!|;2X;u^& zt4u?M>7J}T`N1me8?2|Z0nXdn5NnVBk*}NiCvD@%X1_yv@do~i{+)D-?Hq7+ca^M0 z_rPvs(w49WYqcFotQbF}kJKmeN!k$GUNRBvJE)~7EHLeE&>EQ;iRHZTWSl+Ji^%NW z%1>Jz%%S`hVjC4u4V{sF+z@T)R5FOpM4o0bYqZq_v9%@ij?$dWK~?JW#zL#P(!sGp z%d7ljdOgSW<4OzrHf1}n2p*Cq$ZBiCN+U{Fi*+FDjM=)I*P$KN1-2Q=Xlsj6)7l`0 z0CD(G8A#U~c5@DS$mXk)l?$RWs@$EoN`tfaxYmqa=G(=5>z;9zABD|{RIiB6=5YNK ze@bs6wp@jEpy8sJRUI|*&hThut#(#C0sA{5hu|$cpgtg{fmd3AOz5A~ENVI;!s|#) z{*zgXzY$&8G$4l~&8r@#c|faW%g#>nqGndTi8)=Qu)ov=j3eXnS1_vdwh)I_j%&ZE zE4c#3w+Jx4998SG#=O7zfgb`cau*Q(3i*SVMLe*%)!SGv_G{&ADYS!`;kO%ZniG{( zBj_of3XUKRd9$yOL0*>E1Zrdqc-r#u4`iw`Tb;ss8I`bZYDV9K@hLO8WrcHHtOU|z zJgEU(dkO5gmla0{JV^Rh2;Ru51+JHEG@II&O+aOcUdVJ!hv#|&`JY3870yg~>%DO53qAgihIuRJPjR?F$nxdtqRsqz6v?s z4Z-5k3;FRCd4_ll#M?IB5OwPtie12?^c9~G-^>jJ;XC9nnm{TP;7fs({e|xYM&T5{ zW8Jk{Sqsc!Ru5hP=*L>TkM)z;&HTq|kKCtjVgkQ}ESf=x-RZzIh9IuI0QFoy(y@qc z^h0&ZLDWQGd@8%CTtS^yZ~_Ad-2yd`qfps(md#=B4%B8CauO_fs{@g3a0k^MCjjA_ zuI;xi&_*fG=qModegF^MSk{gmCl$fgFbH>%gP#LJuCO)UynsritAL1`ZPYVHp@wZo zy%(@>CG^9Ah2VQzjf%H_2b`#2_&U%`PXkZdSO2eIQ2*-x!Qa}m1sKGJ8ArVLJin#i zO{tmeNLi7V$#+ISV%|2t83Cg?uSNvxp-#3nah7mjaVNXqgj^1}7dkO)V_56(^zgh9 zd&94XrG%=Xe)kynJok9_LDxG+A^R+C6h_f*R+~0K24pgS3K_)%Sys_>19R4gcgOA- zL7w1MM4f>?VZ~AZq^?#(Ev6JiP2YUl2HRMB8T)?iDe%BvAX1+r%U~rbAlh-=OgG-@ zO#`YguV-<3YTDzW zC#_ks5q~G~VHypThjqE3wxr3>=i2m*-5FXA4Dq|e&W9fb-gwe?=Ir%0>po3M_0#FAW*YA|8?$khS;0fE~t^pHDupB zq2qz*nultut-;=wRqu_8q6blJr?R)NC)#t~^U9OmGuYe3Gs?HgTiJKh)5^Ee``ViT z#QyvAgK3@8=O#Z&G~?UHlf=Ih)8p61=SrIIdC2c;P1tLwQUuhGu;tk5urI!sh!w!@zH8weL^M$Fo%GF4-JSMPujT5K zqf4IOvqy$ZM7_q4h)bD1MGnn;BkJeeN26AUb+OO0odK(~VlM~};4a&y1c={yst0@x zJT*{bvqYf0`3C6fS^C_-#XzW0$Xvy*vCZ~g&hgFx+9PF@cHN%ph;@u~4t7lrxf62Q z)y%Hh`f01d-Wj2-QS;e`+1jXUz$#Q7?0uiXfKZg3WY3g1ii}Y!VBWUkfUdr5jbWX! z*Ye3YuU{~mS#7LAW>!Smbza;orVlbI0h3z7B6AEb)I*uvE|Wv*v{J$Y^$|$>UK63943u~O?<=3buo6N=5u0J0G>OW zIn^2tG&tp#5nJ7?>w$5`Tk{=w3uEDpWd<&J8S0jAMXxRg8{KwmFzS_80z&+yfmtu` zyD!F5&XdcN-=ldNryodF!1g>mxl~HKj6wQNV?o3N*0mqJp(puWd;#a7iuF<;NO-PL+1edu|h z23BJKc^7c3^;s3{OAZp-xsUGN)PUUUx0k9Te( z$6zejkCu~uU{o7K%7Jy}AE3c6i+=ouCAnU6^GK_-H4s`?7Tn77EWg>Er}Fk<4)%z* zV25ulGN=Y1PkJ(T0#}2NXSMavENxaXZt0`+Lq-W>p!piib4vq1`19#^z@b_MJEm6P zR^V-53_QoPV7@(Ws>lXP0_*;DpyIm#wYD9*z|(j(WCA!aGRFeBK81}(%VZC4!9HL! z?2sLRH`5Zi=CkNm`i*U77h${FVQ;e^s|c+7Ds0{FU@O^W)&fj=W5Aj>5UB4I_L-%F zA-NLjlt(i~i2(EKF}4F&Jb`mQQ_0iU4osUb#0GE#4Ir`bH7^Ng z5ZAO8xZ<6_T_Cd#JVepJ4G%^h>sYXvU@sDTqyLJm;M5v{Yy&S(F zZQxNq!Dlxi@%cdaJOiG>1d?tYaKih5IbDst_87c-5Z@UG-{vj$@Ck4rx3R0A0Bk`W z$tDQ|?kphwyW@_xgD=trgm(lu#Tez_9VrVIbDGkB-#v1j~+TtkkG z{g44@uv;VSzexnsEAT_6$m;eI4);TG=Yn~<0e zcMxNg8(6i{K>2k+7D0R9DXQYp7{AjVxW8t2RSDR?96)|ZM7Z zNW6{2%7J+Zk6XB-J9tP0QvyCo58`4%(B2Zr`ECS>wn2Zk#$M^qXnRk{wObH<-3a5b zI`DiF&z%F96DiRoT=fyIcmvvZ9aoij@YA@%lh8?tzkV2$hD1zBTn}n@;f}N*Vn!nT zB+AW&D_KDdN_r6G@&)JJ2+B#KLa*X_62tijl9EWZL}&|z#2q**587DY(K{ zfO(Rb=~B?`+~~usI9%v+*%uNG`4U%>SYvsd4<3*3$s@cUji{kS-%C`tALt;7>Xm4` zytq<6;87$#nBg~Vzz`+l9%NriJ(6g7i7`Eco;v_elH<^-13-H3#Q$f4D3{}SzKUOe zh4xD%*;gU)uv(GL~GI)XZc@DYVL3**g+9Mxn)0lO-ZEfLBtpB%V!L0oi(q zdX`$o@vK3^B=T0~927(wC8DxY@Q|abCMD9NC{c;@OT%4#$G1tnmPl5A@Q$Q3Wm}~jV$t%qkbp$n zO01xiuS6HhPo#Y06(!DG;`n7QgAbn@u-noOOJtwCvPA8pvJ+Y_rSd&jzzXV+M6pY3 zs>Dx9^qfR#JMfSg@NCe9T97^#Gr%{gRPRiE;C3ZJ~9FC$D_?sauSCsH7gEn zmWX4CLw=9 zDR_N}j+Eb#sAh?El-3i)r647#a}rz1F&D@j6p3h*_V0Ud%J(JiRcfsFe}~i?nKvOZ z@@aTZ#OpLjNurOX1ivFjrJOVHzWhvnI~A?>WS_erYuQSP9}UHK-8jnt{@jM|%g+@& zJ8@2UuwTB96PZ6GuOssU{TSi$n-cjeano`{I$%9Y0!^F=961?i=L_7OiEGNmjpk2kw?) z+&#q1p~IF=#@L;QY``?E#TtR}X%+voe_~S&PoC|$0hAI#;w(XwcbL;C}p<&5Ng=G0bg|k z{#b+}qcK7OHVVusx3JE<#=BUWsK`f|Z>^!$H*kYj2CL3QWdT?R{|4fAzQ&l(_{ay zvh#qm>FWCbIp^M)J53p)MIRA8gdn1h8U)cNLX;o~qt}=R!)PIDFoP(;C<$UPO7va^ z5k!w*^fH)Cx%ZU+cir>e&yeSNKL7Wf^O@UD+h^^y*Is3>-?E13s=3^3u3S@lfwfx0 zxapY!J}l@qAbw5I4|@g~i$Pn7H(Q`1Ct5GTgZ5}m{ab^mokVW<3NYGM83#0q4!d22D2;4R~UJ3E+B`IS4YUn~wPRd*vhc1*k(rt|=XBcD0bED5T1 z2FOJbpq-|fjjV!Tn+(NrXsp&^jnpw}vP@8Ou$ENtmEKl^z>sRAE@OrqVx_X$@fWad zn_Ah-)=C$!9d}wq;j_}Kx^$qqlYg@zTrery5iX3Dh!?;7QDGbFB?ZE$+O^TEBWnjZNej`DEVDVug$5x=%kQxE8lnm1#@=hHo`7dMBiTL?BrI}{_YrIOjeZ1S5IgrIXu1cn zIX5DgW?@U#Mnh{s=5RhF&kx|Ur!)GKzyeQXb@whbw+3Y4TTp8p((G%jwsXjZIG!ZN z+y@>2c&*T}t}=ma)?=)-7>W$1nEZ;w6O4=eBp z*WZl)VmR69+p#FW;JWeDF_>jOG0K?rp+Y^@3OpdfG6M{%n`SEOtUH^D)N_P7A3Oc2>ap8-+;NP|{TIk5O`rqhIreqTK(IXn%mT&^&v)+r?)vUJ?pyAXRR8`fBTvTB^rh+V zQO9;vX7S7@YO9wdHq+W2OwHtYcQvM*09II5HO}kBjBupmCG!+@=KrFa!A`tIe}YRf z$`S48>sX|f0GVN|HV4RJ2kWyYD0fX$FRRb=gb@dOWDGN+^~WBkUf4({3Y?P}Baqe8 zg|vat5 z==T6WzyJN=8&>-Hdx_ z5f~{^X{R$Dy4!g^_w@5r@x;38>ix~F;HW>jBrOQuopf7 z(K4sGP5+W=FK&ImvB-SK_(~tM6LiPDU(k^QI8r*a@Q-UDT@ z0%&`yS$#JPM66Kfb1-D?;|E)yy+`eg_KyC(aNy`eIeM^hs*^= zH++3DM3H*n&0Oc{;#uhl)hikYjTwxYyNsq@f*VYx^BmSsA21E$EBIm#$Zo^*hMsDk zW}aPCxH$;=h31KNZ_jK*4WP^JA)d#cbk9Z4aqu#R>c#bu`V0N2KHTu54oo?twz=P! zYP9?4n@km^BKj8m3? z3YcPcGXFp>G|_T_4EMeEi#l3e$x5UN(9wwwu?)HY2po@Jbbq~)LER5y8`u<`(JTD1 zfj$PyFG0KKsOUG#@3Q|L*8POWKS`aibsCm$(fPjETuG#(_{Nw#DfSEN& zE1^~bl{nHoV-&>mo2-M?13yj0_c>m#1;)c-&`x)=0^>0?n~tj0z>jci55eK6qfJw9 zAjj%~kCAK)r|MW`R{488D?qS0lsO|~U&c@>Ms)!x?MKf-&su$jJ{07UCdOf-nDr5B zc|!Sg-i*TZ8$#5trZvO7W29Iy%$!%jDx7OwLMmyj@F+_an;%)B(9ft&4EcMi49%uO z(QVeHCm1)8cWu!cy5Yx8)rYXQCIl&18^3;WW52HJTY080(03}EEv&AVi&?5aT5~Mb zYFK+jW=nT%q&fzikrd6(Sx(ijdvX_ za8-QiSniksmQghjL06&y1~OWkdA7UHxXXCTc>+8iP>btS=84RM?r-!x)L{F}d~Wmy zhbNNxvj;dK>&*pZ<#ady1xkDqqB!}CJ9;!!_}HjMjzJ?Mo};Q-z3EHU9Av{ zvz{An{U#i+gbGZZKn`qde5O++mzdB8Mqk$V-7_juv1N^U-u#-lfP;E*b@64lLOX8aJu&qLgm+APX_YPmE9 z3-kvpm%31LuHd?Y@7A2QtkpXriQ4FGi752ZqdT1be14)!OE(`>}A2@+=pK6RkrsY%xwzRca8ejtBYWIO`>*vR35; zt2KT{!#Rzn(F!f+H@5Hb1m~rCK?1V1xLMN((T7ln_@d{${=N}l)YngX+IwO>jSQFh z5xOXN)u3Xp1Vw9s<0LB<3OIrt8yqX(p?K#vj$}2)8VN;<~c#gRL$ZYDK z>~895ssBmk9u>_ogNiY$$w@m6UDuhT@i^qg8=lLmYqlcpneO>iZ)VJ^Vy&J3h7 z(hg%PGkI$)zVgVP+FFd3sy)+=QdR7XhLmPqNk8Wmv>MHsgIYzuf?%}{Eg=bX&J^{Y zdL2B^3+giHSVf(x{>t|Xnp=IQf`qFU0`=5VOKAD9c6Ni|lT!`DTIfxTA`DDj1$!?N ze@#&$R0*_t7sx*+u$b;6;U^L!8AtYFIV#wdWj6cRs)*)yj{0P(5r%{tP5fpk7)908 zNXI?L&#q0bOa6ePhGU(h8&z-nSgp+2RuR)-BzbmZj7rzj&ZQY?%QH4*b^*ny zt*4l~vAeXVxxUd@z{pNB^Ma6bo?2EewKVk?cA}5h2Lrc;nxfWM!{Mf})c=Y$kAgN; z2CXs|GgN(SuFut6>RMuUr=<$3R?Be{oXSRyQRLXgFmH`xe-~VK2KjbQ=#f($j5XW_ zETFX@=l)Dg#zfm0$EH!s^nJAR%Xo~RlW~`b4y-6wu)ro$iKsH~LFB;2QN!a5t=P-R zXwCmtvl2B(R-$=GeZ>r`Dv?8f@Noh_HByNbP6K5ySQ|?}M3GT59*WOLe_BJ!np3BK zs@g}*uZ3v&HF5<&RPrZy^#W}q+;Kx&PNrIQkh7@=+WR1k#eKJZjev!mIb2w04{&1h$=*WV$soIuUIQl566Cz+KpBQmOI zBxQ`xh)X}0HZc8``xG$J zgN&viA1A^mqwpepYc8k0Ks43#LRb@(hA#e_*4Ob7b;vapd9PH}&Ny~DKL%s)64>l>vQP^>JyyT@z~8uy3H@+Z|0br^QpJrK*YQ?2j^wUT-P{p5k- zhZfpIy-q}>BGyT3(B?9g7v${Z!GDrT#evVT%X=ydtbEK0yNIer&~IN`;mGV6*e}EJ z;AhfPxu{F{2WZAWVWaK@Wx6e^(LN?B*ao}1EttCfk+x@uL~o{-SJKzZLE6{Jiui;0 z*f-cmi^12qPRwrvxK&A1HK?FgSCc`l?tzu}6MYoJ-TvebDSTdN1#so}$h3vsLQouTo1m)Ph20RSIur2Q@q0smtQfHg?SbM3hM(f+g3_XIiX=3>#BM%y(wSc$JTCZ8e=SR@0 zwNy|UgjXpD|9&@mumE|n4)uVtg{nr+mEvl7>~$ACo`_933}3iH1oI?Rky;nypkN}_ z?NTx<{{RmwoG5W0+Av&Qr7luq!NrPTtzQ_moZ6|~!NTf+6%<6hr@!!iJm&v6xakMz zxhwR-ed@xjf;ZZM(;WjM(njp=-f%`+wvKSgUtUD9a1g#GqH_tB^$4E-SNeD;&#)H> z91o)MT#)}ZVABkZJBb660n8Izp#4p_ zesgOFqgFH5f|1vf)s$7y<|`RtAh^BIFHzC;G^>N|>6v<%QQ!ECt)uY;RW6sJDIYan z8l{McPe4nWPsBT(C;J)A^^sWxp8px&C)FTm>p?{j)H|z7!S@T(T2ke1HZ|~mp@!UY ztiM=#J^`$|tJ)JSh=_hMM`1@O)pBoX2U*cV6(g++`gT4ojT&0>)UU7(i>W$M?Bnq(f90TfcV@n5{D6^2qKUkJi!Wp*o~!8~ zYc0pl{T7PNq;KLG*_Yrt{B^u0!EYyt+aE>aI!%A3Qga{=bhr!h`CYs%x1h11-3n zFwb4yDuM4QCzgXBIpr$Z>N)YJ}ATm*P;{kdK& z6#5I#F=+Ql%>ihMD;c8~!D)Pj@2V8zv>dPR;JYdaqW4FPwI%ow&Y(5KFba!MmuutF84plF!;Hx-}ce4%s zy~+H|ykc%g8`{Sww+WWEwHK*Y4j)v1wjt0v4zFo4IR9Oc0*BOV_(Xok%X3UkpiOaX z3(*#5;jx(mX5M(U8?$+c`V4=>GQ3`Wko}GEmPs^Ia2ph6gxm1M75dwON3J??`7X>2 zU3iyO4|VbRwcz_Co^CzsQucD4OJE&7KzhBv19FZ2UCnGY8SWTO>jv>2g(T~PhTnv! zW_4oag8Ed9sA(Z0iKY0L7^lVjB1l138FeZVusR^Gb)?VBq7_hSgSnv> zEw6boBtyV63`g2Tuytmx>xM6F0@37Op+W%Cr!`m+bCH|Jp|AsOyDsz_hrB(B?wE@e zKvCLeESy(bFm~d5jQvuMvTT*u-bEIe)D%s`Bh^)_$$FrYS}5r7`_Nv#MCUB1J_j#h z662`?-pCQuI!d9A<5RFLHMnvSGN1YeaEl8+U^%NGRdVX^eL9xjV{DiMjPM0!1oK{~ z`5t!A0-kIOb2mOOo^K#|8l$KQ@+~W;mV)CL$vTdeTr(Qp?reRIt+AJR>u_SBpcc52reCB}Q6B^uk*B z9P2WIK4T;fWh_RZadpDm+lmo29({N}_q&JHnuGq%O;ji!ecOmRa4@{G7U^(@9QgCd z_2-PS3wTk_z)c4J>OfXk6`)d~s=i?M}wTg`yBNf5F9409yy%=ckU;0 za+X!zPq4ysBWGGbg??OlEPXH^PtZ!_>_uW%5}_)KPq`y=Uk_&BQEj_ls zK1S8zyVStd@R>9=B8?TsX}ou3nfs)U=@O*mA-qn1vu00p*HzwG%LTs`M%Dyl*%#n$ zHIZMD=z?3BWws-g*Fx>J$oA84R%xv20q9gUkm)Ac%50ukkcX&WOFT6d|C7aQ5;OGo z@N|F1(Rlp%G3Yk?kjg*N2Ls^xDva3>D?g*Q1F@xVnW5Iumet;MRxyk`G6R`cuOsmk zBA+2}Kq&9|k%^@laqSrs(a7}WNS$5cSD{`aS*En;m2PBr$)k;oPar(X-+68EAPi%*?U$WLHMF&UoBL&(EdjH`AVj zXn9l7nC&I*GzwYX9XuM`ql5)Ie=Ogj!^KRzXD^e#QrkU6P0 zbH^mOY%|aQJK8F>wHUwE8SkAK>HQe-eL;a8tqxKLVx5116llfh??iT1xLO%q(ZRaG z`^qyU#%=W9n?!g{L5pQjVGQw**3em2HVG=Mf;Az!&t~|3GGp`;G=j49ULG_C1bE7|7J!;6?%me8UY`g%KK`#hTY12m%y+I<6llwY1B0RpkU3)060c&cQc z<`Mg!FdyCH(<_b^iFp+c*+q{}$42T22leo-qLbI&XirVy`m)G$S-Dgl{_g+>3`Xyc z^y*eU&~HQvwq&2IMas=8%5%_T60$qQ+RW$*Fi%oheWo!A9eX;fwI^d8tHwTki9Ssq zi$`o4{U!329K#pCb=KKL;JvHcKj2U=iW1#m! zP^yoy|2+Fxan4%*g5Ww=;Ox#=lM|WY*P{nqg$Ey^8;0OnYKSzCM#3Cnbf=)dx@eQ2 zA=XB&^uh|Ck5oBFOd>auqZTt}l=>t7luK%+S_Y{cN&IZIHdUKJ{9=Q)mZ|}3&~FZF zr|_Na)lO&+Gy^N=miD{0hdmRt)_BvhBg_4?oZzSL!{4!pOr0jg+-1C$hi8kR(FgH= z7kc$a4nM*6PNaR)sJ%E6eX9ne>=COeuHhM*&n(r9*>*C%-G%rlJK{YpK!ohPvB%hj zU4EI&noDSSe$2WdQ12SsWy4MW+;g-%i1}&2CorwzQCtMG>2ex zi2T?s+0iPt;&VBUjxhvZ(@A*r6~67fS|u#R=2|;#CNZ)f@JN={iedlt!AAU*m|+(x zZgnIMbDFi@GuU$y+9#;{siAWpDyH)(P92F%n@`U)!25Pxsg4#e)n+?E)5CCTW31-J z$X5Ik#0s{+G4qJ8lsA6T2k4W@B(24I?rugOs_o2&yE2ULjb(7s7seVRidF7M(TjH* zXP7(N@Ova@cEtxgj8R$&nlEPtjz`*UMN6K}_f15n6*RdhBwIYO{mN*|3ieVj_&E%# zavK#E7cefyAUjGjdb+^*`SFQfVExcWcz+XC)K%t}xy+7fRJsemmuO<|J-{XkRnIXR zUC8n|WCip_YJUobesg5}-}q*>siry<`b~!aVzdTm&bhU((6nnhK6E^Xa$gZeSO$$- zXm4wmkp*2j?g~_GNv`BBd?6c=6_fFe?}uWi@iZ-H5se4(^nK zO>-ODCKb#7ETd{0TIf?W(9*PH3KIPh7S=iBLI@h%OmzDk%2f2l7qq`7cMXQeA1d#` z**)?67F16lH6zuTAlHwkvO}0Ap7Jm%l&zrU-!Z3-Q7@};tRN_@jZK>#^__v994by5>M6Ye6D#w|KIq zXvgm}Bke*?-=!|W7`#4zGE)bzGG(=yZ1zJ(3AWY|)rf`rS25NNB*ASWVLi|^Dq1(N ziAK_E1C2N&K#=(xqof=$xB5ubKe4QJq85GO?)LC?ICpGeo@F$wBHk1aSLmTsY++~Z!+G@+UhX% zsoDxjdX-T%iij&U~^D={fFcH+`lL^JAy2$XWd1<6vXCvfe*YS?JB{TJc~A0 z2Ay*x()9zp@n@M8)?@3;hZc*GHqH)uQkZjFyfHcp;rTBv&+gLmi-C6lZgMK9eV6|XqQGW{ebVN zDg9g%3QGQuWYwHOLQZ1l>cQ&HgVgzKL62{MU!JgXD+BLqC8~e!U`7zc(5tK-E6g0W z5NW&`@74~spP_9z)^a-deI}mHfsC(y_&ukQr}7@xcX7XqtR`%WE|p*11RbifHKq-f z(F4oSzP|MB04TK^>wFHfcObfiKjZlbJvjm0e+Ye=1JuZ~_#2({dn!FM3(cr5PfLAn zD9{ayMB)fvz>7c9*6GX#gQ4GjtkLIKk*P#KJ23NVtOgf<`+W8{#Ii4nt<@9G?t&#M zTv!|1CW2OMf-)npH40%_+@Kvx;j?^5jmON2JD_7Uz0!}ip2S0O7Oz|Yy6{k*p%**-z2>S0-N zeF@6;Cwkivt9Af({Ve9h)A-6Su;TazJ+c5DW()d!K72KiaL6oVNlRK13jc3_YLA(( zqzXbQs1OYIorTJ=jDr;GbJ}?cYdj7cp#~IE&_TANADqJauR_1Izz!jClZ zhV({nuNPn>y47qvEl*SSH&^`xOJy8<*%up5P^}BG zj=(`IMXCWv2HII@zSry1`G`57GA$Qmd0BC}3+Xo!%Xb^Rq#$YXqZd>}o2lpZ307c~ zSH#cM5r5NHctc9SV>h7WQ`$cZYPIFu4c-{fX?zIh@Q{@BdXpuyp%9WI3^^g619%3h zjv>`L_M#guN8{QA)l;F|Iq&MT0K7ADykx-$K1pSGtqkAt!*7E8E@;DV^Gpxm$lIKC z+nd*O0*buy<{RXM_L3hb9GVyY6u+h)er3sQk*o~KdeRuL7JVY9>5^X{>-4j6<#(Bz znxTum1y@UEg~3^}dR|b^4Y*E)PN_&rli!|jm6F`45LcHhJi(_Hzq#OhyS*}pY;8u) z?>yak`bXipvarJK+6K~w`}9p9ZKYN{@$W12PXaPXGL;TN5f3Y7#aC6y8>79*xH$$l z%If#O(cTJS%~zpcigI^5r$%yNB&$LE_kVKl6P$4nFZNcrWIZ$TN!lX$Xb*XcTu@c; z@x|kxn{!^#mRq#oBFD%k)j_0()4lv4xqjk7lNtk(O<`lJOD3LVaY*))jpQvEDmkHi zL7qu^T`FTp=1LA)_=q-1g$}7SU?DN&ibc7iV0ufwjAR1@@GADI2ii!6iDX(xhJ*!` zB&#l%UNHEUiDXZQJ|_QmE{t6RM4myaEIjnKTQU+PKSwqf_mD~(g0C*f)`EsE*+P=% zBiQPanIkyal2amCXOdT`A%`VX$ga&HSralgWL1-cca65lcl!*fhGP2|3vmT`B9&Kp zmp{~R?jRXNlHnnFMDnX-P#Ely%m_JN&JeofWwZ;9xxtYyyxAD9_+)3INdAtb zT$+Q{=7;{WNv@FO!bt9gv`SXW%f38(mTWp3-Cz1ldQ8ym1@~X_uyS*Z;N43WTM^C) z<>>|EU9uMXFw!MIMrbMdGEyZ#@>N8JODiN>MslBI zlOv=mh#V(*B9fQmW|Kd;n>?ZPlWcNV8MAm8O~vf1zQ zt9@mgvi8w(yzfu;3w`b9_WkO6xB1gqSiAIYD6KRee)_*9M)PWJVhe6st~zOtMn z{V7i-82qvcciYb?DEQJMyRG)U?cSHGN`L(GPwsEiK%O(}&%T0vE&0{gM_JFCb*z0q zn~tyVmve-IHf3bL-8-)zf{w357;NA^R8m zRerJQD!=>o$!}Rl$?rCG?X%@5o3b{AWo%|`QV&GF*?l1Il7A`LZPGWsk>oo@ezC7& z-_dS^+`;Bm`;O8|-_y#we7D=0^%Sq~w?7MgZJOG>=^IDF*Ftl76@K<{w2!BK`)z5E zwO7u`I!2E2^`P{UuaA73Yv0{&!J9p6Kf%AZSDwm#p4a#Je*I5RDpZmEZ~nc0w(qmu zf5J0ztnb&XvweH)E6MM6f7m_YyN~>4k33&%vUj8E zf8O=~_g$_llzsEK|8zGWo!>mxcjTMzK05fWDtrIgf3KgB#mn-moaNi=yF%8x?})5l c{_mf%X#amdH|vc59p$CH_OoZ5`QJbNKf%7=hyVZp literal 0 HcmV?d00001 diff --git a/sample-code/spring-app/src/main/resources/static/realtime-api-scheme.svg b/sample-code/spring-app/src/main/resources/static/realtime-api-scheme.svg new file mode 100644 index 000000000..85729e6ba --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/realtime-api-scheme.svg @@ -0,0 +1,3 @@ + + +
      User Browser
      Javascript WebSockets clients
      Actor
      Transmit audio (TX)
      Microphone
      Browser
      Transmit text (TX)
      Keyboard
      Browser
      Receive audio (RX)
      Speakers
      Browser
      Sample App
      Spring DI
      @Configuration WebsocketConfig
      Binds handlers to ws:// paths
      - /speech-to-speech
      - /text-to-speech
      @Component
      TextToSpeechWebsocketHandler
      - Encodes/Decodes WS packets
      - Routes Sample App server traffic to AI SDK input channel and back 
      @Component
      SpeechToSpeechWebsocketHandler
      - Encodes/Decodes WS packets
      - Routes Sample App server traffic to AI SDK input channel and back 
      @Service
      OpenAiService
      - Resolves host address
      - Delegates WSS connection initiation to OpenAiRealtimeClient
      SAP AI SDK
      OpenAiRealtimeClient
      - Initializes appropriate client (Text/Audio)
      - Provides implementation for usage
      WebSockets (TCP)
      ws://localhost:8080
      Transmit audio (TX)
      Receive audio (RX)
      Receive audio (RX)
      Transmit text (TX)
      Secure WebSockets (TCP)
      wss://${API_ADDRESS}
      Transmit data messages (TX)
      Receive data messages (RX)
      SAP Realtime API
       \ No newline at end of file diff --git a/sample-code/spring-app/src/main/resources/static/show-realtime-diagram.js b/sample-code/spring-app/src/main/resources/static/show-realtime-diagram.js new file mode 100644 index 000000000..65eaa11b3 --- /dev/null +++ b/sample-code/spring-app/src/main/resources/static/show-realtime-diagram.js @@ -0,0 +1,20 @@ +var modal = document.getElementById("show-realtime-scheme"); + +// Get the image and insert it inside the modal - use its "alt" text as a caption +var btn = document.getElementById("show-arch-scheme-btn"); +var modalImg = document.getElementById("realtime-scheme"); +var captionText = document.getElementById("modal-caption"); +btn.onclick = function(){ + modal.style.display = "block"; + modalImg.src = "realtime-api-scheme.svg"; + captionText.innerHTML = "Principal interactions scheme"; +} +btn.disabled = false; + +// Get the element that closes the modal +var span = document.getElementsByClassName("modal-close-button")[0]; + +// When the user clicks on (x), close the modal +span.onclick = function() { + modal.style.display = "none"; +} \ No newline at end of file diff --git a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js index 71d224a9c..ff25001e0 100644 --- a/sample-code/spring-app/src/main/resources/static/speech-to-speech.js +++ b/sample-code/spring-app/src/main/resources/static/speech-to-speech.js @@ -17,7 +17,7 @@ async function startSession() { sts_ws.binaryType = 'arraybuffer'; sts_ws.onopen = async () => { - wsStatusEl.textContent = `connected to ${SPEECH_TO_SPEECH_URL}`; + wsStatusEl.textContent = `socket: connected to ${SPEECH_TO_SPEECH_URL}`; wsStatusEl.style.color = 'green'; await startMicrophone(); @@ -38,7 +38,7 @@ async function startMicrophone() { try { sts_mediaStream = await navigator.mediaDevices.getUserMedia({audio: true}); console.log("mediaStream is: ", sts_mediaStream) - micStatusEl.textContent = 'active'; + micStatusEl.textContent = 'microphone: active'; micStatusEl.style.color = 'green'; sts_audioCtx = new (window.AudioContext || window.webkitAudioContext)({sampleRate: SPEECH_TO_SPEECH_SAMPLE_RATE}); @@ -115,9 +115,9 @@ function stopSession() { sts_audioCtx = null; } - wsStatusEl.textContent = 'disconnected'; + wsStatusEl.textContent = 'socket: disconnected'; wsStatusEl.style.color = 'red'; - micStatusEl.textContent = 'disabled'; + micStatusEl.textContent = 'microphone: disabled'; micStatusEl.style.color = 'red'; speechBtn.innerText = 'Start'; sts_started = false; From 87bba817ce16466c6ff6e48511c792f4aca197a0 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Fri, 31 Jul 2026 10:46:19 +0200 Subject: [PATCH 67/67] polish code style --- .../openai/realtime/BufferedWebSocketListener.java | 2 -- .../foundationmodels/openai/realtime/RealtimeParam.java | 1 - .../openai/realtime/RealtimeParamSystemPrompt.java | 1 - .../openai/realtime/RealtimeParamTurnDetection.java | 7 ++++--- .../openai/realtime/RealtimeParamVoice.java | 4 ++-- .../openai/realtime/ToAudioRealtimeClient.java | 1 - .../src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java | 4 ---- .../com/sap/ai/sdk/app/controllers/OpenAiController.java | 3 --- .../sdk/app/realtime/SpeechToSpeechWebsocketHandler.java | 4 ---- .../ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java | 4 ---- 10 files changed, 6 insertions(+), 25 deletions(-) diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java index 390a1d63b..6e5ac52ba 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java @@ -7,8 +7,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nonnull; - -import com.google.common.annotations.Beta; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java index 2909271ab..26214db7b 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; - import javax.annotation.Nonnull; /** Represents possible configuration params of realtime client */ diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java index 7b0fc910f..10ec75d3a 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; - import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java index 980d9479f..cb087bca2 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamTurnDetection.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; - import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -11,7 +10,8 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { /** Model tries to recognize if/when it should respond automatically */ - @Beta public static final RealtimeParamTurnDetection BY_MODEL_AUTO = + @Beta + public static final RealtimeParamTurnDetection BY_MODEL_AUTO = new RealtimeParamTurnDetection("BY_MODEL_AUTO"); /** @@ -19,7 +19,8 @@ public final class RealtimeParamTurnDetection implements RealtimeParam { * Less convenient than the automatic option but may give lower latency in some cases (model does * not need to perform additional turn detection analysis). */ - @Beta public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = + @Beta + public static final RealtimeParamTurnDetection EACH_CALL_IS_A_TURN = new RealtimeParamTurnDetection("EACH_CALL_IS_A_TURN"); private final String turnDetectionKind; diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java index bcc5030af..e5ff23f0b 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamVoice.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.google.common.annotations.Beta; - import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -25,7 +24,8 @@ public final class RealtimeParamVoice implements RealtimeParam { /** * Allows to configure raw voice name as named by model provider. Unsafe because SDK cannot verify * in advance if the provided voice name is correct and supported by the chosen model and use case - * NOTE: this method does not check voice name and incorrect input may produce runtime exceptions (unsafe) + * NOTE: this method does not check voice name and incorrect input may produce runtime exceptions + * (unsafe) * * @param voiceName as named by model provider * @return typed voice client configuration param diff --git a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java index 3416030d5..0187c0b1d 100644 --- a/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java +++ b/foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/ToAudioRealtimeClient.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.foundationmodels.openai.realtime; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.Beta; import com.openai.models.realtime.RealtimeAudioConfig; import com.openai.models.realtime.RealtimeAudioConfigInput; import com.openai.models.realtime.RealtimeAudioConfigOutput; diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java index a93004d48..0b8a56d22 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/WebsocketConfig.java @@ -1,6 +1,5 @@ package com.sap.ai.sdk.app; -import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.realtime.SpeechToSpeechWebsocketHandler; import com.sap.ai.sdk.app.realtime.TextToSpeechWebsocketHandler; import javax.annotation.Nonnull; @@ -12,7 +11,6 @@ /** Implements spring Web Socket configuration to expose Web Socket handlers for Realtime API */ @Configuration @EnableWebSocket -@Beta public class WebsocketConfig implements WebSocketConfigurer { private final TextToSpeechWebsocketHandler textToSpeech; @@ -24,7 +22,6 @@ public class WebsocketConfig implements WebSocketConfigurer { * @param textToSpeech - text to speech realtime api handler * @param speechToSpeech - speech to speech realtime api handler */ - @Beta public WebsocketConfig( @Nonnull final TextToSpeechWebsocketHandler textToSpeech, @Nonnull final SpeechToSpeechWebsocketHandler speechToSpeech) { @@ -38,7 +35,6 @@ public WebsocketConfig( * @param registry - registry where to register handlers */ @Override - @Beta public void registerWebSocketHandlers(@Nonnull final WebSocketHandlerRegistry registry) { registry.addHandler(textToSpeech, "/text-to-speech"); registry.addHandler(speechToSpeech, "/speech-to-speech"); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java index 446ad5061..9d15231e5 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/controllers/OpenAiController.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.core.common.ClientException; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; @@ -56,7 +55,6 @@ Object chatCompletion( @GetMapping(value = "/realtime/smokeTestTextToSpeech", produces = "audio/pcm") @Nonnull - @Beta ResponseEntity smokeTestTextToSpeech() throws TimeoutException { final var respBody = ByteBuffer.allocate(300000); final var lock = new AtomicBoolean(false); @@ -89,7 +87,6 @@ ResponseEntity smokeTestTextToSpeech() throws TimeoutException { @GetMapping(value = "/realtime/smokeTestSpeechToSpeech", produces = "audio/pcm") @Nonnull - @Beta ResponseEntity smokeTestSpeechToSpeech() throws TimeoutException { final var respBody = ByteBuffer.allocate(800000); final var lock = new AtomicBoolean(false); diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java index 52b7938d9..3a43b191d 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/SpeechToSpeechWebsocketHandler.java @@ -1,6 +1,5 @@ package com.sap.ai.sdk.app.realtime; -import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel; import java.io.IOException; @@ -19,7 +18,6 @@ /** Implements handler (Web Socket messages handling) for speech to speech realtime api operation */ @Component @Slf4j -@Beta public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; @@ -31,7 +29,6 @@ public class SpeechToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired - @Beta public SpeechToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); @@ -61,7 +58,6 @@ protected void handleBinaryMessage( } @Override - @Beta public void afterConnectionClosed( @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent( diff --git a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java index 7dba3ca8c..c15f13a03 100644 --- a/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java +++ b/sample-code/spring-app/src/main/java/com/sap/ai/sdk/app/realtime/TextToSpeechWebsocketHandler.java @@ -1,6 +1,5 @@ package com.sap.ai.sdk.app.realtime; -import com.google.common.annotations.Beta; import com.sap.ai.sdk.app.services.OpenAiService; import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel; import java.io.IOException; @@ -20,7 +19,6 @@ /** Implements handler (Web Socket messages handling) for text to speech realtime api operation */ @Component @Slf4j -@Beta public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { private final OpenAiService service; @@ -32,7 +30,6 @@ public class TextToSpeechWebsocketHandler extends BinaryWebSocketHandler { * @param service - handling service */ @Autowired - @Beta public TextToSpeechWebsocketHandler(@Nonnull final OpenAiService service) { this.service = service; channels = new ConcurrentHashMap<>(); @@ -62,7 +59,6 @@ protected void handleBinaryMessage( } @Override - @Beta public void afterConnectionClosed( @Nonnull final WebSocketSession session, @Nonnull final CloseStatus status) throws Exception { channels.computeIfPresent(