feat: OpenAI realtime api

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioInputChannel.java

@@ -0,0 +1,22 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * Functional interface representing audio input channel (audio data consumer)
+ *
+ * <p>Should be closed by application (try-with-resources) when not needed anymore
+ */
+@Beta
+public interface AudioInputChannel extends AutoCloseable {
+
+  /**
+   * This method is sequentially invoked by audio data provider to supply implementer (consumer)
+   * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage
+   * context
+   *
+   * @param rawBytesChunk binary data in the depending on the use case format
+   */
+  @Beta
+  void inputAudio(byte[] rawBytesChunk);
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/AudioOutputChannel.java

@@ -0,0 +1,21 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import com.google.common.annotations.Beta;
+
+/** Functional interface representing audio output channel (audio data consumer) */
+@Beta
+public interface AudioOutputChannel {
+
+  /**
+   * This method is sequentially invoked by audio data provider to supply implementer (consumer)
+   * with the audio data. Exact audio format (encoding, sampling rate, etc.) depends on the usage
+   * context
+   *
+   * @param rawBytesChunk binary data in the depending on the use case format
+   * @param isLast true if this call logically concludes previous and this passed bytes data into a
+   *     single logical entity (e.g. gets called at the end when all byte parts of a single message
+   *     get passed)
+   */
+  @Beta
+  void outputAudio(byte[] rawBytesChunk, boolean isLast);
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiClient.java

@@ -26,6 +26,7 @@
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiChatMessage.OpenAiChatUserMessage;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingOutput;
 import com.sap.ai.sdk.foundationmodels.openai.model.OpenAiEmbeddingParameters;
+import com.sap.ai.sdk.foundationmodels.openai.realtime.OpenAiRealtimeClient;
 import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor;
 import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination;
 import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
@@ -76,6 +77,18 @@ public static OpenAiClient forModel(@Nonnull final OpenAiModel foundationModel)
     return client.withApiVersion(DEFAULT_API_VERSION);
   }
 
+  /**
+   * Creates and configures OpenAI Realtime API client
+   *
+   * @return created client
+   */
+  @Beta
+  @Nonnull
+  public static OpenAiRealtimeClient realtimeClient() {
+    final var withResolvedDestination = OpenAiClient.forModel(OpenAiModel.GPT_REALTIME);
+    return new OpenAiRealtimeClient(withResolvedDestination.destination);
+  }
+
   /**
    * Create a new OpenAI client targeting the specified API version.
    *

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/OpenAiModel.java

@@ -121,7 +121,7 @@ public record OpenAiModel(@Nonnull String name, @Nullable String version) implem
   /** Azure OpenAI GPT-5-nano model */
   public static final OpenAiModel GPT_5_NANO = new OpenAiModel("gpt-5-nano", null);
 
-  /** Azure OpenAI GPT-5-nano model */
+  /** Azure OpenAI GPT-realtime model */
   public static final OpenAiModel GPT_REALTIME = new OpenAiModel("gpt-realtime", null);
 
   /** Azure OpenAI GPT-5.2 model */

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/TextInputChannel.java

@@ -0,0 +1,20 @@
+package com.sap.ai.sdk.foundationmodels.openai;
+
+import com.google.common.annotations.Beta;
+import javax.annotation.Nonnull;
+
+/**
+ * Allows to input (send) text to the open channel, must be closed when not needed anymore (e.g.
+ * try-with-resources)
+ */
+@Beta
+public interface TextInputChannel extends AutoCloseable {
+
+  /**
+   * Sends input text
+   *
+   * @param text text to send
+   */
+  @Beta
+  void sendText(@Nonnull final String text);
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/BufferedWebSocketListener.java

@@ -0,0 +1,60 @@
+package com.sap.ai.sdk.foundationmodels.openai.realtime;
+
+import java.net.http.WebSocket;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+class BufferedWebSocketListener implements WebSocket.Listener {
+
+  private final Consumer<WebSocket> onOpen;
+  private final BiConsumer<WebSocket, CharSequence> onText;
+  private final StringBuilder buffer;
+
+  BufferedWebSocketListener(
+      @Nonnull final Consumer<WebSocket> onOpen,
+      @Nonnull final BiConsumer<WebSocket, CharSequence> onText) {
+    this.onOpen = onOpen;
+    this.onText = onText;
+    this.buffer = new StringBuilder(256 * 1024);
+  }
+
+  @Override
+  public void onOpen(@Nonnull final WebSocket webSocket) {
+    this.onOpen.accept(webSocket);
+    webSocket.request(1);
+  }
+
+  @Override
+  @Nonnull
+  public CompletionStage<?> onText(
+      final @Nonnull WebSocket webSocket, final @Nonnull CharSequence data, final boolean isLast) {
+    buffer.append(data);
+    webSocket.request(1);
+    if (isLast) {
+      final var completeMessage = buffer.toString();
+      buffer.setLength(0);
+      this.onText.accept(webSocket, completeMessage);
+    }
+
+    return CompletableFuture.completedStage(null);
+  }
+
+  @Override
+  public void onError(@Nonnull final WebSocket webSocket, @Nonnull final Throwable error) {
+    log.error("Websocket error occurred during realtime communication", error);
+  }
+
+  @Override
+  @Nonnull
+  public CompletionStage<?> onBinary(
+      @Nonnull final WebSocket webSocket, @Nonnull final ByteBuffer data, final boolean isLast) {
+    log.warn("Received unexpected binary bytes for WebSocket connection");
+    return CompletableFuture.completedStage(null);
+  }
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/OpenAiRealtimeClient.java

@@ -0,0 +1,128 @@
+package com.sap.ai.sdk.foundationmodels.openai.realtime;
+
+import com.google.common.annotations.Beta;
+import com.sap.ai.sdk.foundationmodels.openai.AudioInputChannel;
+import com.sap.ai.sdk.foundationmodels.openai.AudioOutputChannel;
+import com.sap.ai.sdk.foundationmodels.openai.TextInputChannel;
+import com.sap.cloud.sdk.cloudplatform.connectivity.Destination;
+import com.sap.cloud.sdk.cloudplatform.connectivity.Header;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nonnull;
+
+/**
+ * OpenAI client implementation of Realtime API. Abstracts technical implementation, transport and
+ * threading and exposes business-level operations (high level interface)
+ */
+@Beta
+public class OpenAiRealtimeClient {
+
+  static final int PATH_BUFFER_SIZE =
+      400; // existing URLs are ~120 symbols long, 400 has reasonable margin
+
+  final Destination destination;
+
+  /**
+   * Created OpenAI Realtime client for a specific destination
+   *
+   * @param destination - destination to use
+   */
+  @Beta
+  public OpenAiRealtimeClient(@Nonnull final Destination destination) {
+    this.destination = destination;
+  }
+
+  /**
+   * Creates a realtime channel allowing to input text and voice it (receive audio output)
+   *
+   * <p>The input channel should be used with a try-with-resources block to ensure that the
+   * underlying connection is closed.
+   *
+   * <p>Example:
+   *
+   * <pre>{@code
+   * try (var textInputChannel = client.textToSpeech(audioOutputConsumer)) {
+   *       textInputChannel.sendText("...");
+   *       ....
+   * }
+   * }</pre>
+   *
+   * This API implements full duplex (input + output) communication channels. Application should
+   * logically synchronize their state and close the input channel when it is appropriate (e.g. the
+   * last part of the response has been received via the output channel and the application does not
+   * need to send any other input). When the input channel is closed, the output channel will be
+   * closed automatically and the output consumer will not be called anymore.
+   *
+   * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16
+   *     bit depth
+   * @param params - allows for various additional features (e.g. voice configuration or
+   *     conversation turn recognition options)
+   * @return input channel, allowing for text input
+   */
+  @Nonnull
+  @Beta
+  public TextInputChannel textToSpeech(
+      @Nonnull final AudioOutputChannel audioOutputConsumer,
+      @Nonnull final RealtimeParam... params) {
+    return new TextToSpeechRealtimeClient(
+        getRealtimeEndpoint(), buildRealtimeHeaders(), audioOutputConsumer, params);
+  }
+
+  /**
+   * Creates a realtime channel allowing for audio conversation with a model.
+   *
+   * <p>The input channel should be used with a try-with-resources block to ensure that the
+   * underlying connection is closed.
+   *
+   * <p>Example:
+   *
+   * <pre>{@code
+   * try (var audioInputChannel = client.speechToSpeech(audioOutputConsumer)) {
+   *       audioInputChannel.inputAudio(audioBytesData);
+   *       ....
+   * }
+   * }</pre>
+   *
+   * This API implements full duplex (input + output) communication channels. An application should
+   * logically synchronize their state and close the input channel when it is appropriate (e.g. the
+   * last part of the response has been received via the output channel and the application does not
+   * need to send any other input). When the input channel is closed, the output channel will be
+   * closed automatically and the output consumer will not be called anymore.
+   *
+   * @param audioOutputConsumer - audio consumer of raw PCM mono 24000 Hz little endian output, 16
+   *     bit depth
+   * @param params - optional configuration params
+   * @return input channel, allowing for audio data input (bytes, PCM mono 24000 Hz little endian 16
+   *     bit)
+   */
+  @Nonnull
+  @Beta
+  public AudioInputChannel speechToSpeech(
+      @Nonnull final AudioOutputChannel audioOutputConsumer,
+      @Nonnull final RealtimeParam... params) {
+    return new SpeechToSpeechRealtimeClient(
+        getRealtimeEndpoint(), buildRealtimeHeaders(), audioOutputConsumer, params);
+  }
+
+  Map<String, String> buildRealtimeHeaders() {
+    final var extraHeaders = destination.asHttp().getHeaders();
+    final var headers = new HashMap<String, String>(extraHeaders.size() + 1);
+    for (final Header header : extraHeaders) {
+      headers.put(header.getName(), header.getValue());
+    }
+    return headers;
+  }
+
+  String getRealtimeEndpoint() {
+    final var sb = new StringBuilder(PATH_BUFFER_SIZE);
+    sb.append("wss://");
+    final var pathParts = destination.asHttp().getUri().toString().split("//");
+    if (pathParts.length != 2) {
+      throw new IllegalArgumentException(
+          "Invalid destination URI: " + destination.asHttp().getUri());
+    }
+    sb.append(pathParts[1].replaceFirst("^api\\.", "realtime."));
+    sb.append("/v1/realtime");
+    return sb.toString();
+  }
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParam.java

@@ -0,0 +1,37 @@
+package com.sap.ai.sdk.foundationmodels.openai.realtime;
+
+import com.google.common.annotations.Beta;
+import javax.annotation.Nonnull;
+
+/** Represents possible configuration params of realtime client */
+@Beta
+public interface RealtimeParam {
+  /** Represents configurable options */
+  enum ParamName {
+    /** Voice name to use to produce sound */
+    OUTPUT_VOICE,
+    /**
+     * How model will recognize that it is its turn to respond (e.g. explicitly asked, automatically
+     * detected)
+     */
+    TURN_DETECTION,
+    /** Override or specify system prompt given to a model */
+    SYSTEM_PROMPT,
+  }
+
+  /**
+   * Returns param name
+   *
+   * @return name
+   */
+  @Nonnull
+  ParamName getParamName();
+
+  /**
+   * Returns string value representation of the param
+   *
+   * @return string value
+   */
+  @Nonnull
+  String getValueAsString();
+}

foundation-models/openai/src/main/java/com/sap/ai/sdk/foundationmodels/openai/realtime/RealtimeParamSystemPrompt.java

@@ -0,0 +1,51 @@
+package com.sap.ai.sdk.foundationmodels.openai.realtime;
+
+import com.google.common.annotations.Beta;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/** Allows to configure model system prompt */
+@Beta
+public final class RealtimeParamSystemPrompt implements RealtimeParam {
+
+  private final String systemPrompt;
+
+  /**
+   * Constructs RealtimeParamSystemPrompt object
+   *
+   * @param systemPrompt system prompt to use
+   */
+  @Beta
+  public RealtimeParamSystemPrompt(@Nonnull final String systemPrompt) {
+    this.systemPrompt = systemPrompt;
+  }
+
+  @Override
+  @Beta
+  public @Nonnull ParamName getParamName() {
+    return ParamName.SYSTEM_PROMPT;
+  }
+
+  @Override
+  @Beta
+  public @Nonnull String getValueAsString() {
+    return systemPrompt;
+  }
+
+  @Override
+  @Beta
+  public boolean equals(@Nullable final Object o) {
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    final RealtimeParamSystemPrompt that = (RealtimeParamSystemPrompt) o;
+    return Objects.equals(systemPrompt, that.systemPrompt);
+  }
+
+  @Override
+  @Beta
+  public int hashCode() {
+    return Objects.hashCode(systemPrompt);
+  }
+}