diff --git a/.changes/audio-manager-api b/.changes/audio-manager-api new file mode 100644 index 000000000..2fd918ba1 --- /dev/null +++ b/.changes/audio-manager-api @@ -0,0 +1 @@ +minor type="added" "AudioManager audio session options with engine-driven native lifecycle and platform routing controls" diff --git a/README.md b/README.md index 9a80d3dfb..e2357d02e 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,16 @@ Widget build(BuildContext context) { Audio tracks are played automatically as long as you are subscribed to them. +LiveKit owns the platform audio session through `AudioManager`, which lets you choose a session intent, route to the speaker, and pin per platform values. See the [audio session guide](https://github.com/livekit/client-sdk-flutter/blob/main/docs/audio.md) for examples covering communication and media modes, speaker routing, manual mode, per platform overrides, and migration from the older `Hardware` APIs. + +```dart +// Enter call mode and prefer the speaker +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.communication(), +); +await AudioManager.instance.setSpeakerOutputPreferred(true); +``` + ### Handling changes LiveKit client makes it simple to build declarative UI that reacts to state changes. It notifies changes in two ways diff --git a/android/build.gradle b/android/build.gradle index 82e5dd8d7..5a671d109 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -18,6 +18,7 @@ allprojects { repositories { google() mavenCentral() + maven { url 'https://jitpack.io' } } } @@ -61,6 +62,9 @@ android { testImplementation("org.mockito:mockito-core:5.0.0") implementation 'io.github.webrtc-sdk:android:144.7559.09' implementation 'io.livekit:noise:2.0.0' + // Audio device/focus/mode routing. Pinned to the same revision used by + // the LiveKit Android SDK (AudioSwitchHandler). + implementation 'com.github.davidliu:audioswitch:039a35aefab7747c557242fa216c9ea11743b604' } testOptions { diff --git a/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt new file mode 100644 index 000000000..18e48d4fb --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/LKAudioSwitchManager.kt @@ -0,0 +1,265 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.os.HandlerThread +import com.twilio.audioswitch.AbstractAudioSwitch +import com.twilio.audioswitch.AudioDevice +import com.twilio.audioswitch.AudioSwitch +import com.twilio.audioswitch.CommDeviceAudioSwitch +import com.twilio.audioswitch.LegacyAudioSwitch + +/** + * Manages the Android platform audio session (audio mode, audio focus, and + * output routing) for the LiveKit Flutter SDK, built on top of [AudioSwitch]. + * + * This is LiveKit's own port of the audio-handling best practices from the + * LiveKit Android SDK (`AudioSwitchHandler`) and flutter_webrtc + * (`AudioSwitchManager`), so the Flutter SDK can own the platform audio session + * directly instead of delegating to flutter_webrtc's native audio management. + * + * [AudioSwitch] is not thread-safe, so every interaction with it runs on a + * single dedicated [HandlerThread]. + */ +internal class LKAudioSwitchManager(private val context: Context) { + // AudioSwitch is not threadsafe, so confine all access to a single long-lived + // thread. The AudioSwitch instance is recreated per active session, while + // queued lifecycle work stays serialized on this thread. + private val thread = HandlerThread("LKAudioSwitchThread").also { it.start() } + private val handler = Handler(thread.looper) + + private var audioSwitch: AbstractAudioSwitch? = null + private var isActive = false + + // Configuration. Defaults mirror a communication/VoIP session and match the + // AudioSwitchHandler defaults in the LiveKit Android SDK. + private val loggingEnabled = false + private var manageAudioFocus = true + private var audioMode = AudioManager.MODE_IN_COMMUNICATION + private var focusMode = AudioManager.AUDIOFOCUS_GAIN + private var audioStreamType = AudioManager.STREAM_VOICE_CALL + private var audioAttributeUsageType = AudioAttributes.USAGE_VOICE_COMMUNICATION + private var audioAttributeContentType = AudioAttributes.CONTENT_TYPE_SPEECH + private var forceHandleAudioRouting = false + + private var speakerOutputPreferred = true + private var speakerOutputForced = false + private var preferredDeviceList = preferredDeviceList() + + /** + * Apply an audio session configuration. Unspecified keys keep their current + * value. Changes are applied to a running [AudioSwitch] without a restart. + */ + @Synchronized + fun configure(configuration: Map) { + (configuration["manageAudioFocus"] as? Boolean)?.let { manageAudioFocus = it } + audioModeForName(configuration["androidAudioMode"] as? String)?.let { audioMode = it } + focusModeForName(configuration["androidAudioFocusMode"] as? String)?.let { focusMode = it } + streamTypeForName(configuration["androidAudioStreamType"] as? String)?.let { audioStreamType = it } + usageTypeForName(configuration["androidAudioAttributesUsageType"] as? String)?.let { audioAttributeUsageType = it } + contentTypeForName(configuration["androidAudioAttributesContentType"] as? String)?.let { audioAttributeContentType = it } + (configuration["forceHandleAudioRouting"] as? Boolean)?.let { forceHandleAudioRouting = it } + + // Apply to a live switch so reconfiguration (e.g. communication -> media) + // does not require a restart. No-op until the switch exists. + handler.post { audioSwitch?.let { applyConfiguration(it) } } + } + + /** Create (if needed) and activate the audio session: acquire focus, set mode and routing. */ + @Synchronized + fun start() { + handler.post { + val switch = audioSwitch ?: createSwitch().also { audioSwitch = it } + if (!isActive) { + switch.activate() + applySpeakerRouting(switch) + isActive = true + } + } + } + + /** Deactivate and tear down the audio session: release focus and restore the previous mode. */ + @Synchronized + fun stop() { + handler.post { + audioSwitch?.stop() + audioSwitch = null + isActive = false + } + } + + /** Final cleanup when the plugin detaches. The manager must not be used after this. */ + @Synchronized + fun dispose() { + handler.post { + audioSwitch?.stop() + audioSwitch = null + isActive = false + thread.quitSafely() + } + } + + /** + * Prefer routing to/from the speaker, letting a connected headset keep priority + * unless [force] is true. + */ + @Synchronized + fun setSpeakerphoneOn(enable: Boolean, force: Boolean) { + speakerOutputPreferred = enable + speakerOutputForced = enable && force + preferredDeviceList = preferredDeviceList() + handler.post { + val switch = audioSwitch ?: return@post + applySpeakerRouting(switch) + } + } + + private fun createSwitch(): AbstractAudioSwitch { + val focusListener = AudioManager.OnAudioFocusChangeListener { } + // API-aware switch selection, matching the LiveKit Android SDK's + // AudioSwitchHandler: CommDeviceAudioSwitch uses the modern + // AudioManager.setCommunicationDevice routing on API 31+. + val switch = when { + Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> + CommDeviceAudioSwitch(context, loggingEnabled, focusListener, preferredDeviceList) + + Build.VERSION.SDK_INT >= Build.VERSION_CODES.M -> + AudioSwitch(context, loggingEnabled, focusListener, preferredDeviceList) + + else -> + LegacyAudioSwitch(context, loggingEnabled, focusListener, preferredDeviceList) + } + applyConfiguration(switch) + switch.start { _, _ -> } + return switch + } + + private fun applyConfiguration(switch: AbstractAudioSwitch) { + switch.manageAudioFocus = manageAudioFocus + switch.audioMode = audioMode + switch.focusMode = focusMode + switch.audioStreamType = audioStreamType + switch.audioAttributeUsageType = audioAttributeUsageType + switch.audioAttributeContentType = audioAttributeContentType + switch.forceHandleAudioRouting = forceHandleAudioRouting + } + + private fun applySpeakerRouting(switch: AbstractAudioSwitch) { + switch.setPreferredDeviceList(preferredDeviceList) + val forcedSpeaker = if (speakerOutputForced) { + switch.availableAudioDevices.firstOrNull { it is AudioDevice.Speakerphone } + } else { + null + } + // AudioSwitch selections are sticky. Use them only for forced speaker output; + // clearing the selection lets the preferred-device list handle normal routing + // and headset hot-plug priority. + switch.selectDevice(forcedSpeaker) + } + + private fun preferredDeviceList(): List> = + when { + speakerOutputForced -> listOf( + AudioDevice.Speakerphone::class.java, + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Earpiece::class.java, + ) + + speakerOutputPreferred -> listOf( + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Speakerphone::class.java, + AudioDevice.Earpiece::class.java, + ) + + else -> listOf( + AudioDevice.BluetoothHeadset::class.java, + AudioDevice.WiredHeadset::class.java, + AudioDevice.Earpiece::class.java, + AudioDevice.Speakerphone::class.java, + ) + } +} + +// Map the Flutter-side enum names (see android_audio_session_adapter.dart) to +// Android framework constants. Ported from flutter_webrtc's AudioUtils. + +private fun audioModeForName(name: String?): Int? = when (name) { + null -> null + "normal" -> AudioManager.MODE_NORMAL + "callScreening" -> AudioManager.MODE_CALL_SCREENING + "inCall" -> AudioManager.MODE_IN_CALL + "inCommunication" -> AudioManager.MODE_IN_COMMUNICATION + "ringtone" -> AudioManager.MODE_RINGTONE + else -> null +} + +private fun focusModeForName(name: String?): Int? = when (name) { + null -> null + "gain" -> AudioManager.AUDIOFOCUS_GAIN + "gainTransient" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + "gainTransientExclusive" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE + "gainTransientMayDuck" -> AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + else -> null +} + +private fun streamTypeForName(name: String?): Int? = when (name) { + null -> null + "accessibility" -> AudioManager.STREAM_ACCESSIBILITY + "alarm" -> AudioManager.STREAM_ALARM + "dtmf" -> AudioManager.STREAM_DTMF + "music" -> AudioManager.STREAM_MUSIC + "notification" -> AudioManager.STREAM_NOTIFICATION + "ring" -> AudioManager.STREAM_RING + "system" -> AudioManager.STREAM_SYSTEM + "voiceCall" -> AudioManager.STREAM_VOICE_CALL + else -> null +} + +private fun usageTypeForName(name: String?): Int? = when (name) { + null -> null + "alarm" -> AudioAttributes.USAGE_ALARM + "assistanceAccessibility" -> AudioAttributes.USAGE_ASSISTANCE_ACCESSIBILITY + "assistanceNavigationGuidance" -> AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE + "assistanceSonification" -> AudioAttributes.USAGE_ASSISTANCE_SONIFICATION + "assistant" -> AudioAttributes.USAGE_ASSISTANT + "game" -> AudioAttributes.USAGE_GAME + "media" -> AudioAttributes.USAGE_MEDIA + "notification" -> AudioAttributes.USAGE_NOTIFICATION + "notificationEvent" -> AudioAttributes.USAGE_NOTIFICATION_EVENT + "notificationRingtone" -> AudioAttributes.USAGE_NOTIFICATION_RINGTONE + "unknown" -> AudioAttributes.USAGE_UNKNOWN + "voiceCommunication" -> AudioAttributes.USAGE_VOICE_COMMUNICATION + "voiceCommunicationSignalling" -> AudioAttributes.USAGE_VOICE_COMMUNICATION_SIGNALLING + else -> null +} + +private fun contentTypeForName(name: String?): Int? = when (name) { + null -> null + "movie" -> AudioAttributes.CONTENT_TYPE_MOVIE + "music" -> AudioAttributes.CONTENT_TYPE_MUSIC + "sonification" -> AudioAttributes.CONTENT_TYPE_SONIFICATION + "speech" -> AudioAttributes.CONTENT_TYPE_SPEECH + "unknown" -> AudioAttributes.CONTENT_TYPE_UNKNOWN + else -> null +} diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt index 95a242497..925890628 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt @@ -26,6 +26,7 @@ import io.flutter.plugin.common.MethodChannel.MethodCallHandler import io.flutter.plugin.common.MethodChannel.Result import com.cloudwebrtc.webrtc.FlutterWebRTCPlugin +import com.cloudwebrtc.webrtc.audio.AudioSwitchManager import com.cloudwebrtc.webrtc.audio.LocalAudioTrack import io.flutter.plugin.common.BinaryMessenger import org.webrtc.AudioTrack @@ -42,6 +43,7 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private var audioProcessors = mutableMapOf() private var flutterWebRTCPlugin = FlutterWebRTCPlugin.sharedSingleton private var binaryMessenger: BinaryMessenger? = null + private var audioSwitchManager: LKAudioSwitchManager? = null /// The MethodChannel that will the communication between Flutter and native Android /// @@ -50,9 +52,13 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { private lateinit var channel: MethodChannel override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { + // LiveKit owns the platform audio session, so disable flutter_webrtc's own + // native audio management. Set at registration, before any audio op. + AudioSwitchManager.setAudioSessionManagementEnabled(false) channel = MethodChannel(flutterPluginBinding.binaryMessenger, "livekit_client") channel.setMethodCallHandler(this) binaryMessenger = flutterPluginBinding.binaryMessenger + audioSwitchManager = LKAudioSwitchManager(flutterPluginBinding.applicationContext) } @SuppressLint("SuspiciousIndentation") @@ -350,6 +356,26 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { handleGetAudioProcessingState(result) } + "configureAndroidAudioSession" -> { + @Suppress("UNCHECKED_CAST") + val configuration = call.arguments as? Map ?: emptyMap() + audioSwitchManager?.configure(configuration) + audioSwitchManager?.start() + result.success(null) + } + + "stopAndroidAudioSession" -> { + audioSwitchManager?.stop() + result.success(null) + } + + "setAndroidSpeakerphoneOn" -> { + val enable = call.argument("enable") ?: false + val force = call.argument("force") ?: false + audioSwitchManager?.setSpeakerphoneOn(enable, force) + result.success(null) + } + else -> { result.notImplemented() } @@ -359,6 +385,9 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler { override fun onDetachedFromEngine(@NonNull binding: FlutterPlugin.FlutterPluginBinding) { channel.setMethodCallHandler(null) + audioSwitchManager?.dispose() + audioSwitchManager = null + // Cleanup all processors audioProcessors.values.forEach { it.cleanup() } audioProcessors.clear() diff --git a/docs/audio.md b/docs/audio.md new file mode 100644 index 000000000..bb7a15cc5 --- /dev/null +++ b/docs/audio.md @@ -0,0 +1,177 @@ +# Audio session management + +LiveKit owns the platform audio session on iOS and Android through a single process-wide entry point, `AudioManager`. You configure session intent once with typed options and LiveKit applies the right native category, mode, focus, and routing for you. On macOS, `AudioManager` reports native audio-engine state but does not configure a platform audio session. + +`AudioManager` is a singleton, reached through `AudioManager.instance`. + +## Quick start + +Pick a session intent. `communication` is the default and is meant for calls and rooms where you both send and receive audio. `media` is for one way playback or media capture where communication mode is not wanted. + +```dart +import 'package:livekit_client/livekit_client.dart'; + +// VoIP style call (this is the default, so you only need it to be explicit) +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.communication(), +); + +// Media playback or live streaming capture +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.media(), +); +``` + +Set this before connecting when you can. The explicit apply path above works in both automatic and manual management modes. + +## Speaker routing + +```dart +// Prefer the speaker. A wired or Bluetooth headset still takes priority. +await AudioManager.instance.setSpeakerOutputPreferred(true); + +// Force the speaker even when a headset is connected. +await AudioManager.instance.setSpeakerOutputPreferred(true, force: true); + +// Route back to the earpiece or the connected headset. +await AudioManager.instance.setSpeakerOutputPreferred(false); +``` + +Read the current preference through `AudioManager.instance.isSpeakerOutputPreferred` and `AudioManager.instance.isSpeakerOutputForced`. `AudioManager.instance.canSwitchSpeakerphone` is true on iOS and Android. + +`Room.setSpeakerOn(...)` still works and forwards to the same path, so existing call sites do not need to change. + +## Automatic vs manual mode + +In automatic mode (the default) LiveKit updates the audio session from room, connect, and engine lifecycle. In manual mode LiveKit does not touch the session on its own, and your app drives it explicitly with `setAudioSessionOptions` and `deactivateAudioSession`. + +```dart +// Hand session control to the app. +await AudioManager.instance.setAudioSessionManagementMode( + AudioSessionManagementMode.manual, +); + +// Apply a configuration yourself whenever you need it. +await AudioManager.instance.setAudioSessionOptions( + const AudioSessionOptions.communication(), +); + +// Re-apply the current options, for example after an interruption. +await AudioManager.instance.applyCurrentAudioSessionOptions(); + +// Release the session when your manual lifecycle no longer needs it. +await AudioManager.instance.deactivateAudioSession(); +``` + +Prefer setting the mode before connecting to a room. + +## Per platform overrides + +When the presets are not enough you can pin exact platform values. Anything you do not set falls back to the preset behavior. + +```dart +await AudioManager.instance.setAudioSessionOptions( + AudioSessionOptions.communication( + apple: const AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.mixWithOthers, + }, + mode: AppleAudioMode.voiceChat, + ), + android: AndroidAudioSessionConfiguration.communication, + ), +); +``` + +Supplying an explicit `apple` override turns off dynamic category selection, so your configuration is applied as written. + +### Updating options with copyWith + +`copyWith` uses `Value` and `Absent` so it can tell apart leaving a field alone from setting it to null. A bare `copyWith()` keeps the existing value, `Value(x)` sets a new value, and `Value(null)` clears it. + +```dart +final base = AudioSessionOptions.communication( + apple: const AppleAudioSessionConfiguration(mode: AppleAudioMode.voiceChat), +); + +// Change one field, keep the rest. +final updated = base.copyWith(preferSpeakerOutput: const Value(false)); + +// Clear the Apple override entirely. +final cleared = base.copyWith(apple: const Value(null)); +``` + +If `Value` collides with another package in your imports, alias the import. + +```dart +import 'package:livekit_client/livekit_client.dart' as lk; + +final cleared = base.copyWith(apple: const lk.Value(null)); +``` + +## Observing audio engine state + +On iOS and macOS the native audio engine reports when playout and recording turn on and off. This is the source of truth for audio activity. + +```dart +final sub = AudioManager.instance.audioEngineStateStream.listen((state) { + print('playout ${state.isPlayoutEnabled} recording ${state.isRecordingEnabled}'); + if (state.isIdle) { + print('engine is idle'); + } +}); + +// Current snapshot without listening. +final now = AudioManager.instance.audioEngineState; +``` + +## Initialization + +You can optionally call `LiveKitClient.initialize` once at startup. Passing `bypassVoiceProcessing: true` makes the default options media oriented for playback or capture without voice processing. Explicit runtime options that you set with `setAudioSessionOptions` are always preserved. + +```dart +await LiveKitClient.initialize(bypassVoiceProcessing: true); +``` + +flutter_webrtc's own native audio management is disabled automatically when the LiveKit plugin loads, so audio session ownership does not depend on this call. + +## Migrating from the old APIs + +The legacy `Hardware` audio members still work but are deprecated and forward to `AudioManager`. + +| Old | New | +| --- | --- | +| `Hardware.instance.setSpeakerphoneOn(true)` | `AudioManager.instance.setSpeakerOutputPreferred(true)` | +| `Hardware.instance.speakerOn` | `AudioManager.instance.isSpeakerOutputPreferred` | +| `Hardware.instance.preferSpeakerOutput` | `AudioManager.instance.isSpeakerOutputPreferred` | +| `Hardware.instance.forceSpeakerOutput` | `AudioManager.instance.isSpeakerOutputForced` | +| `Hardware.instance.setAutomaticConfigurationEnabled(enable: false)` | `AudioManager.instance.setAudioSessionManagementMode(AudioSessionManagementMode.manual)` | + +The old `onConfigureNativeAudio` hook (a deep src import) is removed. Replace a custom configuration function with explicit options. + +```dart +// Before: assigning onConfigureNativeAudio with a custom function. +// After: +await AudioManager.instance.setAudioSessionOptions( + AudioSessionOptions.communication( + apple: const AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + mode: AppleAudioMode.videoChat, + ), + ), +); +``` + +Dart track counting no longer drives the session. The native audio engine delegate drives it from real lifecycle events, which removes the timing races and missed deactivations of the counting approach. + +## Platform notes + +- **iOS** uses a WebRTC audio engine observer that configures and activates the session when the engine enables, and deactivates when it disables. Category is chosen from live engine state. A listen only or mic muted session uses the playback category until recording starts, and receiver routing from `preferSpeakerOutput` only applies while the effective category is `playAndRecord`. +- **macOS** emits the same engine events but has no `AVAudioSession`, so engine state is reported while no session category is applied. +- **Android** uses an AudioSwitch based manager for audio mode, focus, and routing. The session is activated when LiveKit applies its options (at connect, or on an explicit apply) and released on disconnect. + +## API reference + +The full generated API reference for these types lives at [pub.dev](https://pub.dev/documentation/livekit_client/latest/). Start from `AudioManager`, `AudioSessionOptions`, `AppleAudioSessionConfiguration`, and `AndroidAudioSessionConfiguration`. diff --git a/example/lib/pages/room.dart b/example/lib/pages/room.dart index 27aa394b8..42295242e 100644 --- a/example/lib/pages/room.dart +++ b/example/lib/pages/room.dart @@ -52,7 +52,7 @@ class _RoomPageState extends State { }); if (lkPlatformIs(PlatformType.android)) { - unawaited(Hardware.instance.setSpeakerphoneOn(true)); + unawaited(AudioManager.instance.setSpeakerOutputPreferred(true)); } if (lkPlatformIsDesktop()) { diff --git a/example/lib/widgets/controls.dart b/example/lib/widgets/controls.dart index 3c96b4ebc..b1102c32a 100644 --- a/example/lib/widgets/controls.dart +++ b/example/lib/widgets/controls.dart @@ -36,7 +36,7 @@ class _ControlsWidgetState extends State { StreamSubscription? _subscription; - bool _speakerphoneOn = Hardware.instance.speakerOn ?? false; + bool _speakerphoneOn = AudioManager.instance.isSpeakerOutputPreferred; @override void initState() { diff --git a/lib/livekit_client.dart b/lib/livekit_client.dart index 9756d5a07..20fb5e289 100644 --- a/lib/livekit_client.dart +++ b/lib/livekit_client.dart @@ -43,6 +43,7 @@ export 'src/participant/participant.dart'; export 'src/participant/remote.dart' hide ParticipantCreationResult; export 'src/audio/audio_manager.dart'; export 'src/audio/audio_frame_capture.dart' show AudioFormat, AudioFrame, AudioFrameCallback, AudioRendererOptions; +export 'src/audio/audio_session.dart'; export 'src/preconnect/pre_connect_audio_buffer.dart'; export 'src/publication/local.dart'; export 'src/publication/remote.dart'; diff --git a/lib/src/audio/android_audio_session_adapter.dart b/lib/src/audio/android_audio_session_adapter.dart new file mode 100644 index 000000000..adaaa7131 --- /dev/null +++ b/lib/src/audio/android_audio_session_adapter.dart @@ -0,0 +1,39 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart'; + +import '../support/native.dart'; +import 'audio_session.dart'; + +/// Serializes an [AndroidAudioSessionConfiguration] into the map consumed by +/// LiveKit's native Android audio session manager (and by flutter_webrtc's +/// audio device module at initialization). Unset fields are omitted so the +/// native side keeps its current value. +@internal +Map androidAudioSessionConfigurationToMap(AndroidAudioSessionConfiguration config) => + { + if (config.manageAudioFocus != null) 'manageAudioFocus': config.manageAudioFocus!, + if (config.audioMode != null) 'androidAudioMode': config.audioMode!.name, + if (config.focusMode != null) 'androidAudioFocusMode': config.focusMode!.name, + if (config.streamType != null) 'androidAudioStreamType': config.streamType!.name, + if (config.usageType != null) 'androidAudioAttributesUsageType': config.usageType!.name, + if (config.contentType != null) 'androidAudioAttributesContentType': config.contentType!.name, + if (config.forceAudioRouting != null) 'forceHandleAudioRouting': config.forceAudioRouting!, + }; + +@internal +Future setAndroidAudioSessionConfiguration(AndroidAudioSessionConfiguration config) async { + await Native.configureAndroidAudioSession(androidAudioSessionConfigurationToMap(config)); +} diff --git a/lib/src/audio/audio_manager.dart b/lib/src/audio/audio_manager.dart index 3034a3706..a0e2611c0 100644 --- a/lib/src/audio/audio_manager.dart +++ b/lib/src/audio/audio_manager.dart @@ -12,19 +12,355 @@ // See the License for the specific language governing permissions and // limitations under the License. +import 'dart:async'; + +import 'package:meta/meta.dart'; + +import '../logger.dart'; import '../support/native.dart'; +import '../support/native_audio.dart'; +import '../support/platform.dart'; +import 'android_audio_session_adapter.dart'; import 'audio_processing_state.dart'; +import 'audio_session.dart'; + +/// Snapshot of the WebRTC audio engine's playout/recording state. +/// +/// Surfaced by [AudioManager] from real audio-engine lifecycle events on the +/// native side (iOS and macOS). This is the source of truth for audio activity, +/// replacing the legacy track-counting state. +class AudioEngineState { + /// Whether the engine has playout (output / remote audio) enabled. + final bool isPlayoutEnabled; + + /// Whether the engine has recording (input / local mic) enabled. + final bool isRecordingEnabled; + + const AudioEngineState({ + required this.isPlayoutEnabled, + required this.isRecordingEnabled, + }); + + /// Whether the engine is neither playing out nor recording. + bool get isIdle => !isPlayoutEnabled && !isRecordingEnabled; + + @override + bool operator ==(Object other) => + other is AudioEngineState && + other.isPlayoutEnabled == isPlayoutEnabled && + other.isRecordingEnabled == isRecordingEnabled; + + @override + int get hashCode => Object.hash(isPlayoutEnabled, isRecordingEnabled); + + @override + String toString() => 'AudioEngineState(isPlayoutEnabled: $isPlayoutEnabled, isRecordingEnabled: $isRecordingEnabled)'; +} /// Controls LiveKit's process-wide platform audio behavior. /// -/// The platform audio engine and its audio processing module are global to the -/// app process, so engine-scoped audio state lives here rather than on a `Room` -/// or an individual track. +/// Platform audio sessions and the audio processing module are global to the +/// app process, so session options and engine-scoped audio state live here +/// rather than on a `Room` or an individual track. class AudioManager { AudioManager._(); static final AudioManager instance = AudioManager._(); + AudioSessionOptions _defaultOptions = const AudioSessionOptions.communication(); + AudioSessionOptions _options = const AudioSessionOptions.communication(); + AudioSessionManagementMode _managementMode = AudioSessionManagementMode.automatic; + bool _hasExplicitRuntimeOptions = false; + bool _preferSpeakerOutput = true; + bool _forceSpeakerOutput = false; + bool _isPlayoutEnabled = false; + bool _isRecordingEnabled = false; + final StreamController _audioEngineStateController = StreamController.broadcast(); + + AudioSessionOptions get defaultOptions => _defaultOptions; + AudioSessionOptions get options => _options; + AudioSessionManagementMode get managementMode => _managementMode; + + /// Whether the speaker is the preferred audio output. + bool get isSpeakerOutputPreferred => _preferSpeakerOutput; + + /// Whether speaker output is forced even when a headset/Bluetooth device is + /// connected. + bool get isSpeakerOutputForced => _forceSpeakerOutput && _preferSpeakerOutput; + + /// Whether the platform supports switching the speaker output (iOS/Android). + bool get canSwitchSpeakerphone => lkPlatformIsMobile(); + + /// The current audio engine state, derived from native engine lifecycle + /// events (iOS/macOS). On platforms without engine events this stays idle. + AudioEngineState get audioEngineState => + AudioEngineState(isPlayoutEnabled: _isPlayoutEnabled, isRecordingEnabled: _isRecordingEnabled); + + /// A broadcast stream of audio engine state changes (native engine lifecycle). + Stream get audioEngineStateStream => _audioEngineStateController.stream; + + // Derived from [managementMode]. Kept internal so the public surface exposes + // a single way to read the mode. + @internal + bool get isAutomaticConfigurationEnabled => _managementMode == AudioSessionManagementMode.automatic; + + @visibleForTesting + void resetForTest() { + _defaultOptions = const AudioSessionOptions.communication(); + _options = const AudioSessionOptions.communication(); + _managementMode = AudioSessionManagementMode.automatic; + _hasExplicitRuntimeOptions = false; + _preferSpeakerOutput = true; + _forceSpeakerOutput = false; + _isPlayoutEnabled = false; + _isRecordingEnabled = false; + } + + @internal + void configureDefaults({ + required bool bypassVoiceProcessing, + }) { + _defaultOptions = + bypassVoiceProcessing ? const AudioSessionOptions.media() : const AudioSessionOptions.communication(); + if (!_hasExplicitRuntimeOptions) { + _options = _defaultOptions; + } + } + + /// Invoked from native when the WebRTC audio engine's playout/recording state + /// changes. Audio-engine lifecycle events are the single source of truth for + /// audio activity. This replaces the legacy track-counting path, which had + /// timing races and could miss session deactivation. + /// + /// On iOS the native engine delegate also owns audio-session activation + /// timing (configure + activate on enable, deactivate on disable). This Dart + /// hop is non-blocking and only keeps the observable state in sync. macOS + /// emits the same events (no `AVAudioSession` to configure) so engine state + /// stays authoritative there too. + @internal + void handleAudioEngineState({ + required bool isPlayoutEnabled, + required bool isRecordingEnabled, + }) { + final nextState = AudioEngineState( + isPlayoutEnabled: isPlayoutEnabled, + isRecordingEnabled: isRecordingEnabled, + ); + if (nextState == audioEngineState) { + return; + } + + _isPlayoutEnabled = isPlayoutEnabled; + _isRecordingEnabled = isRecordingEnabled; + _audioEngineStateController.add(nextState); + } + + /// Applies a new audio session configuration immediately. + /// + /// Use [AudioSessionOptions.communication] to enter VoIP/call mode and + /// [AudioSessionOptions.media] to leave communication mode for + /// media/live-streaming capture. This explicit apply path works in both + /// automatic and manual management modes. + Future setAudioSessionOptions(AudioSessionOptions options) async { + _hasExplicitRuntimeOptions = true; + _syncSpeakerPreferenceFromOptions(options); + _forceSpeakerOutput = false; + _options = options; + await applyCurrentAudioSessionOptions(); + } + + /// Selects whether LiveKit manages the platform audio session automatically. + /// + /// In [AudioSessionManagementMode.manual], LiveKit does not update the audio + /// session from room, connect, or track lifecycle. The app can still apply a + /// configuration explicitly with [setAudioSessionOptions] and release it with + /// [deactivateAudioSession]. + /// + /// Prefer setting this before connecting to a room. flutter_webrtc's own + /// native audio management is always disabled (LiveKit owns the session). + /// Changing the mode at runtime only affects LiveKit's own automatic + /// configuration. + Future setAudioSessionManagementMode(AudioSessionManagementMode mode) async { + _managementMode = mode; + await _syncAppleAudioSessionManagementMode(); + } + + /// Deactivates the current platform audio session. + /// + /// In manual mode this is the explicit release counterpart to + /// [setAudioSessionOptions] or [applyCurrentAudioSessionOptions]. In automatic + /// mode, LiveKit normally releases the session from room/engine lifecycle, so + /// apps rarely need to call this directly. + Future deactivateAudioSession() async { + if (lkPlatformIs(PlatformType.iOS)) { + await Native.deactivateAppleAudioSession(); + } else if (lkPlatformIs(PlatformType.android)) { + await Native.stopAndroidAudioSession(); + } + } + + /// Prefers routing audio output to/from the speaker. + /// + /// By default a connected wired/Bluetooth headset still takes priority even + /// when [preferred] is true. Set [force] to force the speaker even when a + /// headset is connected. + /// + /// LiveKit owns this routing on both platforms (Android via its own + /// audioswitch handler and iOS via its audio session), so it does not depend + /// on flutter_webrtc. + Future setSpeakerOutputPreferred(bool preferred, {bool force = false}) async { + if (!canSwitchSpeakerphone) { + logger.warning('setSpeakerOutputPreferred is only supported on iOS/Android'); + return; + } + _preferSpeakerOutput = preferred; + _forceSpeakerOutput = preferred && force; + _options = _optionsWithSpeakerPreference(_options, preferred); + + if (lkPlatformIs(PlatformType.iOS)) { + if (isAutomaticConfigurationEnabled) { + final policy = _resolvedAudioSessionPolicy(_options); + // Automatic mode: the native audio-engine delegate owns activation + // timing, so this caches the policy and applies now only if the engine + // is already running. Category is resolved natively from engine state + // unless the app supplied an explicit Apple override. + await Native.configureAudio( + policy.appleConfiguration, + automatic: true, + selectCategoryByEngineState: policy.usesDynamicAppleCategory, + forceSpeakerOutput: policy.forceSpeakerOutput, + ); + } else { + // Manual mode: this is an explicit routing request, so re-apply the + // resolved session policy immediately. Plain speaker preference is + // expressed by category/mode; force is carried separately to native. + await _configureAppleAudioSession(_options); + } + } else if (lkPlatformIs(PlatformType.android)) { + await Native.setAndroidSpeakerphoneOn(preferred, force: _forceSpeakerOutput); + } + } + + /// Re-applies the current audio session options. + /// + /// This is useful after platform interruptions or app lifecycle changes when + /// the app wants LiveKit to restore its currently selected session mode. + Future applyCurrentAudioSessionOptions() async { + if (lkPlatformIs(PlatformType.iOS)) { + await _configureAppleAudioSession(_options); + } else if (lkPlatformIs(PlatformType.android)) { + await _configureAndroidAudioSession(_options); + } + } + + @internal + Map? androidAudioConfigurationForInitialize({ + @visibleForTesting bool assumeAndroid = false, + }) { + if (!assumeAndroid && !lkPlatformIs(PlatformType.android)) { + return null; + } + + // Preserve today's implicit initialize behavior. Only send Android audio + // attributes when the bypassVoiceProcessing path needs media attributes. + if (!isAutomaticConfigurationEnabled || !Native.bypassVoiceProcessing) { + return null; + } + + return androidAudioSessionConfigurationToMap(_resolvedAudioSessionPolicy(_options).androidConfiguration); + } + + @internal + Future applyOptionsForConnect() async { + await _syncAppleAudioSessionManagementMode(); + if (isAutomaticConfigurationEnabled) { + await applyCurrentAudioSessionOptions(); + } + } + + Future _syncAppleAudioSessionManagementMode() async { + if (lkPlatformIs(PlatformType.iOS)) { + await Native.setAppleAudioSessionAutomaticManagementEnabled(isAutomaticConfigurationEnabled); + } + } + + Future _configureAppleAudioSession(AudioSessionOptions options) async { + final policy = _resolvedAudioSessionPolicy(options); + final config = policy.appleConfiguration; + logger.fine('configuring Apple audio session using $config...'); + // In automatic mode the native audio-engine delegate owns activation timing, + // so this caches the policy and applies now only if the engine is already + // running. Automatic mode can resolve the category from engine state unless + // the app gave an explicit Apple override. Manual mode applies the resolved + // config immediately and verbatim. + await Native.configureAudio( + config, + automatic: isAutomaticConfigurationEnabled, + selectCategoryByEngineState: isAutomaticConfigurationEnabled && policy.usesDynamicAppleCategory, + forceSpeakerOutput: policy.forceSpeakerOutput, + ); + } + + Future _configureAndroidAudioSession(AudioSessionOptions options) async { + final policy = _resolvedAudioSessionPolicy(options); + final config = policy.androidConfiguration; + logger.fine( + 'configuring Android audio session using ${androidAudioSessionConfigurationToMap(config)}...', + ); + await setAndroidAudioSessionConfiguration(config); + await Native.setAndroidSpeakerphoneOn(policy.preferSpeakerOutput, force: policy.forceSpeakerOutput); + } + + _ResolvedAudioSessionPolicy _resolvedAudioSessionPolicy(AudioSessionOptions options) { + final preferSpeakerOutput = _speakerPreferenceForOptions(options); + return _ResolvedAudioSessionPolicy( + options: options, + preferSpeakerOutput: preferSpeakerOutput, + forceSpeakerOutput: _forceSpeakerOutput && preferSpeakerOutput, + ); + } + + @visibleForTesting + NativeAudioConfiguration resolveAppleAudioConfigurationForTest( + AudioSessionOptions options, { + bool forceSpeakerOutput = false, + }) { + final preferSpeakerOutput = _speakerPreferenceForOptions(options); + return _ResolvedAudioSessionPolicy( + options: options, + preferSpeakerOutput: preferSpeakerOutput, + forceSpeakerOutput: forceSpeakerOutput && preferSpeakerOutput, + ).appleConfiguration; + } + + @visibleForTesting + AndroidAudioSessionConfiguration resolveAndroidAudioConfigurationForTest(AudioSessionOptions options) => + _resolvedAudioSessionPolicy(options).androidConfiguration; + + bool _speakerPreferenceForOptions(AudioSessionOptions options) => + options.apple?.preferSpeakerOutput ?? options.preferSpeakerOutput; + + void _syncSpeakerPreferenceFromOptions(AudioSessionOptions options) { + _preferSpeakerOutput = _speakerPreferenceForOptions(options); + if (!_preferSpeakerOutput) { + _forceSpeakerOutput = false; + } + } + + AudioSessionOptions _optionsWithSpeakerPreference(AudioSessionOptions options, bool preferSpeakerOutput) { + final apple = options.apple; + return options.copyWith( + preferSpeakerOutput: Value(preferSpeakerOutput), + apple: apple == null + ? const Absent() + : Value( + apple.copyWith( + preferSpeakerOutput: Value(preferSpeakerOutput), + ), + ), + ); + } + /// Diagnostic snapshot of the resolved audio processing state. /// /// The audio processing module is owned by the native peer connection factory @@ -38,3 +374,67 @@ class AudioManager { return AudioProcessingState.fromMap(response); } } + +class _ResolvedAudioSessionPolicy { + const _ResolvedAudioSessionPolicy({ + required this.options, + required this.preferSpeakerOutput, + required this.forceSpeakerOutput, + }); + + final AudioSessionOptions options; + final bool preferSpeakerOutput; + final bool forceSpeakerOutput; + + bool get usesDynamicAppleCategory => options.apple == null; + + NativeAudioConfiguration get appleConfiguration { + final apple = options.apple; + if (apple != null) { + return NativeAudioConfiguration( + appleAudioCategory: apple.category, + appleAudioCategoryOptions: apple.categoryOptions, + appleAudioMode: apple.mode, + ); + } + + if (options.isCommunication) { + return NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + appleAudioMode: preferSpeakerOutput ? AppleAudioMode.videoChat : AppleAudioMode.voiceChat, + ); + } + + // Media (non-communication) base policy. The category here is a base. In + // automatic mode the native engine delegate overrides it from the live + // engine state (playAndRecord while recording, playback for playout-only), + // so it no longer depends on stale track/engine flags resolved at connect. + return NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.mixWithOthers, + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + appleAudioMode: AppleAudioMode.default_, + ); + } + + AndroidAudioSessionConfiguration get androidConfiguration { + final android = options.android; + if (android != null) { + return android; + } + + if (options.isCommunication) { + return AndroidAudioSessionConfiguration.communication; + } + return AndroidAudioSessionConfiguration.media; + } +} diff --git a/lib/src/audio/audio_session.dart b/lib/src/audio/audio_session.dart new file mode 100644 index 000000000..79e030783 --- /dev/null +++ b/lib/src/audio/audio_session.dart @@ -0,0 +1,303 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export '../support/value_or_absent.dart'; + +import 'package:meta/meta.dart'; + +import '../support/value_or_absent.dart'; + +enum AudioSessionManagementMode { + /// LiveKit updates the platform audio session based on room/track lifecycle. + automatic, + + /// LiveKit does not update the platform audio session automatically. + /// + /// The app must call AudioManager APIs when it wants to apply a session + /// configuration. + manual, +} + +enum _AudioSessionPreset { communication, media } + +class AudioSessionOptions { + final _AudioSessionPreset _preset; + + /// Whether communication sessions should prefer speaker output. + /// + /// This is used by the communication preset and by Apple `playAndRecord` + /// policies. Media/playback policies leave routing to the platform unless an + /// exact platform override says otherwise. + final bool preferSpeakerOutput; + + /// Optional exact iOS session override. + final AppleAudioSessionConfiguration? apple; + + /// Optional exact Android session override. + final AndroidAudioSessionConfiguration? android; + + const AudioSessionOptions._({ + required _AudioSessionPreset preset, + this.preferSpeakerOutput = true, + this.apple, + this.android, + }) : _preset = preset; + + /// Two-way audio preset for calls, rooms, and microphone capture. + /// + /// LiveKit resolves this to communication-oriented platform policies. Use + /// [preferSpeakerOutput] for the default speaker preference unless [apple] or + /// [android] provides a more exact platform policy. + /// + /// On Apple platforms in automatic mode, listen-only playout uses playback + /// until recording starts. Receiver routing from [preferSpeakerOutput] only + /// applies while the effective category is `playAndRecord`. + const AudioSessionOptions.communication({ + bool preferSpeakerOutput = true, + AppleAudioSessionConfiguration? apple, + AndroidAudioSessionConfiguration? android, + }) : this._( + preset: _AudioSessionPreset.communication, + preferSpeakerOutput: preferSpeakerOutput, + apple: apple, + android: android, + ); + + /// One-way media playback preset. + /// + /// This intentionally does not expose an initial [preferSpeakerOutput] value: + /// Apple playback policy leaves routing to the platform, while Android + /// speaker routing remains a runtime preference. Use [apple] or [android] for + /// exact platform behavior, or switch at runtime with + /// `AudioManager.setSpeakerOutputPreferred`. + const AudioSessionOptions.media({ + AppleAudioSessionConfiguration? apple, + AndroidAudioSessionConfiguration? android, + }) : this._( + preset: _AudioSessionPreset.media, + preferSpeakerOutput: true, + apple: apple, + android: android, + ); + + /// Returns a copy with selected fields replaced. + /// + /// The preset chosen by [AudioSessionOptions.communication] or + /// [AudioSessionOptions.media] is intentionally retained. Create a new + /// options object with the other constructor to switch presets. + AudioSessionOptions copyWith({ + ValueOrAbsent preferSpeakerOutput = const Absent(), + ValueOrAbsent apple = const Absent(), + ValueOrAbsent android = const Absent(), + }) => + AudioSessionOptions._( + preset: _preset, + preferSpeakerOutput: preferSpeakerOutput.valueOr(this.preferSpeakerOutput), + apple: apple.valueOr(this.apple), + android: android.valueOr(this.android), + ); + + @internal + bool get isCommunication => _preset == _AudioSessionPreset.communication; + + @internal + bool get isMedia => _preset == _AudioSessionPreset.media; +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/category +enum AppleAudioCategory { + soloAmbient, + playback, + record, + playAndRecord, + multiRoute, +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/categoryoptions +enum AppleAudioCategoryOption { + mixWithOthers, // Only playAndRecord, playback, or multiRoute. + duckOthers, // Only playAndRecord, playback, or multiRoute. + interruptSpokenAudioAndMixWithOthers, + allowBluetooth, // Only playAndRecord or record. + allowBluetoothA2DP, + allowAirPlay, + defaultToSpeaker, +} + +// https://developer.apple.com/documentation/avfaudio/avaudiosession/mode +enum AppleAudioMode { + default_, + gameChat, + measurement, + moviePlayback, + spokenAudio, + videoChat, + videoRecording, + voiceChat, + voicePrompt, +} + +class AppleAudioSessionConfiguration { + /// AVAudioSession category. + final AppleAudioCategory? category; + + /// AVAudioSession category options. + final Set? categoryOptions; + + /// AVAudioSession mode. + final AppleAudioMode? mode; + + /// Whether AVAudioSession should prefer speaker output when supported. + final bool? preferSpeakerOutput; + + const AppleAudioSessionConfiguration({ + this.category, + this.categoryOptions, + this.mode, + this.preferSpeakerOutput, + }); + + AppleAudioSessionConfiguration copyWith({ + ValueOrAbsent category = const Absent(), + ValueOrAbsent?> categoryOptions = const Absent(), + ValueOrAbsent mode = const Absent(), + ValueOrAbsent preferSpeakerOutput = const Absent(), + }) => + AppleAudioSessionConfiguration( + category: category.valueOr(this.category), + categoryOptions: categoryOptions.valueOr(this.categoryOptions), + mode: mode.valueOr(this.mode), + preferSpeakerOutput: preferSpeakerOutput.valueOr(this.preferSpeakerOutput), + ); +} + +enum AndroidAudioMode { + normal, + callScreening, + inCall, + inCommunication, + ringtone, +} + +enum AndroidAudioFocusMode { + gain, + gainTransient, + gainTransientExclusive, + gainTransientMayDuck, +} + +enum AndroidAudioStreamType { + accessibility, + alarm, + dtmf, + music, + notification, + ring, + system, + voiceCall, +} + +enum AndroidAudioAttributesUsageType { + alarm, + assistanceAccessibility, + assistanceNavigationGuidance, + assistanceSonification, + assistant, + game, + media, + notification, + notificationEvent, + notificationRingtone, + unknown, + voiceCommunication, + voiceCommunicationSignalling, +} + +enum AndroidAudioAttributesContentType { + movie, + music, + sonification, + speech, + unknown, +} + +class AndroidAudioSessionConfiguration { + /// Android AudioManager mode. + final AndroidAudioMode? audioMode; + + /// Whether LiveKit should manage Android audio focus. + final bool? manageAudioFocus; + + /// Requested Android audio focus gain type. + final AndroidAudioFocusMode? focusMode; + + /// Legacy Android stream type. + final AndroidAudioStreamType? streamType; + + /// Android AudioAttributes usage. + final AndroidAudioAttributesUsageType? usageType; + + /// Android AudioAttributes content type. + final AndroidAudioAttributesContentType? contentType; + + /// Forces LiveKit audio routing even outside communication/call modes. + final bool? forceAudioRouting; + + const AndroidAudioSessionConfiguration({ + this.audioMode, + this.manageAudioFocus, + this.focusMode, + this.streamType, + this.usageType, + this.contentType, + this.forceAudioRouting, + }); + + static const communication = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.inCommunication, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.voiceCall, + usageType: AndroidAudioAttributesUsageType.voiceCommunication, + contentType: AndroidAudioAttributesContentType.speech, + ); + + static const media = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.music, + usageType: AndroidAudioAttributesUsageType.media, + contentType: AndroidAudioAttributesContentType.unknown, + ); + + AndroidAudioSessionConfiguration copyWith({ + ValueOrAbsent audioMode = const Absent(), + ValueOrAbsent manageAudioFocus = const Absent(), + ValueOrAbsent focusMode = const Absent(), + ValueOrAbsent streamType = const Absent(), + ValueOrAbsent usageType = const Absent(), + ValueOrAbsent contentType = const Absent(), + ValueOrAbsent forceAudioRouting = const Absent(), + }) => + AndroidAudioSessionConfiguration( + audioMode: audioMode.valueOr(this.audioMode), + manageAudioFocus: manageAudioFocus.valueOr(this.manageAudioFocus), + focusMode: focusMode.valueOr(this.focusMode), + streamType: streamType.valueOr(this.streamType), + usageType: usageType.valueOr(this.usageType), + contentType: contentType.valueOr(this.contentType), + forceAudioRouting: forceAudioRouting.valueOr(this.forceAudioRouting), + ); +} diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index f274a0289..97f89440a 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -19,6 +19,7 @@ import 'package:collection/collection.dart'; import 'package:http/http.dart' as http; import 'package:meta/meta.dart'; +import '../audio/audio_manager.dart'; import '../core/signal_client.dart'; import '../data_stream/errors.dart'; import '../data_stream/stream_reader.dart'; @@ -291,6 +292,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { // configure audio for native platform await NativeAudioManagement.start(); + var didConnect = false; try { await engine.connect( _regionUrl ?? url, @@ -300,6 +302,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { fastConnectOptions: fastConnectOptions, regionUrlProvider: _regionUrlProvider, ); + didConnect = true; } catch (e) { logger.warning('could not connect to $url $e'); if (_regionUrlProvider != null && @@ -322,12 +325,17 @@ class Room extends DisposableChangeNotifier with EventsEmittable { fastConnectOptions: fastConnectOptions, regionUrlProvider: _regionUrlProvider, ); + didConnect = true; } else { rethrow; } } else { rethrow; } + } finally { + if (!didConnect) { + await NativeAudioManagement.stop(); + } } } @@ -1186,10 +1194,10 @@ extension RoomHardwareManagementMethods on Room { /// [speakerOn] set speakerphone on or off, by default wired/bluetooth headsets will still /// be prioritized even if set to true. /// [forceSpeakerOutput] if true, will force speaker output even if headphones - /// or bluetooth is connected, only supported on iOS for now + /// or bluetooth is connected. Future setSpeakerOn(bool speakerOn, {bool forceSpeakerOutput = false}) async { if (lkPlatformIsMobile()) { - await Hardware.instance.setSpeakerphoneOn(speakerOn, forceSpeakerOutput: forceSpeakerOutput); + await AudioManager.instance.setSpeakerOutputPreferred(speakerOn, force: forceSpeakerOutput); engine.roomOptions = engine.roomOptions.copyWith( defaultAudioOutputOptions: roomOptions.defaultAudioOutputOptions.copyWith( speakerOn: speakerOn, @@ -1203,7 +1211,7 @@ extension RoomHardwareManagementMethods on Room { Future applyAudioSpeakerSettings() async { if (roomOptions.defaultAudioOutputOptions.speakerOn != null) { if (lkPlatformIsMobile()) { - await Hardware.instance.setSpeakerphoneOn(roomOptions.defaultAudioOutputOptions.speakerOn!); + await AudioManager.instance.setSpeakerOutputPreferred(roomOptions.defaultAudioOutputOptions.speakerOn!); } } } diff --git a/lib/src/hardware/hardware.dart b/lib/src/hardware/hardware.dart index 065d4f361..9182015e6 100644 --- a/lib/src/hardware/hardware.dart +++ b/lib/src/hardware/hardware.dart @@ -17,11 +17,10 @@ import 'dart:async'; import 'package:collection/collection.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import '../audio/audio_manager.dart'; +import '../audio/audio_session.dart'; import '../logger.dart'; -import '../support/native.dart'; -import '../support/native_audio.dart'; import '../support/platform.dart'; -import '../track/audio_management.dart'; class MediaDevice { const MediaDevice(this.deviceId, this.label, this.kind, this.groupId); @@ -69,27 +68,32 @@ class Hardware { MediaDevice? selectedVideoInput; - bool? get speakerOn => _preferSpeakerOutput; + @Deprecated('Use AudioManager.instance.isSpeakerOutputPreferred instead') + bool? get speakerOn => AudioManager.instance.isSpeakerOutputPreferred; - bool _preferSpeakerOutput = true; - - bool get preferSpeakerOutput => _preferSpeakerOutput; - - bool _forceSpeakerOutput = false; + @Deprecated('Use AudioManager.instance.isSpeakerOutputPreferred instead') + bool get preferSpeakerOutput => AudioManager.instance.isSpeakerOutputPreferred; /// if true, will force speaker output even if headphones or bluetooth is connected - /// only supported on iOS for now - bool get forceSpeakerOutput => _forceSpeakerOutput && _preferSpeakerOutput; - - // This flag is used to determine if automatic native configuration - // of audio is enabled. If set to false Natvive.configureAudio - // will not be called, and the user is responsible for configuring - // the native audio configuration manually. - bool _isAutomaticConfigurationEnabled = true; - bool get isAutomaticConfigurationEnabled => _isAutomaticConfigurationEnabled; - + @Deprecated('Use AudioManager.instance.isSpeakerOutputForced instead') + bool get forceSpeakerOutput => AudioManager.instance.isSpeakerOutputForced; + + // Whether automatic native audio configuration is enabled. If disabled, + // Native.configureAudio is not called and the app is responsible for + // configuring the native audio session manually. + // + // Backed by [AudioManager] so there is a single source of truth for the + // management mode. See [AudioManager.setAudioSessionManagementMode]. + @Deprecated('Use AudioManager.instance.managementMode instead') + bool get isAutomaticConfigurationEnabled => AudioManager.instance.isAutomaticConfigurationEnabled; + + @Deprecated('Use AudioManager.instance.setAudioSessionManagementMode instead') void setAutomaticConfigurationEnabled({required bool enable}) { - _isAutomaticConfigurationEnabled = enable; + unawaited( + AudioManager.instance.setAudioSessionManagementMode( + enable ? AudioSessionManagementMode.automatic : AudioSessionManagementMode.manual, + ), + ); } Future> enumerateDevices({String? type}) async { @@ -131,48 +135,19 @@ class Hardware { await rtc.Helper.selectAudioInput(device.deviceId); } - @Deprecated('use setSpeakerphoneOn') - Future setPreferSpeakerOutput(bool enable) => setSpeakerphoneOn(enable); + @Deprecated('Use AudioManager.instance.setSpeakerOutputPreferred instead') + Future setPreferSpeakerOutput(bool enable) => AudioManager.instance.setSpeakerOutputPreferred(enable); - bool get canSwitchSpeakerphone => lkPlatformIsMobile(); + @Deprecated('Use AudioManager.instance.canSwitchSpeakerphone instead') + bool get canSwitchSpeakerphone => AudioManager.instance.canSwitchSpeakerphone; /// [enable] set speakerphone on or off, by default wired/bluetooth headsets will still /// be prioritized even if set to true. /// [forceSpeakerOutput] if true, will force speaker output even if headphones - /// or bluetooth is connected, only supported on iOS for now - Future setSpeakerphoneOn(bool enable, {bool forceSpeakerOutput = false}) async { - if (canSwitchSpeakerphone) { - _preferSpeakerOutput = enable; - _forceSpeakerOutput = forceSpeakerOutput; - if (lkPlatformIs(PlatformType.iOS)) { - NativeAudioConfiguration? config; - if (lkPlatformIs(PlatformType.iOS)) { - // Only iOS for now... - config = await onConfigureNativeAudio.call(audioTrackState); - if (_preferSpeakerOutput && _forceSpeakerOutput) { - config = config.copyWith( - appleAudioCategoryOptions: { - ...?config.appleAudioCategoryOptions, - AppleAudioCategoryOption.defaultToSpeaker, - }, - ); - } - logger.fine('configuring for ${audioTrackState} using ${config}...'); - try { - if (_isAutomaticConfigurationEnabled) { - await Native.configureAudio(config); - } - } catch (error) { - logger.warning('failed to configure ${error}'); - } - } - } else { - await rtc.Helper.setSpeakerphoneOn(enable); - } - } else { - logger.warning('setSpeakerphoneOn only support on iOS/Android'); - } - } + /// or bluetooth is connected. + @Deprecated('Use AudioManager.instance.setSpeakerOutputPreferred instead') + Future setSpeakerphoneOn(bool enable, {bool forceSpeakerOutput = false}) => + AudioManager.instance.setSpeakerOutputPreferred(enable, force: forceSpeakerOutput); Future openCamera({MediaDevice? device, bool? facingMode}) async { final constraints = { diff --git a/lib/src/livekit.dart b/lib/src/livekit.dart index 6cdc2cba8..802c66475 100644 --- a/lib/src/livekit.dart +++ b/lib/src/livekit.dart @@ -14,6 +14,7 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'audio/audio_manager.dart'; import 'support/native.dart'; import 'support/platform.dart' show lkPlatformIsMobile; @@ -22,16 +23,32 @@ import 'support/platform.dart' show lkPlatformIsMobile; class LiveKitClient { static const version = '2.8.0'; - /// Initialize the WebRTC plugin. If this is not manually called, will be - /// initialized with default settings. - /// This method must be called before calling any LiveKit SDK API. - static Future initialize({bool bypassVoiceProcessing = false}) async { + /// Initialize the WebRTC plugin. + /// + /// Optional: call once at startup to enable [bypassVoiceProcessing] before + /// connecting. Otherwise WebRTC initializes lazily with defaults. + /// + /// LiveKit owns the platform audio session, and flutter_webrtc's own native + /// audio management is disabled automatically when the LiveKit plugin loads + /// (done natively at registration), so that does not depend on this call. + /// + /// Configure audio-session behavior through [AudioManager] before connecting, + /// e.g. `await AudioManager.instance.setAudioSessionManagementMode(...)` and + /// `await AudioManager.instance.setAudioSessionOptions(...)`. + static Future initialize({ + bool bypassVoiceProcessing = false, + }) async { if (lkPlatformIsMobile()) { + Native.bypassVoiceProcessing = bypassVoiceProcessing; + AudioManager.instance.configureDefaults( + bypassVoiceProcessing: bypassVoiceProcessing, + ); + final androidAudioConfiguration = AudioManager.instance.androidAudioConfigurationForInitialize(); + await rtc.WebRTC.initialize(options: { if (bypassVoiceProcessing) 'bypassVoiceProcessing': bypassVoiceProcessing, + if (androidAudioConfiguration != null) 'androidAudioConfiguration': androidAudioConfiguration, }); - - Native.bypassVoiceProcessing = bypassVoiceProcessing; } } } diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index 9cdfe928a..b71dd18ab 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -18,6 +18,7 @@ import 'package:flutter/services.dart' show MethodChannel, MethodCall; import 'package:meta/meta.dart'; +import '../audio/audio_manager.dart'; import '../logger.dart'; import '../managers/broadcast_manager.dart'; import 'native_audio.dart'; @@ -36,12 +37,29 @@ class Native { @internal static bool bypassVoiceProcessing = false; + /// Configures (and caches) the Apple audio session. + /// + /// When [automatic] is true, the native audio-engine delegate owns activation + /// timing: the configuration is cached and (re)applied on engine lifecycle + /// events, and only applied immediately here if the engine is already + /// running. When false (manual mode / explicit apply) it is applied + /// immediately. @internal - static Future configureAudio(NativeAudioConfiguration configuration) async { + static Future configureAudio( + NativeAudioConfiguration configuration, { + bool automatic = false, + bool selectCategoryByEngineState = false, + bool forceSpeakerOutput = false, + }) async { try { final result = await channel.invokeMethod( 'configureNativeAudio', - configuration.toMap(), + { + ...configuration.toMap(), + 'automatic': automatic, + 'selectCategoryByEngineState': selectCategoryByEngineState, + 'forceSpeakerOutput': forceSpeakerOutput, + }, ); return result == true; } catch (error) { @@ -93,6 +111,63 @@ class Native { return null; } + /// Configure and activate LiveKit's Android audio session (mode/focus/routing). + @internal + static Future configureAndroidAudioSession(Map configuration) async { + try { + await channel.invokeMethod('configureAndroidAudioSession', configuration); + } catch (error) { + logger.warning('configureAndroidAudioSession did throw $error'); + } + } + + /// Deactivate LiveKit's Android audio session (release focus, restore mode). + @internal + static Future stopAndroidAudioSession() async { + try { + await channel.invokeMethod('stopAndroidAudioSession'); + } catch (error) { + logger.warning('stopAndroidAudioSession did throw $error'); + } + } + + /// Deactivate LiveKit's Apple audio session. + @internal + static Future deactivateAppleAudioSession() async { + try { + await channel.invokeMethod('deactivateAppleAudioSession', {}); + } catch (error) { + logger.warning('deactivateAppleAudioSession did throw $error'); + } + } + + /// Route Android audio output to/from the speakerphone. + @internal + static Future setAndroidSpeakerphoneOn(bool enable, {bool force = false}) async { + try { + await channel.invokeMethod( + 'setAndroidSpeakerphoneOn', + {'enable': enable, 'force': force}, + ); + } catch (error) { + logger.warning('setAndroidSpeakerphoneOn did throw $error'); + } + } + + /// Enable or disable LiveKit's automatic iOS audio-session management from + /// native WebRTC audio-engine lifecycle callbacks. + @internal + static Future setAppleAudioSessionAutomaticManagementEnabled(bool enabled) async { + try { + await channel.invokeMethod( + 'setAppleAudioSessionAutomaticManagementEnabled', + {'enabled': enabled}, + ); + } catch (error) { + logger.warning('setAppleAudioSessionAutomaticManagementEnabled did throw $error'); + } + } + @internal static Future startVisualizer( String trackId, { @@ -196,6 +271,15 @@ class Native { } _broadcastStateChanged(call.arguments as bool); return null; + case 'onAudioEngineState': + final args = call.arguments; + if (args is Map) { + AudioManager.instance.handleAudioEngineState( + isPlayoutEnabled: args['isPlayoutEnabled'] == true, + isRecordingEnabled: args['isRecordingEnabled'] == true, + ); + } + return null; default: logger.warning('Method ${call.method} is not implemented.'); return null; diff --git a/lib/src/support/native_audio.dart b/lib/src/support/native_audio.dart index 70bc59dce..474f543ed 100644 --- a/lib/src/support/native_audio.dart +++ b/lib/src/support/native_audio.dart @@ -12,38 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// https://developer.apple.com/documentation/avfaudio/avaudiosession/category -enum AppleAudioCategory { - soloAmbient, - playback, - record, - playAndRecord, - multiRoute, -} - -// https://developer.apple.com/documentation/avfaudio/avaudiosession/categoryoptions -enum AppleAudioCategoryOption { - mixWithOthers, // Only playAndRecord, playback, or multiRoute. - duckOthers, // Only playAndRecord, playback, or multiRoute. - interruptSpokenAudioAndMixWithOthers, - allowBluetooth, // Only playAndRecord or record. - allowBluetoothA2DP, - allowAirPlay, - defaultToSpeaker, -} - -// https://developer.apple.com/documentation/avfaudio/avaudiosession/mode -enum AppleAudioMode { - default_, - gameChat, - measurement, - moviePlayback, - spokenAudio, - videoChat, - videoRecording, - voiceChat, - voicePrompt, -} +import '../audio/audio_session.dart' show AppleAudioCategory, AppleAudioCategoryOption, AppleAudioMode; +import 'value_or_absent.dart'; extension AppleAudioCategoryExt on AppleAudioCategory { String toStringValue() => { @@ -85,47 +55,13 @@ class NativeAudioConfiguration { final AppleAudioCategory? appleAudioCategory; final Set? appleAudioCategoryOptions; final AppleAudioMode? appleAudioMode; - final bool? preferSpeakerOutput; - - static final soloAmbient = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.soloAmbient, - appleAudioCategoryOptions: {}, - appleAudioMode: AppleAudioMode.default_, - ); - - static final playback = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playback, - appleAudioCategoryOptions: {AppleAudioCategoryOption.mixWithOthers}, - appleAudioMode: AppleAudioMode.spokenAudio, - ); - - static final playAndRecordSpeaker = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playAndRecord, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.allowBluetooth, - AppleAudioCategoryOption.allowBluetoothA2DP, - AppleAudioCategoryOption.allowAirPlay, - }, - appleAudioMode: AppleAudioMode.videoChat, - ); - - static final playAndRecordReceiver = NativeAudioConfiguration( - appleAudioCategory: AppleAudioCategory.playAndRecord, - appleAudioCategoryOptions: { - AppleAudioCategoryOption.allowBluetooth, - AppleAudioCategoryOption.allowBluetoothA2DP, - AppleAudioCategoryOption.allowAirPlay, - }, - appleAudioMode: AppleAudioMode.voiceChat, - ); NativeAudioConfiguration( { // for iOS / Mac this.appleAudioCategory, this.appleAudioCategoryOptions, - this.appleAudioMode, - this.preferSpeakerOutput + this.appleAudioMode // Android options // ... }); @@ -135,19 +71,16 @@ class NativeAudioConfiguration { if (appleAudioCategoryOptions != null) 'appleAudioCategoryOptions': appleAudioCategoryOptions!.map((e) => e.toStringValue()).toList(), if (appleAudioMode != null) 'appleAudioMode': appleAudioMode!.toStringValue(), - if (preferSpeakerOutput != null) 'preferSpeakerOutput': preferSpeakerOutput, }; NativeAudioConfiguration copyWith({ - AppleAudioCategory? appleAudioCategory, - Set? appleAudioCategoryOptions, - AppleAudioMode? appleAudioMode, - bool? preferSpeakerOutput, + ValueOrAbsent appleAudioCategory = const Absent(), + ValueOrAbsent?> appleAudioCategoryOptions = const Absent(), + ValueOrAbsent appleAudioMode = const Absent(), }) => NativeAudioConfiguration( - appleAudioCategory: appleAudioCategory ?? this.appleAudioCategory, - appleAudioCategoryOptions: appleAudioCategoryOptions ?? this.appleAudioCategoryOptions, - appleAudioMode: appleAudioMode ?? this.appleAudioMode, - preferSpeakerOutput: preferSpeakerOutput ?? this.preferSpeakerOutput, + appleAudioCategory: appleAudioCategory.valueOr(this.appleAudioCategory), + appleAudioCategoryOptions: appleAudioCategoryOptions.valueOr(this.appleAudioCategoryOptions), + appleAudioMode: appleAudioMode.valueOr(this.appleAudioMode), ); } diff --git a/lib/src/support/value_or_absent.dart b/lib/src/support/value_or_absent.dart new file mode 100644 index 000000000..1b5b03283 --- /dev/null +++ b/lib/src/support/value_or_absent.dart @@ -0,0 +1,65 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Distinguishes an omitted copy value from an explicit replacement value. +/// +/// This is useful for `copyWith` methods where a nullable field must be able to +/// keep its current value, change to a non-null value, or change to `null`. +/// +/// ```dart +/// class Example { +/// const Example({this.name}); +/// +/// final String? name; +/// +/// Example copyWith({ +/// ValueOrAbsent name = const Absent(), +/// }) => +/// Example(name: name.valueOr(this.name)); +/// } +/// +/// example.copyWith(); // keep existing name +/// example.copyWith(name: Value('room')); // set name +/// example.copyWith(name: Value(null)); // clear name +/// ``` +sealed class ValueOrAbsent { + const ValueOrAbsent(); + + /// Creates an explicit replacement value. + const factory ValueOrAbsent.value(T value) = Value; + + /// Creates an omitted value that preserves the current field. + const factory ValueOrAbsent.absent() = Absent; + + /// Returns the explicit value, or [other] when this value is absent. + T valueOr(T other); +} + +/// An explicit replacement value, including `null`. +final class Value extends ValueOrAbsent { + const Value(this.value); + + final T value; + + @override + T valueOr(T other) => value; +} + +/// An omitted value that keeps the current field. +final class Absent extends ValueOrAbsent { + const Absent(); + + @override + T valueOr(T other) => other; +} diff --git a/lib/src/track/audio_management.dart b/lib/src/track/audio_management.dart index f287ca03b..3c462d1a9 100644 --- a/lib/src/track/audio_management.dart +++ b/lib/src/track/audio_management.dart @@ -12,170 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; -import 'package:synchronized/synchronized.dart' as sync; - -import '../hardware/hardware.dart'; -import '../logger.dart'; +import '../audio/audio_manager.dart'; import '../support/native.dart'; -import '../support/native_audio.dart'; import '../support/platform.dart'; -import 'local/local.dart'; -import 'remote/remote.dart'; - -enum AudioTrackState { - none, - remoteOnly, - localOnly, - localAndRemote, -} - -typedef ConfigureNativeAudioFunc = Future Function(AudioTrackState state); - -// it's possible to set custom function here to customize audio session configuration -ConfigureNativeAudioFunc onConfigureNativeAudio = defaultNativeAudioConfigurationFunc; - -final _trackCounterLock = sync.Lock(); -AudioTrackState _audioTrackState = AudioTrackState.none; - -AudioTrackState get audioTrackState => _audioTrackState; - -int _localTrackCount = 0; -int _remoteTrackCount = 0; - -mixin LocalAudioManagementMixin on LocalTrack, AudioTrack { - @override - Future onPublish() async { - final didUpdate = await super.onPublish(); - if (didUpdate) { - // update counter - await _trackCounterLock.synchronized(() async { - _localTrackCount++; - await _onAudioTrackCountDidChange(); - }); - } - return didUpdate; - } - - @override - Future onUnpublish() async { - final didUpdate = await super.onUnpublish(); - if (didUpdate) { - // update counter - await _trackCounterLock.synchronized(() async { - _localTrackCount--; - await _onAudioTrackCountDidChange(); - }); - } - return didUpdate; - } -} -mixin RemoteAudioManagementMixin on RemoteTrack, AudioTrack { - /// Start playing audio track. On web platform, create an audio element and - /// start playback - @override - Future start() async { - final didStart = await super.start(); - if (didStart) { - await _trackCounterLock.synchronized(() async { - _remoteTrackCount++; - await _onAudioTrackCountDidChange(); - }); - } - return didStart; - } - - @override - Future stop() async { - final didStop = await super.stop(); - if (didStop) { - await _trackCounterLock.synchronized(() async { - _remoteTrackCount--; - await _onAudioTrackCountDidChange(); - }); - } - return didStop; - } -} - -Future _onAudioTrackCountDidChange() async { - logger.fine('onAudioTrackCountDidChange: ' - 'local: $_localTrackCount, remote: $_remoteTrackCount'); - - final newState = _computeAudioTrackState(); - - if (_audioTrackState != newState) { - _audioTrackState = newState; - logger.fine('didUpdateSate: $_audioTrackState'); - - NativeAudioConfiguration? config; - if (lkPlatformIs(PlatformType.iOS)) { - // Only iOS for now... - config = await onConfigureNativeAudio.call(_audioTrackState); - - if (Hardware.instance.forceSpeakerOutput) { - config = config.copyWith( - appleAudioCategoryOptions: { - ...?config.appleAudioCategoryOptions, - AppleAudioCategoryOption.defaultToSpeaker, - }, - ); - } - } - - if (config != null) { - logger.fine('configuring for ${_audioTrackState} using ${config}...'); - try { - if (Hardware.instance.isAutomaticConfigurationEnabled) { - logger.fine('configuring native audio...'); - await Native.configureAudio(config); - } - } catch (error) { - logger.warning('failed to configure ${error}'); - } - } - } -} - -AudioTrackState _computeAudioTrackState() { - if (_localTrackCount > 0 && _remoteTrackCount == 0) { - return AudioTrackState.localOnly; - } else if (_localTrackCount == 0 && _remoteTrackCount > 0) { - return AudioTrackState.remoteOnly; - } else if (_localTrackCount > 0 && _remoteTrackCount > 0) { - return AudioTrackState.localAndRemote; - } - // Default - return AudioTrackState.none; -} - -Future defaultNativeAudioConfigurationFunc(AudioTrackState state) async { - if (state == AudioTrackState.none) { - return NativeAudioConfiguration.soloAmbient; - } else if (state == AudioTrackState.remoteOnly && Hardware.instance.preferSpeakerOutput) { - return NativeAudioConfiguration.playback; - } - - return Hardware.instance.preferSpeakerOutput - ? NativeAudioConfiguration.playAndRecordSpeaker - : NativeAudioConfiguration.playAndRecordReceiver; -} class NativeAudioManagement { static Future start() async { - // Audio configuration for Android. - if (lkPlatformIs(PlatformType.android)) { - if (Native.bypassVoiceProcessing) { - await rtc.Helper.setAndroidAudioConfiguration(rtc.AndroidAudioConfiguration.media); - } else { - await rtc.Helper.setAndroidAudioConfiguration(rtc.AndroidAudioConfiguration.communication); - } - } + await AudioManager.instance.applyOptionsForConnect(); } static Future stop() async { - if (lkPlatformIs(PlatformType.android)) { - await rtc.Helper.clearAndroidCommunicationDevice(); + if (lkPlatformIs(PlatformType.android) && AudioManager.instance.isAutomaticConfigurationEnabled) { + await Native.stopAndroidAudioSession(); } } } diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index 48468b8cc..1edc688c8 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -26,11 +26,10 @@ import '../../stats/audio_source_stats.dart'; import '../../stats/stats.dart'; import '../../support/native.dart'; import '../../types/other.dart'; -import '../audio_management.dart'; import '../options.dart' as track_options; import 'local.dart'; -class LocalAudioTrack extends LocalTrack with AudioTrack, LocalAudioManagementMixin { +class LocalAudioTrack extends LocalTrack with AudioTrack { // Options used for this track @override covariant track_options.AudioCaptureOptions currentOptions; diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index 5038df572..c7eb4bf4e 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -21,12 +21,11 @@ import '../../logger.dart'; import '../../stats/audio_source_stats.dart'; import '../../stats/stats.dart'; import '../../types/other.dart'; -import '../audio_management.dart'; import '../local/local.dart'; import '../web/_audio_api.dart' if (dart.library.js_interop) '../web/_audio_html.dart' as audio; import 'remote.dart'; -class RemoteAudioTrack extends RemoteTrack with AudioTrack, RemoteAudioManagementMixin { +class RemoteAudioTrack extends RemoteTrack with AudioTrack { String? _deviceId; RemoteAudioTrack(TrackSource source, rtc.MediaStream stream, rtc.MediaStreamTrack track, {rtc.RTCRtpReceiver? receiver}) diff --git a/pubspec.lock b/pubspec.lock index 3aefe0abc..c78edefdc 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -284,10 +284,10 @@ packages: dependency: "direct main" description: name: flutter_webrtc - sha256: d8c89028d29e5693742190285b2e3c8a117531b0960ae0693d84273a53968d28 + sha256: c4e8db6ed337b8c30d76cd7cd8c91693f5495e1aeb556cbcb73f1e5d5bdcf020 url: "https://pub.dev" source: hosted - version: "1.5.0" + version: "1.5.1" frontend_server_client: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 2c768472e..0665dc206 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -46,7 +46,7 @@ dependencies: json_annotation: ^4.9.0 # Fix version to avoid version conflicts between WebRTC-SDK pods, which both this package and flutter_webrtc depend on. - flutter_webrtc: 1.5.0 + flutter_webrtc: 1.5.1 dart_webrtc: ^1.8.0 dev_dependencies: diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index fa2367bbe..f06f3ece3 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import AVFoundation import flutter_webrtc import WebRTC @@ -52,6 +53,11 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { var binaryMessenger: FlutterBinaryMessenger? + // Retained strong reference to the audio-engine delegate. flutter_webrtc and + // the audio device module both hold it weakly, so LiveKit must keep it alive. + var channel: FlutterMethodChannel? + var audioEngineObserver: LKAudioEngineObserver? + #if os(iOS) var cancellable = Set() #endif @@ -68,6 +74,21 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { instance.binaryMessenger = messenger registrar.addMethodCallDelegate(instance, channel: channel) + // LiveKit owns the platform audio session, so disable flutter_webrtc's + // own native audio management. Set at registration, before any audio op. + FlutterWebRTCPlugin.setAudioSessionManagementEnabled(false) + + // Own the audio device module's engine-lifecycle delegate so LiveKit + // drives the audio session from real engine events (configure + activate + // on enable, deactivate on disable) instead of track counting. The + // engine emits these events on both iOS and macOS. macOS has no + // AVAudioSession to configure, so there it only surfaces engine state. + // Set before the peer connection factory is created. + instance.channel = channel + let audioEngineObserver = LKAudioEngineObserver(channel: channel) + instance.audioEngineObserver = audioEngineObserver + FlutterWebRTCPlugin.setAudioDeviceModuleObserver(audioEngineObserver) + #if os(iOS) BroadcastManager.shared.isBroadcastingPublisher .sink { isBroadcasting in @@ -310,13 +331,11 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let category = categoryMap[string] { configuration.category = category.rawValue - print("[LiveKit] Configuring category: ", configuration.category) } // CategoryOptions if let strings = args["appleAudioCategoryOptions"] as? [String] { configuration.categoryOptions = categoryOptions(fromFlutter: strings) - print("[LiveKit] Configuring categoryOptions: ", strings) } // Mode @@ -324,50 +343,58 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let mode = modeMap[string] { configuration.mode = mode.rawValue - print("[LiveKit] Configuring mode: ", configuration.mode) } - // get `RTCAudioSession` and lock - let rtcSession = RTCAudioSession.sharedInstance() - rtcSession.lockForConfiguration() - - var isLocked = true - let unlock = { - guard isLocked else { - print("[LiveKit] not locked, ignoring unlock") - return - } - rtcSession.unlockForConfiguration() - isLocked = false + // Cache the policy so the audio-engine delegate can (re)apply it on + // engine lifecycle events. In automatic mode the delegate owns + // activation timing (configure + activate on engine enable), so here we + // only apply immediately if the engine is already running. Manual mode + // (or no `automatic` flag) applies immediately. + let automatic = args["automatic"] as? Bool ?? false + let selectCategoryByEngineState = args["selectCategoryByEngineState"] as? Bool ?? false + let forceSpeakerOutput = args["forceSpeakerOutput"] as? Bool ?? false + audioEngineObserver?.updatePolicy(configuration, + automaticManagementEnabled: automatic, + selectCategoryByEngineState: selectCategoryByEngineState, + forceSpeakerOutput: forceSpeakerOutput) + + let shouldApplyNow = !automatic || (audioEngineObserver?.isSessionActive ?? false) + guard shouldApplyNow else { + result(true) + return } - // always `unlock()` when exiting scope, calling multiple times has no side-effect - defer { - unlock() + // Apply through the observer so the category is resolved from the live + // engine state (when category selection is enabled), matching what the + // engine-lifecycle callbacks would apply. + if let error = audioEngineObserver?.applyCachedConfiguration() { + print("[LiveKit] Configure audio error: ", error) + result(FlutterError(code: "configure", message: error.localizedDescription, details: nil)) + } else { + result(true) } + #endif + } - do { - try rtcSession.setConfiguration(configuration, active: true) - // unlock here before configuring `AVAudioSession` - // unlock() - print("[LiveKit] RTCAudioSession Configure success") - - // also configure longFormAudio - // let avSession = AVAudioSession.sharedInstance() - // try avSession.setCategory(AVAudioSession.Category(rawValue: configuration.category), - // mode: AVAudioSession.Mode(rawValue: configuration.mode), - // policy: .default, - // options: configuration.categoryOptions) - // print("[LiveKit] AVAudioSession Configure success") - - // preferSpeakerOutput - if let preferSpeakerOutput = args["preferSpeakerOutput"] as? Bool { - try rtcSession.overrideOutputAudioPort(preferSpeakerOutput ? .speaker : .none) - } + public func handleSetAppleAudioSessionAutomaticManagementEnabled(args: [String: Any?], result: @escaping FlutterResult) { + #if os(macOS) + result(FlutterMethodNotImplemented) + #else + let enabled = (args["enabled"] as? Bool) ?? true + audioEngineObserver?.setAutomaticManagementEnabled(enabled) + result(true) + #endif + } + + public func handleDeactivateAppleAudioSession(result: @escaping FlutterResult) { + #if os(macOS) + result(FlutterMethodNotImplemented) + #else + if let error = LiveKitPlugin.deactivateAudioSession() { + print("[LiveKit] Deactivate audio session error: ", error) + result(FlutterError(code: "deactivateAudioSession", message: error.localizedDescription, details: nil)) + } else { result(true) - } catch { - print("[LiveKit] Configure audio error: ", error) - result(FlutterError(code: "configure", message: error.localizedDescription, details: nil)) } #endif } @@ -513,6 +540,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { switch call.method { case "configureNativeAudio": handleConfigureNativeAudio(args: args, result: result) + case "setAppleAudioSessionAutomaticManagementEnabled": + handleSetAppleAudioSessionAutomaticManagementEnabled(args: args, result: result) + case "deactivateAppleAudioSession": + handleDeactivateAppleAudioSession(result: result) case "startVisualizer": handleStartAudioVisualizer(args: args, result: result) case "stopVisualizer": @@ -541,3 +572,262 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { } } } + +#if !os(macOS) +@available(iOS 13.0, *) +extension LiveKitPlugin { + /// SDK-side audio engine error code (mirrors client-sdk-swift): returned + /// from a delegate callback to make WebRTC abort / roll back the engine + /// operation when the audio session cannot be configured. + static let kAudioEngineErrorFailedToConfigureAudioSession = -4100 + + /// Applies an `RTCAudioSessionConfiguration` to the shared `RTCAudioSession`. + /// Returns `nil` on success or the thrown error. Safe to call on any thread. + static func applyAudioSessionConfiguration(_ configuration: RTCAudioSessionConfiguration, + forceSpeakerOutput: Bool, + active: Bool) -> Error? { + let rtcSession = RTCAudioSession.sharedInstance() + rtcSession.lockForConfiguration() + defer { rtcSession.unlockForConfiguration() } + do { + try rtcSession.setConfiguration(configuration, active: active) + // overrideOutputAudioPort hard-routes to the speaker even over a + // connected headset. Plain speaker preference is expressed by the + // selected audio mode/category options, so clear any stale hard + // override unless the app explicitly forced speaker output. + // Only valid for the playAndRecord category. + if active, configuration.category == AVAudioSession.Category.playAndRecord.rawValue { + try rtcSession.overrideOutputAudioPort(forceSpeakerOutput ? .speaker : .none) + } + return nil + } catch { + return error + } + } + + /// Deactivates the shared `RTCAudioSession`. Returns `nil` on success. + static func deactivateAudioSession() -> Error? { + let rtcSession = RTCAudioSession.sharedInstance() + rtcSession.lockForConfiguration() + defer { rtcSession.unlockForConfiguration() } + do { + try rtcSession.setActive(false) + return nil + } catch { + return error + } + } +} +#endif + +/// Receives the WebRTC audio device module's engine-lifecycle callbacks and, +/// on iOS, drives the audio session: configure + activate when the engine +/// enables, deactivate when it disables. Replaces the old track-counting +/// trigger. On macOS there is no `AVAudioSession`, so it only surfaces engine +/// state to Dart (keeping engine state the single source of truth there too). +/// +/// The engine-lifecycle methods are invoked synchronously on WebRTC's worker +/// thread. The engine blocks on the return value (`0` = proceed, non-zero = +/// abort / roll back), so the session work here is synchronous and never calls +/// back into the audio device module. The Dart notification is dispatched +/// asynchronously and is purely informational. It never blocks the engine. +@available(iOS 13.0, macOS 10.15, *) +class LKAudioEngineObserver: NSObject, RTCAudioDeviceModuleDelegate { + private let lock = NSLock() + private weak var channel: FlutterMethodChannel? + + #if !os(macOS) + private var cachedConfiguration: RTCAudioSessionConfiguration? + // When true, the category is chosen from the live engine state at apply time + // (playAndRecord while recording, playback for playout-only) rather than + // taken from the pushed config. This is what keeps the category correct as + // recording/playout come and go. The pushed config still supplies the mode, + // options and speaker preference. False for an explicit per-platform + // override or manual mode, where the config is applied verbatim. + private var selectCategoryByEngineState = false + private var forceSpeakerOutput = false + private var isAutomaticManagementEnabled = true + private var sessionActive = false + // Last engine state seen, so an immediate re-apply (e.g. speaker toggle + // while the engine is running) can resolve the category from current state. + private var lastIsPlayoutEnabled = false + private var lastIsRecordingEnabled = false + #endif + + init(channel: FlutterMethodChannel) { + self.channel = channel + super.init() + } + + #if !os(macOS) + var isSessionActive: Bool { + lock.lock(); defer { lock.unlock() } + return sessionActive + } + + func setAutomaticManagementEnabled(_ enabled: Bool) { + lock.lock() + isAutomaticManagementEnabled = enabled + lock.unlock() + } + + /// Stores the audio session policy pushed from Dart. Pure cache, where the + /// delegate callbacks apply it. Callers decide whether to apply immediately. + func updatePolicy(_ configuration: RTCAudioSessionConfiguration, + automaticManagementEnabled: Bool, + selectCategoryByEngineState: Bool, + forceSpeakerOutput: Bool) { + let cachedConfiguration = copyConfiguration(configuration) + lock.lock() + self.cachedConfiguration = cachedConfiguration + self.isAutomaticManagementEnabled = automaticManagementEnabled + self.selectCategoryByEngineState = selectCategoryByEngineState + self.forceSpeakerOutput = forceSpeakerOutput + lock.unlock() + } + + /// Applies the cached configuration immediately, resolving the category from + /// the last known engine state when category selection is enabled. Used for + /// manual mode and for re-applying while the engine is already running. + func applyCachedConfiguration() -> Error? { + lock.lock() + let configuration = effectiveConfigurationLocked(isRecordingEnabled: lastIsRecordingEnabled) + let forceSpeakerOutput = self.forceSpeakerOutput + lock.unlock() + guard let configuration else { return nil } + return LiveKitPlugin.applyAudioSessionConfiguration(configuration, + forceSpeakerOutput: forceSpeakerOutput, + active: true) + } + + /// Resolves the configuration to apply for a given engine state. Must be + /// called with `lock` held. + /// + /// When category selection is disabled (explicit Apple override or manual + /// mode) the pushed config is applied verbatim. When enabled, recording + /// uses the pushed config (resolved as a playAndRecord policy by Dart), + /// while playout-only uses a coherent playback policy: flipping only the + /// category would leave playAndRecord-only mode/options (e.g. videoChat, + /// allowBluetooth) that are invalid for the playback category. Mirrors the + /// Swift SDK's `.playback` preset (playback + spokenAudio + mixWithOthers). + private func effectiveConfigurationLocked(isRecordingEnabled: Bool) -> RTCAudioSessionConfiguration? { + guard let configuration = cachedConfiguration else { return nil } + guard selectCategoryByEngineState, !isRecordingEnabled else { return configuration } + + let playback = copyConfiguration(configuration) + playback.category = AVAudioSession.Category.playback.rawValue + playback.categoryOptions = [.mixWithOthers] + playback.mode = AVAudioSession.Mode.spokenAudio.rawValue + return playback + } + + private func copyConfiguration(_ configuration: RTCAudioSessionConfiguration) -> RTCAudioSessionConfiguration { + let copy = RTCAudioSessionConfiguration() + copy.category = configuration.category + copy.categoryOptions = configuration.categoryOptions + copy.mode = configuration.mode + copy.sampleRate = configuration.sampleRate + copy.ioBufferDuration = configuration.ioBufferDuration + copy.inputNumberOfChannels = configuration.inputNumberOfChannels + copy.outputNumberOfChannels = configuration.outputNumberOfChannels + return copy + } + #endif + + // MARK: RTCAudioDeviceModuleDelegate, engine lifecycle + + func audioDeviceModule(_: RTCAudioDeviceModule, + willEnableEngine _: AVAudioEngine, + isPlayoutEnabled: Bool, + isRecordingEnabled: Bool) -> Int { + var resultCode = 0 + #if !os(macOS) + if isPlayoutEnabled || isRecordingEnabled { + lock.lock() + let shouldManageSession = isAutomaticManagementEnabled + let configuration = effectiveConfigurationLocked(isRecordingEnabled: isRecordingEnabled) + let forceSpeakerOutput = self.forceSpeakerOutput + lock.unlock() + + if shouldManageSession, + let configuration = configuration, + let error = LiveKitPlugin.applyAudioSessionConfiguration(configuration, + forceSpeakerOutput: forceSpeakerOutput, + active: true) { + print("[LiveKit] AudioEngine willEnable: failed to configure audio session: \(error)") + resultCode = LiveKitPlugin.kAudioEngineErrorFailedToConfigureAudioSession + } + if resultCode == 0 { + lock.lock() + lastIsPlayoutEnabled = isPlayoutEnabled + lastIsRecordingEnabled = isRecordingEnabled + if shouldManageSession { + sessionActive = true + } + lock.unlock() + } + } + #endif + if resultCode == 0 { + notifyEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + return resultCode + } + + func audioDeviceModule(_: RTCAudioDeviceModule, + didDisableEngine _: AVAudioEngine, + isPlayoutEnabled: Bool, + isRecordingEnabled: Bool) -> Int { + var resultCode = 0 + #if !os(macOS) + if !isPlayoutEnabled, !isRecordingEnabled { + lock.lock() + let shouldManageSession = isAutomaticManagementEnabled + lock.unlock() + + if shouldManageSession, let error = LiveKitPlugin.deactivateAudioSession() { + // Leave sessionActive unchanged (still true) so cached state + // keeps reflecting the live session. Flipping it to false here + // would make a later configureNativeAudio(automatic:) cache-only + // while the session is in fact still active. + print("[LiveKit] AudioEngine didDisable: failed to deactivate audio session: \(error)") + resultCode = LiveKitPlugin.kAudioEngineErrorFailedToConfigureAudioSession + } else if shouldManageSession { + lock.lock(); sessionActive = false; lock.unlock() + } + } + if resultCode == 0 { + lock.lock() + lastIsPlayoutEnabled = isPlayoutEnabled + lastIsRecordingEnabled = isRecordingEnabled + lock.unlock() + } + #endif + if resultCode == 0 { + notifyEngineState(isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + return resultCode + } + + // Remaining callbacks are not needed for session management (proceed / no-op). + func audioDeviceModule(_: RTCAudioDeviceModule, didCreateEngine _: AVAudioEngine) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, willStartEngine _: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, didStopEngine _: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, willReleaseEngine _: AVAudioEngine) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, engine _: AVAudioEngine, configureInputFromSource _: AVAudioNode?, toDestination _: AVAudioNode, format _: AVAudioFormat, context _: [AnyHashable: Any]) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, engine _: AVAudioEngine, configureOutputFromSource _: AVAudioNode, toDestination _: AVAudioNode?, format _: AVAudioFormat, context _: [AnyHashable: Any]) -> Int { 0 } + func audioDeviceModule(_: RTCAudioDeviceModule, didReceiveSpeechActivityEvent _: RTCSpeechActivityEvent) {} + func audioDeviceModuleDidUpdateDevices(_ audioDeviceModule: RTCAudioDeviceModule) { + FlutterWebRTCPlugin.sharedSingleton()?.audioDeviceModuleDidUpdateDevices(audioDeviceModule) + } + + private func notifyEngineState(isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let channel = channel else { return } + DispatchQueue.main.async { + channel.invokeMethod("onAudioEngineState", arguments: [ + "isPlayoutEnabled": isPlayoutEnabled, + "isRecordingEnabled": isRecordingEnabled, + ]) + } + } +} diff --git a/test/audio/audio_session_test.dart b/test/audio/audio_session_test.dart new file mode 100644 index 000000000..abe60400c --- /dev/null +++ b/test/audio/audio_session_test.dart @@ -0,0 +1,534 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter/services.dart'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/audio/android_audio_session_adapter.dart'; +import 'package:livekit_client/src/audio/audio_manager.dart'; +import 'package:livekit_client/src/audio/audio_session.dart'; +import 'package:livekit_client/src/support/native.dart'; +import 'package:livekit_client/src/support/native_audio.dart' as native_audio; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + setUp(() { + AudioManager.instance.resetForTest(); + Native.bypassVoiceProcessing = false; + }); + + group('AudioSessionManagementMode', () { + test('supports automatic and manual management', () { + expect( + AudioSessionManagementMode.values, + [ + AudioSessionManagementMode.automatic, + AudioSessionManagementMode.manual, + ], + ); + }); + }); + + group('AudioSessionOptions', () { + test('defaults to communication', () { + final options = AudioSessionOptions.communication(); + + expect(options.isCommunication, isTrue); + expect(options.isMedia, isFalse); + }); + + test('communication and media constructors describe session intent', () { + final communication = AudioSessionOptions.communication(); + final media = AudioSessionOptions.media(); + + expect(communication.isCommunication, isTrue); + expect(communication.isMedia, isFalse); + expect(media.isCommunication, isFalse); + expect(media.isMedia, isTrue); + }); + + test('copyWith updates and clears platform overrides', () { + const options = AudioSessionOptions.communication( + preferSpeakerOutput: true, + apple: AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + ), + android: AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.inCommunication, + ), + ); + + final updated = options.copyWith( + preferSpeakerOutput: const Value(false), + apple: const Value( + AppleAudioSessionConfiguration( + mode: AppleAudioMode.voiceChat, + ), + ), + ); + + expect(updated.preferSpeakerOutput, isFalse); + expect(updated.apple?.category, isNull); + expect(updated.apple?.mode, AppleAudioMode.voiceChat); + expect(updated.android?.audioMode, AndroidAudioMode.inCommunication); + + final cleared = updated.copyWith( + apple: const Value(null), + android: const Value(null), + ); + + expect(cleared.apple, isNull); + expect(cleared.android, isNull); + }); + + test('Apple configuration copyWith updates and clears nullable fields', () { + const config = AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: {AppleAudioCategoryOption.allowBluetooth}, + mode: AppleAudioMode.voiceChat, + preferSpeakerOutput: false, + ); + + final updated = config.copyWith( + category: const Value(AppleAudioCategory.playback), + categoryOptions: const Value({AppleAudioCategoryOption.mixWithOthers}), + mode: const Value(AppleAudioMode.spokenAudio), + preferSpeakerOutput: const Value(true), + ); + + expect(updated.category, AppleAudioCategory.playback); + expect(updated.categoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + expect(updated.mode, AppleAudioMode.spokenAudio); + expect(updated.preferSpeakerOutput, isTrue); + + final cleared = updated.copyWith( + category: const Value(null), + categoryOptions: const Value(null), + mode: const Value(null), + preferSpeakerOutput: const Value(null), + ); + + expect(cleared.category, isNull); + expect(cleared.categoryOptions, isNull); + expect(cleared.mode, isNull); + expect(cleared.preferSpeakerOutput, isNull); + }); + + test('Android configuration copyWith updates and clears nullable fields', () { + const config = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.inCommunication, + manageAudioFocus: true, + focusMode: AndroidAudioFocusMode.gain, + streamType: AndroidAudioStreamType.voiceCall, + usageType: AndroidAudioAttributesUsageType.voiceCommunication, + contentType: AndroidAudioAttributesContentType.speech, + forceAudioRouting: true, + ); + + final updated = config.copyWith( + audioMode: const Value(AndroidAudioMode.normal), + manageAudioFocus: const Value(false), + focusMode: const Value(AndroidAudioFocusMode.gainTransient), + streamType: const Value(AndroidAudioStreamType.music), + usageType: const Value(AndroidAudioAttributesUsageType.media), + contentType: const Value(AndroidAudioAttributesContentType.unknown), + forceAudioRouting: const Value(false), + ); + + expect(updated.audioMode, AndroidAudioMode.normal); + expect(updated.manageAudioFocus, isFalse); + expect(updated.focusMode, AndroidAudioFocusMode.gainTransient); + expect(updated.streamType, AndroidAudioStreamType.music); + expect(updated.usageType, AndroidAudioAttributesUsageType.media); + expect(updated.contentType, AndroidAudioAttributesContentType.unknown); + expect(updated.forceAudioRouting, isFalse); + + final cleared = updated.copyWith( + audioMode: const Value(null), + manageAudioFocus: const Value(null), + focusMode: const Value(null), + streamType: const Value(null), + usageType: const Value(null), + contentType: const Value(null), + forceAudioRouting: const Value(null), + ); + + expect(cleared.audioMode, isNull); + expect(cleared.manageAudioFocus, isNull); + expect(cleared.focusMode, isNull); + expect(cleared.streamType, isNull); + expect(cleared.usageType, isNull); + expect(cleared.contentType, isNull); + expect(cleared.forceAudioRouting, isNull); + }); + }); + + group('AudioManager', () { + test('management mode can be set independently from options', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionManagementMode(AudioSessionManagementMode.manual); + + expect(manager.managementMode, AudioSessionManagementMode.manual); + expect(manager.isAutomaticConfigurationEnabled, isFalse); + expect(manager.options.isCommunication, isTrue); + + await manager.setAudioSessionManagementMode(AudioSessionManagementMode.automatic); + }); + + test('setAudioSessionOptions syncs communication speaker preference', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionOptions( + const AudioSessionOptions.communication(preferSpeakerOutput: false), + ); + + expect(manager.isSpeakerOutputPreferred, isFalse); + expect(manager.options.preferSpeakerOutput, isFalse); + + await manager.setAudioSessionOptions( + const AudioSessionOptions.communication(preferSpeakerOutput: true), + ); + + expect(manager.isSpeakerOutputPreferred, isTrue); + expect(manager.options.preferSpeakerOutput, isTrue); + }); + + test('setAudioSessionOptions syncs explicit Apple speaker preference', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionOptions( + const AudioSessionOptions.communication( + apple: AppleAudioSessionConfiguration( + preferSpeakerOutput: false, + ), + ), + ); + + expect(manager.isSpeakerOutputPreferred, isFalse); + + await manager.setAudioSessionOptions( + const AudioSessionOptions.communication( + apple: AppleAudioSessionConfiguration( + preferSpeakerOutput: true, + ), + ), + ); + + expect(manager.isSpeakerOutputPreferred, isTrue); + }); + + test('resolves communication Apple session policy from speaker preference', () { + final manager = AudioManager.instance; + + final speaker = manager.resolveAppleAudioConfigurationForTest( + const AudioSessionOptions.communication(preferSpeakerOutput: true), + ); + + expect(speaker.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(speaker.appleAudioMode, AppleAudioMode.videoChat); + expect( + speaker.appleAudioCategoryOptions, + { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + + final receiver = manager.resolveAppleAudioConfigurationForTest( + const AudioSessionOptions.communication(preferSpeakerOutput: false), + ); + + expect(receiver.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(receiver.appleAudioMode, AppleAudioMode.voiceChat); + }); + + test('resolves media Apple session policy as dynamic playAndRecord base', () { + final config = AudioManager.instance.resolveAppleAudioConfigurationForTest( + const AudioSessionOptions.media(), + ); + + expect(config.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect(config.appleAudioMode, AppleAudioMode.default_); + expect( + config.appleAudioCategoryOptions, + { + AppleAudioCategoryOption.mixWithOthers, + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.allowBluetoothA2DP, + AppleAudioCategoryOption.allowAirPlay, + }, + ); + }); + + test('forced speaker does not mutate Apple category options', () { + final manager = AudioManager.instance; + + final playback = manager.resolveAppleAudioConfigurationForTest( + const AudioSessionOptions.media( + apple: AppleAudioSessionConfiguration( + category: AppleAudioCategory.playback, + categoryOptions: {AppleAudioCategoryOption.mixWithOthers}, + ), + ), + forceSpeakerOutput: true, + ); + + expect(playback.appleAudioCategory, AppleAudioCategory.playback); + expect(playback.appleAudioCategoryOptions, {AppleAudioCategoryOption.mixWithOthers}); + + final playAndRecord = manager.resolveAppleAudioConfigurationForTest( + const AudioSessionOptions.communication( + apple: AppleAudioSessionConfiguration( + category: AppleAudioCategory.playAndRecord, + categoryOptions: {AppleAudioCategoryOption.allowBluetooth}, + ), + ), + forceSpeakerOutput: true, + ); + + expect(playAndRecord.appleAudioCategory, AppleAudioCategory.playAndRecord); + expect( + playAndRecord.appleAudioCategoryOptions, + {AppleAudioCategoryOption.allowBluetooth}, + ); + }); + + test('resolves Android session policy from preset or explicit override', () { + final manager = AudioManager.instance; + + final communication = manager.resolveAndroidAudioConfigurationForTest( + const AudioSessionOptions.communication(), + ); + + expect(communication.audioMode, AndroidAudioMode.inCommunication); + expect(communication.streamType, AndroidAudioStreamType.voiceCall); + + final media = manager.resolveAndroidAudioConfigurationForTest( + const AudioSessionOptions.media(), + ); + + expect(media.audioMode, AndroidAudioMode.normal); + expect(media.streamType, AndroidAudioStreamType.music); + + final explicit = manager.resolveAndroidAudioConfigurationForTest( + const AudioSessionOptions.communication( + android: AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + forceAudioRouting: true, + ), + ), + ); + + expect(explicit.audioMode, AndroidAudioMode.normal); + expect(explicit.forceAudioRouting, isTrue); + }); + + test('Android initialize configuration uses active runtime options', () async { + final manager = AudioManager.instance; + + await manager.setAudioSessionOptions(const AudioSessionOptions.communication()); + manager.configureDefaults(bypassVoiceProcessing: true); + Native.bypassVoiceProcessing = true; + + expect( + manager.androidAudioConfigurationForInitialize(assumeAndroid: true), + containsPair('androidAudioMode', 'inCommunication'), + ); + }); + + test('handleAudioEngineState updates snapshot and stream', () async { + final manager = AudioManager.instance; + final states = []; + final subscription = manager.audioEngineStateStream.listen(states.add); + + manager.handleAudioEngineState( + isPlayoutEnabled: true, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect( + manager.audioEngineState, + const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false), + ); + expect(states, [const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false)]); + + manager.handleAudioEngineState( + isPlayoutEnabled: true, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect(states, [const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false)]); + + manager.handleAudioEngineState( + isPlayoutEnabled: false, + isRecordingEnabled: false, + ); + await pumpEventQueue(); + + expect(manager.audioEngineState.isIdle, isTrue); + expect( + states, + [ + const AudioEngineState(isPlayoutEnabled: true, isRecordingEnabled: false), + const AudioEngineState(isPlayoutEnabled: false, isRecordingEnabled: false), + ], + ); + + await subscription.cancel(); + }); + }); + + group('AndroidAudioSessionConfiguration', () { + test('communication preset uses voice communication values', () { + final config = AndroidAudioSessionConfiguration.communication; + + expect(config.manageAudioFocus, isTrue); + expect(config.audioMode, AndroidAudioMode.inCommunication); + expect(config.focusMode, AndroidAudioFocusMode.gain); + expect(config.streamType, AndroidAudioStreamType.voiceCall); + expect(config.usageType, AndroidAudioAttributesUsageType.voiceCommunication); + expect(config.contentType, AndroidAudioAttributesContentType.speech); + }); + + test('media preset uses non-communication media values', () { + final config = AndroidAudioSessionConfiguration.media; + + expect(config.manageAudioFocus, isTrue); + expect(config.audioMode, AndroidAudioMode.normal); + expect(config.focusMode, AndroidAudioFocusMode.gain); + expect(config.streamType, AndroidAudioStreamType.music); + expect(config.usageType, AndroidAudioAttributesUsageType.media); + expect(config.contentType, AndroidAudioAttributesContentType.unknown); + }); + }); + + group('NativeAudioConfiguration', () { + test('serializes Apple audio wire format', () { + final map = native_audio.NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioCategoryOptions: { + AppleAudioCategoryOption.allowBluetooth, + AppleAudioCategoryOption.defaultToSpeaker, + }, + appleAudioMode: AppleAudioMode.default_, + ).toMap(); + + expect(map['appleAudioCategory'], 'playAndRecord'); + expect( + map['appleAudioCategoryOptions'], + unorderedEquals([ + 'allowBluetooth', + 'defaultToSpeaker', + ]), + ); + expect(map['appleAudioMode'], 'default'); + expect(map.containsKey('preferSpeakerOutput'), isFalse); + }); + }); + + group('Native audio channel', () { + late List calls; + + setUp(() { + calls = []; + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + (call) async { + calls.add(call); + return call.method == 'configureNativeAudio' ? true : null; + }, + ); + }); + + tearDown(() { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger.setMockMethodCallHandler( + Native.channel, + null, + ); + }); + + test('passes forced speaker routing to Android platform method', () async { + await Native.setAndroidSpeakerphoneOn(true, force: true); + + expect(calls.single.method, 'setAndroidSpeakerphoneOn'); + expect(calls.single.arguments, {'enable': true, 'force': true}); + }); + + test('passes audio session deactivation to platform methods', () async { + await Native.stopAndroidAudioSession(); + await Native.deactivateAppleAudioSession(); + + expect(calls[0].method, 'stopAndroidAudioSession'); + expect(calls[0].arguments, isNull); + expect(calls[1].method, 'deactivateAppleAudioSession'); + expect(calls[1].arguments, {}); + }); + + test('passes forced speaker routing to automatic Apple configuration', () async { + final result = await Native.configureAudio( + native_audio.NativeAudioConfiguration( + appleAudioCategory: AppleAudioCategory.playAndRecord, + appleAudioMode: AppleAudioMode.videoChat, + ), + automatic: true, + selectCategoryByEngineState: true, + forceSpeakerOutput: true, + ); + + expect(result, isTrue); + expect(calls.single.method, 'configureNativeAudio'); + expect( + calls.single.arguments, + containsPair('forceSpeakerOutput', true), + ); + }); + }); + + group('androidAudioSessionConfigurationToMap', () { + test('serializes communication preset for WebRTC initialization', () { + expect( + androidAudioSessionConfigurationToMap(AndroidAudioSessionConfiguration.communication), + { + 'manageAudioFocus': true, + 'androidAudioMode': 'inCommunication', + 'androidAudioFocusMode': 'gain', + 'androidAudioStreamType': 'voiceCall', + 'androidAudioAttributesUsageType': 'voiceCommunication', + 'androidAudioAttributesContentType': 'speech', + }, + ); + }); + + test('omits unset Android fields', () { + final config = AndroidAudioSessionConfiguration( + audioMode: AndroidAudioMode.normal, + forceAudioRouting: true, + ); + + expect( + androidAudioSessionConfigurationToMap(config), + { + 'androidAudioMode': 'normal', + 'forceHandleAudioRouting': true, + }, + ); + }); + }); +}