diff --git a/docs/content/docs/video-output.mdx b/docs/content/docs/video-output.mdx index 932361d3d5..1eabe31e4c 100644 --- a/docs/content/docs/video-output.mdx +++ b/docs/content/docs/video-output.mdx @@ -95,6 +95,34 @@ const videoOutput = VisionCamera.createVideoOutput({ > [!WARNING] > Enabling Audio requires microphone permission. +#### Custom iOS Audio Session Control + +On iOS, `AVCaptureSession` normally configures the app's `AVAudioSession` automatically when audio recording is active. + +If you only want to keep already-playing audio alive while recording, enable [`allowBackgroundAudioPlayback`](/api/react-native-vision-camera/interfaces/CameraProps#allowbackgroundaudioplayback): + +```tsx +const camera = useCamera({ + isActive: true, + device: device, + outputs: [videoOutput], + allowBackgroundAudioPlayback: true, +}) +``` + +If you need full control over the app's `AVAudioSession` category or mode, disable [`automaticallyConfiguresApplicationAudioSession`](/api/react-native-vision-camera/interfaces/CameraProps#automaticallyconfiguresapplicationaudiosession) and configure `AVAudioSession` yourself using your preferred native integration: + +```tsx +const camera = useCamera({ + isActive: true, + device: device, + outputs: [videoOutput], + automaticallyConfiguresApplicationAudioSession: false, +}) +``` + +This is especially useful for FaceTime-style experiences where you want to use a custom `AVAudioSession` mode such as `videoChat`, `voiceChat`, or `measurement` through a library like `react-native-volume-manager`. + #### Persistent Video Outputs By default, an active recording will be automatically stopped when the input [`CameraDevice`](/api/react-native-vision-camera/hybrid-objects/CameraDevice) changes (e.g. when the user flips the Camera from front to back). diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Audio/AudioSession.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Audio/AudioSession.swift index 6b097dd2f2..e8b9d0f987 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Audio/AudioSession.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Audio/AudioSession.swift @@ -16,7 +16,10 @@ class AudioSession { private let queue: DispatchQueue private let delegate: AudioFrameDelegate - init() throws { + init( + automaticallyConfiguresApplicationAudioSession: Bool, + allowBackgroundAudioPlayback: Bool? + ) throws { logger.log("Initializing AudioSession...") // 1. Create AVCaptureSession self.audioSession = AVCaptureSession() @@ -25,6 +28,12 @@ class AudioSession { let audioSession = self.audioSession defer { audioSession.commitConfiguration() } + updateConfiguration( + automaticallyConfiguresApplicationAudioSession: + automaticallyConfiguresApplicationAudioSession, + allowBackgroundAudioPlayback: allowBackgroundAudioPlayback + ) + // 3. Create audio input guard let microphone = AVCaptureDevice.default(for: .audio) else { throw RuntimeError.error(withMessage: "No microphone available!") @@ -58,6 +67,18 @@ class AudioSession { output.setSampleBufferDelegate(delegate, queue: queue) } + func updateConfiguration( + automaticallyConfiguresApplicationAudioSession: Bool, + allowBackgroundAudioPlayback: Bool? + ) { + audioSession.automaticallyConfiguresApplicationAudioSession = + automaticallyConfiguresApplicationAudioSession + if #available(iOS 18.0, *) { + audioSession.configuresApplicationAudioSessionToMixWithOthers = + automaticallyConfiguresApplicationAudioSession && (allowBackgroundAudioPlayback ?? false) + } + } + func setOnFrameListener(onFrame: @escaping (CMSampleBuffer, CMTime) -> Void) { delegate.onFrame = onFrame } diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/HybridCameraSession.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/HybridCameraSession.swift index 9d7beb1300..56d8eaf442 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/HybridCameraSession.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/HybridCameraSession.swift @@ -62,6 +62,11 @@ class HybridCameraSession: HybridCameraSessionSpec { ) } + let automaticallyConfiguresApplicationAudioSession = + config?.automaticallyConfiguresApplicationAudioSession ?? true + self.session.automaticallyConfiguresApplicationAudioSession = + automaticallyConfiguresApplicationAudioSession + // Remove all unwanted inputs and add all new inputs try self.updateInputs(connections) // Remove all unwanted outputs and add all new outputs @@ -79,7 +84,7 @@ class HybridCameraSession: HybridCameraSessionSpec { switch outputConfiguration.output { case let output as any NativeCameraOutput: // Configure AVCaptureOutput - output.configure(config: outputConfiguration) + output.configure(config: outputConfiguration, sessionConfig: config) case let previewOutput as any NativePreviewViewOutput: // Configure AVCaptureVideoPreviewLayer previewOutput.configure(config: outputConfiguration) @@ -100,10 +105,10 @@ class HybridCameraSession: HybridCameraSessionSpec { self.session.automaticallyConfiguresCaptureDeviceForWideColor = !hasCustomDynamicRangeConstraint // Background Audio Playback - if let allowBackgroundAudioPlayback = config?.allowBackgroundAudioPlayback { - if #available(iOS 18.0, *) { - self.session.configuresApplicationAudioSessionToMixWithOthers = allowBackgroundAudioPlayback - } + if #available(iOS 18.0, *) { + self.session.configuresApplicationAudioSessionToMixWithOthers = + automaticallyConfiguresApplicationAudioSession + && (config?.allowBackgroundAudioPlayback ?? false) } // Return CameraControllers per connection to adjust camera settings (focus, etc) diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift index ccb84b870e..7e9e5647a5 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift @@ -14,6 +14,7 @@ class HybridCameraVideoFrameOutput: HybridCameraVideoOutputSpec, NativeCameraOut private let queue: DispatchQueue private let videoQueue: DispatchQueue private var audioSession: AudioSession? = nil + private var sessionConfiguration: CameraSessionConfiguration? = nil let mediaType: MediaType = .video let requiresAudioInput: Bool = false let requiresDepthFormat: Bool = false @@ -85,6 +86,22 @@ class HybridCameraVideoFrameOutput: HybridCameraVideoOutputSpec, NativeCameraOut try? connection.setOrientation(outputOrientation) } + func configure( + config: CameraOutputConfiguration, + sessionConfig: CameraSessionConfiguration? + ) { + self.sessionConfiguration = sessionConfig + self.configure(config: config) + + if let audioSession { + audioSession.updateConfiguration( + automaticallyConfiguresApplicationAudioSession: + sessionConfig?.automaticallyConfiguresApplicationAudioSession ?? true, + allowBackgroundAudioPlayback: sessionConfig?.allowBackgroundAudioPlayback + ) + } + } + func getSupportedVideoCodecs() throws -> [VideoCodec] { guard output.connection(with: .video) != nil else { throw RuntimeError.error( @@ -212,7 +229,11 @@ class HybridCameraVideoFrameOutput: HybridCameraVideoOutputSpec, NativeCameraOut return audioSession } // 1. Create session - let audioSession = try AudioSession() + let audioSession = try AudioSession( + automaticallyConfiguresApplicationAudioSession: + sessionConfiguration?.automaticallyConfiguresApplicationAudioSession ?? true, + allowBackgroundAudioPlayback: sessionConfiguration?.allowBackgroundAudioPlayback + ) // 2. Add on frame listener audioSession.setOnFrameListener { [weak self] buffer, timestamp in guard let self else { return } diff --git a/packages/react-native-vision-camera/ios/Public/NativeCameraOutput.swift b/packages/react-native-vision-camera/ios/Public/NativeCameraOutput.swift index d46547994b..c33d448028 100644 --- a/packages/react-native-vision-camera/ios/Public/NativeCameraOutput.swift +++ b/packages/react-native-vision-camera/ios/Public/NativeCameraOutput.swift @@ -56,4 +56,25 @@ public protocol NativeCameraOutput: AnyObject, ResolutionNegotionParticipant { * such as orientation or mirroring settings in here. */ func configure(config: CameraOutputConfiguration) + /** + * Called whenever the `CameraOutputConfiguration` or + * session-wide configuration might change. + * + * Outputs that need session-wide settings can override + * this method. The default implementation delegates to + * `configure(config:)`. + */ + func configure( + config: CameraOutputConfiguration, + sessionConfig: CameraSessionConfiguration? + ) +} + +public extension NativeCameraOutput { + func configure( + config: CameraOutputConfiguration, + sessionConfig: CameraSessionConfiguration? + ) { + self.configure(config: config) + } } diff --git a/packages/react-native-vision-camera/nitrogen/generated/android/c++/JCameraSessionConfiguration.hpp b/packages/react-native-vision-camera/nitrogen/generated/android/c++/JCameraSessionConfiguration.hpp index 25ffaecb97..5077487a20 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/android/c++/JCameraSessionConfiguration.hpp +++ b/packages/react-native-vision-camera/nitrogen/generated/android/c++/JCameraSessionConfiguration.hpp @@ -31,9 +31,12 @@ namespace margelo::nitro::camera { [[nodiscard]] CameraSessionConfiguration toCpp() const { static const auto clazz = javaClassStatic(); + static const auto fieldAutomaticallyConfiguresApplicationAudioSession = clazz->getField("automaticallyConfiguresApplicationAudioSession"); + jni::local_ref automaticallyConfiguresApplicationAudioSession = this->getFieldValue(fieldAutomaticallyConfiguresApplicationAudioSession); static const auto fieldAllowBackgroundAudioPlayback = clazz->getField("allowBackgroundAudioPlayback"); jni::local_ref allowBackgroundAudioPlayback = this->getFieldValue(fieldAllowBackgroundAudioPlayback); return CameraSessionConfiguration( + automaticallyConfiguresApplicationAudioSession != nullptr ? std::make_optional(static_cast(automaticallyConfiguresApplicationAudioSession->value())) : std::nullopt, allowBackgroundAudioPlayback != nullptr ? std::make_optional(static_cast(allowBackgroundAudioPlayback->value())) : std::nullopt ); } @@ -44,11 +47,12 @@ namespace margelo::nitro::camera { */ [[maybe_unused]] static jni::local_ref fromCpp(const CameraSessionConfiguration& value) { - using JSignature = JCameraSessionConfiguration(jni::alias_ref); + using JSignature = JCameraSessionConfiguration(jni::alias_ref, jni::alias_ref); static const auto clazz = javaClassStatic(); static const auto create = clazz->getStaticMethod("fromCpp"); return create( clazz, + value.automaticallyConfiguresApplicationAudioSession.has_value() ? jni::JBoolean::valueOf(value.automaticallyConfiguresApplicationAudioSession.value()) : nullptr, value.allowBackgroundAudioPlayback.has_value() ? jni::JBoolean::valueOf(value.allowBackgroundAudioPlayback.value()) : nullptr ); } diff --git a/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/CameraSessionConfiguration.kt b/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/CameraSessionConfiguration.kt index 3c273c3e5b..abb45a655b 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/CameraSessionConfiguration.kt +++ b/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/CameraSessionConfiguration.kt @@ -17,6 +17,9 @@ import com.facebook.proguard.annotations.DoNotStrip @DoNotStrip @Keep data class CameraSessionConfiguration( + @DoNotStrip + @Keep + val automaticallyConfiguresApplicationAudioSession: Boolean?, @DoNotStrip @Keep val allowBackgroundAudioPlayback: Boolean? @@ -31,8 +34,8 @@ data class CameraSessionConfiguration( @Keep @Suppress("unused") @JvmStatic - private fun fromCpp(allowBackgroundAudioPlayback: Boolean?): CameraSessionConfiguration { - return CameraSessionConfiguration(allowBackgroundAudioPlayback) + private fun fromCpp(automaticallyConfiguresApplicationAudioSession: Boolean?, allowBackgroundAudioPlayback: Boolean?): CameraSessionConfiguration { + return CameraSessionConfiguration(automaticallyConfiguresApplicationAudioSession, allowBackgroundAudioPlayback) } } } diff --git a/packages/react-native-vision-camera/nitrogen/generated/ios/swift/CameraSessionConfiguration.swift b/packages/react-native-vision-camera/nitrogen/generated/ios/swift/CameraSessionConfiguration.swift index dc7a4798d8..bbdfa38a23 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/ios/swift/CameraSessionConfiguration.swift +++ b/packages/react-native-vision-camera/nitrogen/generated/ios/swift/CameraSessionConfiguration.swift @@ -18,8 +18,14 @@ public extension CameraSessionConfiguration { /** * Create a new instance of `CameraSessionConfiguration`. */ - init(allowBackgroundAudioPlayback: Bool?) { + init(automaticallyConfiguresApplicationAudioSession: Bool?, allowBackgroundAudioPlayback: Bool?) { self.init({ () -> bridge.std__optional_bool_ in + if let __unwrappedValue = automaticallyConfiguresApplicationAudioSession { + return bridge.create_std__optional_bool_(__unwrappedValue) + } else { + return .init() + } + }(), { () -> bridge.std__optional_bool_ in if let __unwrappedValue = allowBackgroundAudioPlayback { return bridge.create_std__optional_bool_(__unwrappedValue) } else { @@ -28,6 +34,18 @@ public extension CameraSessionConfiguration { }()) } + @inline(__always) + var automaticallyConfiguresApplicationAudioSession: Bool? { + return { () -> Bool? in + if bridge.has_value_std__optional_bool_(self.__automaticallyConfiguresApplicationAudioSession) { + let __unwrapped = bridge.get_std__optional_bool_(self.__automaticallyConfiguresApplicationAudioSession) + return __unwrapped + } else { + return nil + } + }() + } + @inline(__always) var allowBackgroundAudioPlayback: Bool? { return { () -> Bool? in diff --git a/packages/react-native-vision-camera/nitrogen/generated/shared/c++/CameraSessionConfiguration.hpp b/packages/react-native-vision-camera/nitrogen/generated/shared/c++/CameraSessionConfiguration.hpp index 1f8a422ded..36d32eb539 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/shared/c++/CameraSessionConfiguration.hpp +++ b/packages/react-native-vision-camera/nitrogen/generated/shared/c++/CameraSessionConfiguration.hpp @@ -39,11 +39,12 @@ namespace margelo::nitro::camera { */ struct CameraSessionConfiguration final { public: + std::optional automaticallyConfiguresApplicationAudioSession SWIFT_PRIVATE; std::optional allowBackgroundAudioPlayback SWIFT_PRIVATE; public: CameraSessionConfiguration() = default; - explicit CameraSessionConfiguration(std::optional allowBackgroundAudioPlayback): allowBackgroundAudioPlayback(allowBackgroundAudioPlayback) {} + explicit CameraSessionConfiguration(std::optional automaticallyConfiguresApplicationAudioSession, std::optional allowBackgroundAudioPlayback): automaticallyConfiguresApplicationAudioSession(automaticallyConfiguresApplicationAudioSession), allowBackgroundAudioPlayback(allowBackgroundAudioPlayback) {} public: friend bool operator==(const CameraSessionConfiguration& lhs, const CameraSessionConfiguration& rhs) = default; @@ -59,11 +60,13 @@ namespace margelo::nitro { static inline margelo::nitro::camera::CameraSessionConfiguration fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) { jsi::Object obj = arg.asObject(runtime); return margelo::nitro::camera::CameraSessionConfiguration( + JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "automaticallyConfiguresApplicationAudioSession"))), JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "allowBackgroundAudioPlayback"))) ); } static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::camera::CameraSessionConfiguration& arg) { jsi::Object obj(runtime); + obj.setProperty(runtime, PropNameIDCache::get(runtime, "automaticallyConfiguresApplicationAudioSession"), JSIConverter>::toJSI(runtime, arg.automaticallyConfiguresApplicationAudioSession)); obj.setProperty(runtime, PropNameIDCache::get(runtime, "allowBackgroundAudioPlayback"), JSIConverter>::toJSI(runtime, arg.allowBackgroundAudioPlayback)); return obj; } @@ -75,6 +78,7 @@ namespace margelo::nitro { if (!nitro::isPlainObject(runtime, obj)) { return false; } + if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "automaticallyConfiguresApplicationAudioSession")))) return false; if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "allowBackgroundAudioPlayback")))) return false; return true; } diff --git a/packages/react-native-vision-camera/src/hooks/internal/useCameraController.ts b/packages/react-native-vision-camera/src/hooks/internal/useCameraController.ts index 3a8de606cd..1e4fdac043 100644 --- a/packages/react-native-vision-camera/src/hooks/internal/useCameraController.ts +++ b/packages/react-native-vision-camera/src/hooks/internal/useCameraController.ts @@ -10,6 +10,7 @@ import { useMemoizedArray } from './useMemoizedArray' import { useStableCallback } from './useStableCallback' interface Config { + automaticallyConfiguresApplicationAudioSession?: boolean mirrorMode?: MirrorMode allowBackgroundAudioPlayback?: boolean constraints?: Constraint[] @@ -32,6 +33,7 @@ export function useCameraController( device: CameraDevice | undefined, outputs: CameraOutput[], { + automaticallyConfiguresApplicationAudioSession, mirrorMode = 'auto', constraints = [], onSessionConfigSelected, @@ -98,7 +100,11 @@ export function useCameraController( onSessionConfigSelected: stableOnSessionConfigSelected, }, ], - { allowBackgroundAudioPlayback: allowBackgroundAudioPlayback }, + { + automaticallyConfiguresApplicationAudioSession: + automaticallyConfiguresApplicationAudioSession, + allowBackgroundAudioPlayback: allowBackgroundAudioPlayback, + }, ) if (isCanceled) { controllers.forEach((c) => { @@ -116,6 +122,7 @@ export function useCameraController( } }, [ device, + automaticallyConfiguresApplicationAudioSession, mirrorMode, session, allowBackgroundAudioPlayback, diff --git a/packages/react-native-vision-camera/src/hooks/useCamera.ts b/packages/react-native-vision-camera/src/hooks/useCamera.ts index c647099175..a2d7272015 100644 --- a/packages/react-native-vision-camera/src/hooks/useCamera.ts +++ b/packages/react-native-vision-camera/src/hooks/useCamera.ts @@ -27,6 +27,30 @@ export interface CameraProps { // Session Configuration isActive: boolean enableMultiCamSupport?: boolean + /** + * Whether the underlying iOS `AVCaptureSession`s should automatically + * configure the application's `AVAudioSession` when audio recording is enabled. + * + * Disable this if you want to manage `AVAudioSession` yourself, + * for example to set a custom `videoChat` or `measurement` mode via + * another library. + * + * @default true + * @platform iOS + */ + automaticallyConfiguresApplicationAudioSession?: boolean + /** + * If enabled, audio that is already playing can continue while a recording + * is in progress instead of being interrupted by the Camera. + * + * This only applies when + * {@linkcode CameraProps.automaticallyConfiguresApplicationAudioSession} + * is enabled. + * + * @default false + * @platform iOS + */ + allowBackgroundAudioPlayback?: boolean // Connection Configuration device: CameraDevice | CameraPosition @@ -100,6 +124,8 @@ function defaultOnErrorHandler(error: Error) { export function useCamera({ isActive, enableMultiCamSupport = false, + automaticallyConfiguresApplicationAudioSession, + allowBackgroundAudioPlayback, device, outputs = [], constraints, @@ -153,6 +179,9 @@ export function useCamera({ // 4. Configure the session with the input + outputs to create a `CameraController` const controller = useCameraController(session, input, outputs, { + automaticallyConfiguresApplicationAudioSession: + automaticallyConfiguresApplicationAudioSession, + allowBackgroundAudioPlayback: allowBackgroundAudioPlayback, mirrorMode: mirrorMode, onConfigured: onConfigured, getInitialExposureBias: getInitialExposureBias, diff --git a/packages/react-native-vision-camera/src/specs/session/CameraSession.nitro.ts b/packages/react-native-vision-camera/src/specs/session/CameraSession.nitro.ts index 6b19b9ab29..c5cee21481 100644 --- a/packages/react-native-vision-camera/src/specs/session/CameraSession.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/session/CameraSession.nitro.ts @@ -115,7 +115,8 @@ export interface CameraSession * as a multi-cam session (see {@linkcode CameraFactory.createCameraSession | createCameraSession(enableMultiCam)}), * you can add multiple {@linkcode CameraSessionConnection}s. * @param config Configures session-wide configuration, - * such as {@linkcode CameraSessionConfiguration.allowBackgroundAudioPlayback}. + * such as {@linkcode CameraSessionConfiguration.automaticallyConfiguresApplicationAudioSession} + * or {@linkcode CameraSessionConfiguration.allowBackgroundAudioPlayback}. * * @returns One {@linkcode CameraController} per {@linkcode CameraSessionConnection}. * diff --git a/packages/react-native-vision-camera/src/specs/session/CameraSessionConfiguration.ts b/packages/react-native-vision-camera/src/specs/session/CameraSessionConfiguration.ts index 9f7fc2400c..c1b3621396 100644 --- a/packages/react-native-vision-camera/src/specs/session/CameraSessionConfiguration.ts +++ b/packages/react-native-vision-camera/src/specs/session/CameraSessionConfiguration.ts @@ -4,6 +4,23 @@ import type { CameraSession } from './CameraSession.nitro' * Configuration for a {@linkcode CameraSession}. */ export interface CameraSessionConfiguration { + /** + * Whether the underlying iOS `AVCaptureSession`s should automatically + * configure the application's `AVAudioSession` while recording audio. + * + * Disable this if you want to fully manage the app-wide + * `AVAudioSession` yourself, for example via + * `react-native-volume-manager`. + * + * This setting also applies to the persistent recorder's + * dedicated audio capture session. + * + * If disabled, {@linkcode allowBackgroundAudioPlayback} has no effect. + * + * @default true + * @platform iOS + */ + automaticallyConfiguresApplicationAudioSession?: boolean /** * If enabled, the {@linkcode CameraSession} does not interrupt * currently playing audio (e.g. music from a different app), @@ -11,6 +28,10 @@ export interface CameraSessionConfiguration { * * If disabled (the default), any playing audio will be stopped * when a recording is started. + * + * This only applies when + * {@linkcode automaticallyConfiguresApplicationAudioSession} + * is enabled. * @default false * @platform iOS */