Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/content/docs/constraints.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,20 @@ const constraints = [
] satisfies Constraint[]
```

#### High-speed and slow-motion

On Android, you can request a high-speed capture session with a [`{ videoRecordingMode: ... }`](/api/react-native-vision-camera/interfaces/VideoRecordingModeConstraint) constraint.
Use `'high-speed'` to save a high frame-rate file directly, or `'slow-motion'` to save the recording with a slow-motion playback effect.

If you do not also provide an explicit [`{ fps: ... }`](/api/react-native-vision-camera/interfaces/FPSConstraint) constraint, VisionCamera automatically picks the lowest supported high-speed FPS for the negotiated session.

```ts
const constraints = [
{ videoRecordingMode: 'slow-motion' },
{ fps: 240 },
] satisfies Constraint[]
```

#### Binned preference

Binned formats combine multiple neighboring sensor pixels into one larger effective pixel.
Expand Down
15 changes: 15 additions & 0 deletions docs/content/docs/fps.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,18 @@ await session.start()
```
</Tab>
</Tabs>

### High-speed and Slow-motion

On Android, you can turn a Video recording into a negotiated high-speed capture session by combining your Video output with a [`{ videoRecordingMode: ... }`](/api/react-native-vision-camera/interfaces/VideoRecordingModeConstraint) constraint:

```ts
const constraints = [
{ videoRecordingMode: 'slow-motion' },
{ fps: 240 },
]
```

- Use `'high-speed'` to save the file at the negotiated high FPS.
- Use `'slow-motion'` to record at a high FPS and save the file for slow-motion playback.
- If you omit the [`{ fps: ... }`](/api/react-native-vision-camera/interfaces/FPSConstraint) constraint, VisionCamera automatically picks the lowest supported high-speed FPS for that session.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import com.margelo.nitro.camera.HybridCameraSessionConfigSpec
import com.margelo.nitro.camera.PixelFormat
import com.margelo.nitro.camera.TargetDynamicRange
import com.margelo.nitro.camera.TargetStabilizationMode
import com.margelo.nitro.camera.VideoRecordingMode
import com.margelo.nitro.camera.public.NativeCameraOutput
import com.margelo.nitro.camera.public.NativeCameraSessionConfig

Expand All @@ -31,6 +32,9 @@ class HybridCameraSessionConfig(
override val isPhotoHDREnabled: Boolean
get() = resolvedConfig.photoHDR ?: false

override val selectedVideoRecordingMode: VideoRecordingMode?
get() = resolvedConfig.videoRecordingMode

override val nativePixelFormat: PixelFormat
// TODO: Do we always stream in PRIVATE?
get() = PixelFormat.PRIVATE
Expand All @@ -52,6 +56,7 @@ class HybridCameraSessionConfig(
"selectedPreviewStabilizationMode: $selectedPreviewStabilizationMode",
"selectedVideoDynamicRange: $selectedVideoDynamicRange",
"isPhotoHDREnabled: $isPhotoHDREnabled",
"selectedVideoRecordingMode: $selectedVideoRecordingMode",
)
return "CameraSessionConfig(${components.joinToString(", ")})"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package com.margelo.nitro.camera.hybrids.outputs

import android.annotation.SuppressLint
import android.content.Context
import android.util.Log
import androidx.camera.core.DynamicRange
import androidx.camera.video.FileOutputOptions
import androidx.camera.video.Recorder
import androidx.camera.video.VideoCapture
Expand All @@ -14,7 +12,6 @@ import com.margelo.nitro.camera.HybridRecorderSpec
import com.margelo.nitro.camera.MediaType
import com.margelo.nitro.camera.MirrorMode
import com.margelo.nitro.camera.RecorderSettings
import com.margelo.nitro.camera.Size
import com.margelo.nitro.camera.TargetStabilizationMode
import com.margelo.nitro.camera.VideoCodec
import com.margelo.nitro.camera.VideoOutputOptions
Expand Down Expand Up @@ -100,8 +97,10 @@ class HybridVideoOutput(
VideoCapture
.Builder(videoOutput)
.apply {
// isMirrored={...}
setMirrorMode(mirrorMode.toMirrorMode())
if (config.videoRecordingMode == null) {
// High-speed sessions cannot set mirror mode on VideoCapture.
setMirrorMode(mirrorMode.toMirrorMode())
}
Comment on lines +100 to +103
Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Limitations like these are super weird, and I personally would love to completely avoid that unsafe code.
It's possible by API design to write code that crashes here, and I don't understand why a High Speed Video Session doesn't support mirror mode tbh.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. The guard is only there because CameraX crashes when mirror mode is set on a high-speed VideoCapture, which is itself a sign that this is too leaky for a public first-class API.

// orientation from previous value
setTargetRotation(outputOrientation.surfaceRotation)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import androidx.camera.core.UseCase
import com.margelo.nitro.camera.MirrorMode
import com.margelo.nitro.camera.TargetDynamicRange
import com.margelo.nitro.camera.TargetStabilizationMode
import com.margelo.nitro.camera.VideoRecordingMode

interface NativeCameraOutput {
/**
Expand All @@ -17,6 +18,7 @@ interface NativeCameraOutput {
val videoStabilizationMode: TargetStabilizationMode?,
val videoDynamicRange: TargetDynamicRange?,
val photoHDR: Boolean?,
val videoRecordingMode: VideoRecordingMode?,
)

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import androidx.camera.core.CameraInfo
import androidx.camera.core.ImageCapture
import androidx.camera.core.Preview
import androidx.camera.core.SessionConfig
import androidx.camera.video.HighSpeedVideoSessionConfig
import androidx.camera.video.Recorder
import androidx.camera.video.VideoCapture
import com.margelo.nitro.camera.CameraOutputConfiguration
import com.margelo.nitro.camera.Constraint
import com.margelo.nitro.camera.FPSConstraint
Expand All @@ -17,6 +19,8 @@ import com.margelo.nitro.camera.TargetDynamicRange
import com.margelo.nitro.camera.TargetDynamicRangeBitDepth
import com.margelo.nitro.camera.TargetStabilizationMode
import com.margelo.nitro.camera.VideoDynamicRangeConstraint
import com.margelo.nitro.camera.VideoRecordingMode
import com.margelo.nitro.camera.VideoRecordingModeConstraint
import com.margelo.nitro.camera.VideoStabilizationModeConstraint
import com.margelo.nitro.camera.extensions.converters.toDynamicRange
import com.margelo.nitro.camera.public.NativeCameraOutput
Expand Down Expand Up @@ -67,23 +71,19 @@ object ConstraintResolver {
return@map output.createUseCase(outputConfiguration.mirrorMode, config)
}
val sessionConfig =
SessionConfig
.Builder(preparedUseCases.map { it.useCase })
.build()
buildSessionConfig(cameraInfo, preparedUseCases, config)

if (cameraInfo.isSessionConfigSupported(sessionConfig) || activeConstraints.isEmpty()) {
if (sessionConfig != null && (cameraInfo.isSessionConfigSupported(sessionConfig) || activeConstraints.isEmpty())) {
// Pass 2: Resolve FPS against the validated use-case combination.
val resolvedFPSRange = fpsConstraint?.resolveFPSRange(cameraInfo, sessionConfig)
val resolvedFPSRange =
resolveFPSRange(
cameraInfo = cameraInfo,
sessionConfig = sessionConfig,
targetFps = fpsConstraint?.fps?.toInt(),
shouldAutoSelectLowest = config.videoRecordingMode != null,
)
val finalConfig = config.copy(fpsRange = resolvedFPSRange)
val finalSessionConfig =
if (resolvedFPSRange != null) {
SessionConfig
.Builder(preparedUseCases.map { it.useCase })
.apply { setFrameRateRange(resolvedFPSRange) }
.build()
} else {
sessionConfig
}
val finalSessionConfig = buildSessionConfig(cameraInfo, preparedUseCases, finalConfig) ?: sessionConfig
Log.i(TAG, "Resolved constraints to: $finalConfig")
return CameraSessionConfig(finalSessionConfig, preparedUseCases, finalConfig)
}
Expand All @@ -109,13 +109,20 @@ object ConstraintResolver {
* Picks the supported range whose upper bound is closest to the target FPS.
* On ties, prefers tighter ranges (higher lower bound).
*/
private fun FPSConstraint.resolveFPSRange(
private fun resolveFPSRange(
cameraInfo: CameraInfo,
sessionConfig: SessionConfig,
targetFps: Int?,
shouldAutoSelectLowest: Boolean,
): Range<Int>? {
val targetFps = fps.toInt()
val supportedRanges = cameraInfo.getSupportedFrameRateRanges(sessionConfig)
if (supportedRanges.isEmpty()) return null
if (targetFps == null) {
if (!shouldAutoSelectLowest) return null
return supportedRanges.minWith(
compareBy<Range<Int>> { it.upper }.thenBy { it.lower },
)
}

return supportedRanges.minWith(
compareBy<Range<Int>> {
Expand All @@ -127,6 +134,49 @@ object ConstraintResolver {
},
)
}

private fun buildSessionConfig(
cameraInfo: CameraInfo,
preparedUseCases: List<NativeCameraOutput.PreparedUseCase>,
config: NativeCameraOutput.Config,
): SessionConfig? {
return when (config.videoRecordingMode) {
null -> {
SessionConfig
.Builder(preparedUseCases.map { it.useCase })
.apply {
config.fpsRange?.let { setFrameRateRange(it) }
}.build()
}
else -> buildHighSpeedSessionConfig(cameraInfo, preparedUseCases, config)
}
}

private fun buildHighSpeedSessionConfig(
cameraInfo: CameraInfo,
preparedUseCases: List<NativeCameraOutput.PreparedUseCase>,
config: NativeCameraOutput.Config,
): SessionConfig? {
if (Recorder.getHighSpeedVideoCapabilities(cameraInfo) == null) return null

val previewUseCases = preparedUseCases.mapNotNull { it.useCase as? Preview }
val videoCaptureUseCases = preparedUseCases.mapNotNull { it.useCase as? VideoCapture<*> }
val containsUnsupportedUseCases =
preparedUseCases.any { preparedUseCase ->
preparedUseCase.useCase !is Preview && preparedUseCase.useCase !is VideoCapture<*>
}
if (containsUnsupportedUseCases || videoCaptureUseCases.size != 1 || previewUseCases.size > 1) {
return null
}

return HighSpeedVideoSessionConfig
.Builder(videoCaptureUseCases.single())
.apply {
previewUseCases.singleOrNull()?.let { setPreview(it) }
setSlowMotionEnabled(config.videoRecordingMode == VideoRecordingMode.SLOW_MOTION)
config.fpsRange?.let { setFrameRateRange(it) }
}.build()
}
}

// MARK: - Config Resolution
Expand All @@ -138,7 +188,7 @@ object ConstraintResolver {
* constraint are `null`, meaning "platform decides".
*
* FPS is not resolved here — it requires a validated [SessionConfig]
* and is resolved separately in [FPSConstraint.resolveFPSRange].
* and is resolved separately in [ConstraintResolver.resolveConstraints].
*/
internal fun List<Constraint>.toConfig(): NativeCameraOutput.Config {
return NativeCameraOutput.Config(
Expand All @@ -147,6 +197,7 @@ internal fun List<Constraint>.toConfig(): NativeCameraOutput.Config {
videoStabilizationMode = firstNotNullOfOrNull { it.asType<VideoStabilizationModeConstraint>() }?.videoStabilizationMode,
videoDynamicRange = firstNotNullOfOrNull { it.asType<VideoDynamicRangeConstraint>() }?.videoDynamicRange,
photoHDR = firstNotNullOfOrNull { it.asType<PhotoHDRConstraint>() }?.photoHDR,
videoRecordingMode = firstNotNullOfOrNull { it.asType<VideoRecordingModeConstraint>() }?.videoRecordingMode,
)
}

Expand All @@ -161,7 +212,9 @@ internal fun List<Constraint>.toConfig(): NativeCameraOutput.Config {
* When in doubt, returns `true` — a constraint that wrongly returns `false`
* is silently dropped with no chance of recovery.
*/
private fun Constraint.isSupportedIndividually(cameraInfo: CameraInfo): Boolean {
private fun Constraint.isSupportedIndividually(
cameraInfo: CameraInfo,
): Boolean {
return this.match(
{ true }, // FPS: resolved separately after features
{ videoStabilizationMode ->
Expand Down Expand Up @@ -195,6 +248,7 @@ private fun Constraint.isSupportedIndividually(cameraInfo: CameraInfo): Boolean
},
{ true }, // PixelFormat: let downgrade loop handle it
{ true }, // Binning: not configurable in CameraX
{ Recorder.getHighSpeedVideoCapabilities(cameraInfo) != null },
Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not very explicit - I would probably add an extension (or a var above) like isHighSpeedVideoRecordingSupported instead of comparing for null

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. If this direction were to continue, I would rename that for clarity. But after your broader feedback, I think the bigger fix is to avoid surfacing this mode directly in the public API at all.

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep that makes sense

)
}

Expand Down Expand Up @@ -261,5 +315,6 @@ private fun Constraint.getNextBestOption(): Constraint? {
{ null }, // PhotoHDR: on or off, no middle ground
{ null }, // PixelFormat: no fallback
{ null }, // Binning: not configurable
{ null }, // VideoRecordingMode: falls back to regular recording
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,8 @@ enum ConstraintResolver {
selectedFPS: fps,
selectedVideoStabilizationMode: videoStabilizationMode,
selectedPreviewStabilizationMode: previewStabilizationMode,
selectedVideoDynamicRange: videoDynamicRange)
selectedVideoDynamicRange: videoDynamicRange,
selectedVideoRecordingMode: nil)

return FormatEvaluation(
format: format,
Expand Down Expand Up @@ -238,6 +239,9 @@ extension Constraint {
case .eigth(let binned):
let r = binned.resolve(for: format)
return ConstraintEvaluation(penalty: r.penalty, resolved: .formatOnly)
case .ninth( /* videoRecordingMode */ _):
// High-speed session orchestration is currently Android-only.
Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not currently; it is an Android only concept. iOS doesnt care about high speed or "normal" - everything is normal. You can just record higher FPS. Just like a Camera library should work, not two completely separate concepts just because FPS go higher.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, agreed. I only threaded it through iOS to keep the bridge shape compiling consistently, not because I think iOS should expose a separate “high-speed vs normal” concept.

return ConstraintEvaluation(penalty: .noPenalty, resolved: .formatOnly)
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ public struct EnabledConstraints {
let selectedVideoStabilizationMode: TargetStabilizationMode?
let selectedPreviewStabilizationMode: TargetStabilizationMode?
let selectedVideoDynamicRange: TargetDynamicRange?
let selectedVideoRecordingMode: VideoRecordingMode?
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ final class HybridCameraSessionConfig: HybridCameraSessionConfigSpec, NativeCame
return negotiatedFormat.format.isHighestPhotoQualitySupported
}

var selectedVideoRecordingMode: VideoRecordingMode? {
return enabledConstraints.selectedVideoRecordingMode
}

var nativePixelFormat: PixelFormat {
let format = negotiatedFormat.format
return PixelFormat(mediaSubType: format.formatDescription.mediaSubType)
Expand All @@ -60,6 +64,7 @@ final class HybridCameraSessionConfig: HybridCameraSessionConfigSpec, NativeCame
"selectedPreviewStabilizationMode: \(String(describing: selectedPreviewStabilizationMode))",
"selectedVideoDynamicRange: \(String(describing: selectedVideoDynamicRange))",
"isPhotoHDREnabled: \(isPhotoHDREnabled)",
"selectedVideoRecordingMode: \(String(describing: selectedVideoRecordingMode))",
]
return "CameraSessionConfig(\(components.joined(separator: ", ")))"
}
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading