Skip to content

Commit 8ba1000

Browse files
Merge pull request #3010 from CapSoftware/fix/avf-fallback-only-on-failure
fix(rendering): keep AVAssetReader decoding when a frame request only times out
2 parents b9ba57e + 61ca585 commit 8ba1000

2 files changed

Lines changed: 297 additions & 24 deletions

File tree

‎crates/rendering/src/decoder/mod.rs‎

Lines changed: 121 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,10 @@ pub(crate) const EXPORT_STALL_LIMIT: Duration = Duration::from_secs(120);
727727
enum FrameDeadline {
728728
/// Playback: a missed deadline returns no frame so the caller can move on.
729729
Drop,
730+
/// The first frame after a seek or for a paused preview. Nothing asks for
731+
/// it again, so on macOS a missed deadline switches to software decoding
732+
/// at once, which then answers it.
733+
Initial,
730734
/// Offline export: a missed deadline keeps waiting for the same reply.
731735
Wait,
732736
}
@@ -754,6 +758,60 @@ struct MacOsDecoderFallback {
754758
path: PathBuf,
755759
fps: u32,
756760
decoder: tokio::sync::OnceCell<Result<AsyncVideoDecoderHandle, String>>,
761+
stall: Mutex<PrimaryStall>,
762+
}
763+
764+
#[cfg(target_os = "macos")]
765+
#[derive(Default)]
766+
struct PrimaryStall {
767+
timed_out_rounds: u32,
768+
last_timeout: Option<tokio::time::Instant>,
769+
}
770+
771+
#[cfg(target_os = "macos")]
772+
impl MacOsDecoderFallback {
773+
/// A single playback timeout is a busy machine, not a dead reader: only a
774+
/// reader that misses this many playback rounds in a row without
775+
/// delivering any frame is replaced by software decoding.
776+
const MAX_TIMED_OUT_ROUNDS: u32 = 3;
777+
778+
fn new(name: &'static str, path: PathBuf, fps: u32) -> Self {
779+
Self {
780+
name,
781+
path,
782+
fps,
783+
decoder: tokio::sync::OnceCell::new(),
784+
stall: Mutex::new(PrimaryStall::default()),
785+
}
786+
}
787+
788+
fn primary_answered(&self) {
789+
*self
790+
.stall
791+
.lock()
792+
.unwrap_or_else(std::sync::PoisonError::into_inner) = PrimaryStall::default();
793+
}
794+
795+
/// Playback keeps many requests in flight, and one slow decode times them
796+
/// out together; only a request sent after the previous timeout starts a
797+
/// new round.
798+
fn primary_stalled(&self, sent_at: tokio::time::Instant) -> bool {
799+
let mut stall = self
800+
.stall
801+
.lock()
802+
.unwrap_or_else(std::sync::PoisonError::into_inner);
803+
if stall.last_timeout.is_none_or(|last| sent_at >= last) {
804+
stall.timed_out_rounds += 1;
805+
stall.last_timeout = Some(tokio::time::Instant::now());
806+
}
807+
stall.timed_out_rounds >= Self::MAX_TIMED_OUT_ROUNDS
808+
}
809+
}
810+
811+
enum FrameReply {
812+
Frame(DecodedFrame),
813+
NoFrame,
814+
TimedOut,
757815
}
758816

759817
impl AsyncVideoDecoderHandle {
@@ -789,7 +847,7 @@ impl AsyncVideoDecoderHandle {
789847
Self::INITIAL_SEEK_TIMEOUT_MS,
790848
self.max_fallback_distance
791849
.min(Self::INITIAL_MAX_FALLBACK_DISTANCE),
792-
FrameDeadline::Drop,
850+
FrameDeadline::Initial,
793851
)
794852
.await
795853
.ok()
@@ -836,12 +894,24 @@ impl AsyncVideoDecoderHandle {
836894
Err(_) => Ok(None),
837895
};
838896
}
839-
let primary = self
840-
.request_frame(time, timeout_ms, max_fallback_distance, deadline)
841-
.await;
842-
if let Ok(Some(frame)) = primary {
843-
return Ok(Some(frame));
844-
}
897+
let sent_at = tokio::time::Instant::now();
898+
let (fallback_reason, primary) = match self
899+
.send_frame_request(time, timeout_ms, max_fallback_distance, deadline)
900+
.await
901+
{
902+
Ok(FrameReply::Frame(frame)) => {
903+
fallback.primary_answered();
904+
return Ok(Some(frame));
905+
}
906+
Ok(FrameReply::TimedOut)
907+
if deadline == FrameDeadline::Drop && !fallback.primary_stalled(sent_at) =>
908+
{
909+
return Ok(None);
910+
}
911+
Ok(FrameReply::TimedOut) => ("AVAssetReader stopped answering", Ok(None)),
912+
Ok(FrameReply::NoFrame) => ("AVAssetReader returned no frame", Ok(None)),
913+
Err(stalled) => ("AVAssetReader stalled", Err(stalled)),
914+
};
845915
let result = fallback
846916
.decoder
847917
.get_or_init(|| async {
@@ -858,8 +928,7 @@ impl AsyncVideoDecoderHandle {
858928
false,
859929
)
860930
.await?;
861-
decoder.status.fallback_reason =
862-
Some("AVAssetReader returned no frame".to_string());
931+
decoder.status.fallback_reason = Some(fallback_reason.to_string());
863932
Ok(decoder)
864933
})
865934
.await;
@@ -886,6 +955,21 @@ impl AsyncVideoDecoderHandle {
886955
max_fallback_distance: u32,
887956
deadline: FrameDeadline,
888957
) -> Result<Option<DecodedFrame>, DecodeStalled> {
958+
self.send_frame_request(time, timeout_ms, max_fallback_distance, deadline)
959+
.await
960+
.map(|reply| match reply {
961+
FrameReply::Frame(frame) => Some(frame),
962+
FrameReply::NoFrame | FrameReply::TimedOut => None,
963+
})
964+
}
965+
966+
async fn send_frame_request(
967+
&self,
968+
time: f32,
969+
timeout_ms: u64,
970+
max_fallback_distance: u32,
971+
deadline: FrameDeadline,
972+
) -> Result<FrameReply, DecodeStalled> {
889973
let (tx, mut rx) = tokio::sync::oneshot::channel();
890974
let adjusted_time = self.get_time(time);
891975

@@ -902,13 +986,14 @@ impl AsyncVideoDecoderHandle {
902986
time = adjusted_time,
903987
"decoder thread is gone; frame request dropped"
904988
);
905-
return Ok(None);
989+
return Ok(FrameReply::NoFrame);
906990
}
907991

908992
let started = tokio::time::Instant::now();
909993
loop {
910994
match tokio::time::timeout(Duration::from_millis(timeout_ms), &mut rx).await {
911-
Ok(result) => return Ok(result.ok()),
995+
Ok(Ok(frame)) => return Ok(FrameReply::Frame(frame)),
996+
Ok(Err(_)) => return Ok(FrameReply::NoFrame),
912997
Err(_) if deadline == FrameDeadline::Wait => {
913998
let waited = started.elapsed();
914999
if waited >= EXPORT_STALL_LIMIT {
@@ -932,7 +1017,7 @@ impl AsyncVideoDecoderHandle {
9321017
timeout_ms = timeout_ms,
9331018
"Frame decode request timed out"
9341019
);
935-
return Ok(None);
1020+
return Ok(FrameReply::TimedOut);
9361021
}
9371022
}
9381023
}
@@ -1515,12 +1600,11 @@ pub async fn spawn_decoder(
15151600
status,
15161601
max_fallback_distance: DEFAULT_MAX_FALLBACK_DISTANCE,
15171602
#[cfg(target_os = "macos")]
1518-
fallback: Some(Arc::new(MacOsDecoderFallback {
1603+
fallback: Some(Arc::new(MacOsDecoderFallback::new(
15191604
name,
1520-
path: path.clone(),
1605+
path.clone(),
15211606
fps,
1522-
decoder: tokio::sync::OnceCell::new(),
1523-
})),
1607+
))),
15241608
#[cfg(target_os = "macos")]
15251609
sequential_access: Some(sequential_access),
15261610
})
@@ -2448,15 +2532,22 @@ mod managed_worker_tests {
24482532
fallback_reason: None,
24492533
},
24502534
max_fallback_distance: 2,
2451-
fallback: Some(Arc::new(MacOsDecoderFallback {
2452-
name: "screen",
2453-
path: directory.path().to_path_buf(),
2454-
fps: 30,
2455-
decoder: tokio::sync::OnceCell::new(),
2456-
})),
2535+
fallback: Some(Arc::new(MacOsDecoderFallback::new(
2536+
"screen",
2537+
directory.path().to_path_buf(),
2538+
30,
2539+
))),
24572540
sequential_access: None,
24582541
};
24592542
let cloned = decoder.clone();
2543+
for _ in 1..MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS {
2544+
let (first, second) = tokio::join!(
2545+
decoder.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop),
2546+
cloned.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop)
2547+
);
2548+
assert!(matches!((first, second), (Ok(None), Ok(None))));
2549+
assert_eq!(decoder.decoder_type(), DecoderType::AVAssetReader);
2550+
}
24602551
let (first, second) = tokio::join!(
24612552
decoder.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop),
24622553
cloned.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop)
@@ -2468,14 +2559,20 @@ mod managed_worker_tests {
24682559
);
24692560
assert_eq!(decoder.decoder_type(), DecoderType::FFmpegSoftware);
24702561
assert_eq!(cloned.decoder_type(), DecoderType::FFmpegSoftware);
2471-
assert!(decoder.fallback_reason().is_some());
2562+
assert_eq!(
2563+
decoder.fallback_reason(),
2564+
Some("AVAssetReader stopped answering")
2565+
);
24722566
let (reference, join) = ordinary_worker(directory.path().to_path_buf(), 0.25, false).await;
24732567
assert_eq!(
24742568
frame_pixels(&first),
24752569
frame_pixels(&reference.get_frame_initial(0.0).await.unwrap())
24762570
);
24772571
let pending_before = receiver.try_iter().count();
2478-
assert_eq!(pending_before, 2);
2572+
assert_eq!(
2573+
pending_before,
2574+
2 * MacOsDecoderFallback::MAX_TIMED_OUT_ROUNDS as usize
2575+
);
24792576
assert!(decoder.get_frame_initial(0.5).await.is_some());
24802577
assert_eq!(receiver.try_iter().count(), 0);
24812578
drop(reference);

0 commit comments

Comments
 (0)