diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rs index 0a887bef0a..fe3a1e0b53 100644 --- a/crates/editor/src/export_audio.rs +++ b/crates/editor/src/export_audio.rs @@ -20,6 +20,7 @@ pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096; const MAX_PARALLEL_AUDIO_SOURCES: usize = 4; const MAX_ENHANCED_WINDOW_SAMPLES: usize = EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES; +const STUDIO_SOUND_HISTORY_SAMPLES: usize = VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES; #[derive(Clone, Debug)] pub enum ExportAudioError { @@ -325,17 +326,6 @@ impl ExportAudioRenderer { { return false; } - // Studio Sound re-reads up to VOICE_PREROLL_SAMPLES before each cut to warm up the - // enhancer. The streaming reader only moves forward, so a cut shorter than that - // within one clip has to use the in-memory export path instead. An exact split - // (nothing removed) keeps the enhancer contiguous and still streams. - let studio_sound = - project.audio.improve && !project.audio.mute && project.audio.mic_volume_db > -30.0; - let min_gap = if studio_sound { - (VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES) as f64 / 48_000.0 - } else { - 0.0 - }; let mut ends = HashMap::new(); let mut duration = holds.iter().map(|(start, end)| end - start).sum::(); if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 { @@ -356,11 +346,7 @@ impl ExportAudioRenderer { return false; } if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end) - && (!timeline.transitions.is_empty() - || segment.start < previous_end - || (min_gap > 0.0 - && segment.start > previous_end - && segment.start < previous_end + min_gap)) + && (!timeline.transitions.is_empty() || segment.start < previous_end) { return false; } @@ -532,12 +518,16 @@ impl ExportAudioSources { ) }); let range = enhancer.source_range(start, end.saturating_sub(start)); - track.prepare(range.start as i128, range.end as i128)?; + track.prepare( + range.start as i128, + range.end as i128, + STUDIO_SOUND_HISTORY_SAMPLES, + )?; track.enhanced = Some(enhancer.render(&track.view(), start, end.saturating_sub(start))); track.enhancer = Some(enhancer); } else { - track.prepare(start, end)?; + track.prepare(start, end, 0)?; track.enhanced = None; track.enhancer = None; } @@ -657,20 +647,21 @@ impl ExportAudioTrack { ((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize } - fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> { + fn prepare(&mut self, start: i128, end: i128, history: usize) -> Result<(), ExportAudioError> { let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start { return Err(ExportAudioError::InvalidWindow); } let channels = self.source.channels() as usize; - let discard = start + let retained_start = start.saturating_sub(history).max(self.source_start); + let discard = retained_start .saturating_sub(self.source_start) .min(self.samples.len() / channels) * channels; self.samples.copy_within(discard.., 0); self.samples.truncate(self.samples.len() - discard); - self.source_start = start; + self.source_start = retained_start; while self.eof.is_none() && self.source.position() < end as u64 { let wanted = (end as u64 - self.source.position()).min(12_000) as usize; match self @@ -684,7 +675,7 @@ impl ExportAudioTrack { ChunkRead::Chunk(chunk) => { let chunk_start = usize::try_from(chunk.source_start_sample) .map_err(|_| ExportAudioError::InvalidWindow)?; - let discard = start + let discard = retained_start .saturating_sub(chunk_start) .min(chunk.samples.len() / channels) * channels; @@ -1240,7 +1231,7 @@ mod tests { .unwrap(), ); for track in candidate.sources.tracks.iter_mut().flatten() { - track.prepare(10, 100).unwrap(); + track.prepare(10, 100, 0).unwrap(); } let identity = |track: &ExportAudioTrack| { ( @@ -1498,7 +1489,7 @@ mod tests { } #[test] - fn eligibility_rejects_studio_sound_cuts_shorter_than_voice_preroll() { + fn eligibility_accepts_studio_sound_short_forward_cuts() { let mut short_cut = project(); short_cut .timeline @@ -1508,7 +1499,7 @@ mod tests { .push(segment(0, 0.34, 0.36)); assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3)); short_cut.audio.improve = true; - assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3)); + assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3)); short_cut.audio.mute = true; assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3)); @@ -1531,6 +1522,9 @@ mod tests { .segments .push(segment(0, 0.33, 0.9)); assert!(ExportAudioRenderer::eligible_sources(&exact_split, 3)); + + short_cut.timeline.as_mut().unwrap().segments[2].start = 0.32; + assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3)); } #[test] @@ -1538,6 +1532,100 @@ mod tests { assert_bounded_sink_variants([19], &[1]); } + #[test] + fn short_studio_sound_cuts_match_preview_after_history_rolls() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let paths = + ["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name)); + for ((path, frames), channels) in + paths.iter().zip([144_013, 96_007, 168_013]).zip([1, 2, 2]) + { + write_wav(path, frames, channels); + } + let data = paths + .iter() + .map(|path| Arc::new(AudioData::from_file(path).unwrap())) + .collect::>(); + for (tier, isolation) in [ + cap_project::VoiceIsolation::Light, + cap_project::VoiceIsolation::Balanced, + cap_project::VoiceIsolation::Strong, + ] + .into_iter() + .enumerate() + { + for offset in [0, 1] { + let mut project = project(); + project.audio.improve = true; + project.audio.isolation = isolation; + if offset == 1 { + project.clips[0].offsets.mic = 0.137_125; + project.clips[0].offsets.system_audio = -0.031_25; + project.clips[1].offsets.mic = -0.117_125; + project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR; + } + project.timeline.as_mut().unwrap().segments = vec![ + segment(0, 0.001_01, 0.72), + segment(0, 0.73, 0.8), + segment(0, 0.800_02, 0.92), + segment(0, 1.0, 1.63), + segment(1, 0.35, 1.4), + segment(1, 1.5, 2.2), + segment(0, 1.8, 2.5), + segment(0, 2.500_02, 3.05), + ]; + let requests = [997, 4_096, 48_001] + .into_iter() + .chain((tier == 1 && offset == 0).then_some(1)); + for request in requests { + let (mut reference, mut candidate) = fixture(&paths, &data); + reference.set_playhead(0.0, &project); + loop { + let expected = reference.render_frame_raw(request, &project); + let mut actual = Vec::new(); + let written = candidate + .render_chunks(request, &project, |offset, samples| { + assert_eq!(offset * 2, actual.len()); + assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2); + actual.extend_from_slice(samples); + Ok(()) + }) + .unwrap_or_else(|error| { + panic!("{isolation:?}, offset {offset}, request {request}: {error}") + }); + assert_eq!(written, expected.as_ref().map(|(count, _)| *count)); + let expected = expected.as_ref().map_or(&[][..], |(_, data)| data); + assert_eq!(actual.len(), expected.len()); + assert!( + actual + .iter() + .map(|sample| sample.to_bits()) + .eq(expected.iter().map(|sample| sample.to_bits())), + "{isolation:?}, offset {offset}, request {request}" + ); + for track in candidate.sources.tracks.iter().flatten() { + let limit = if track.enhancer.is_some() { + MAX_ENHANCED_WINDOW_SAMPLES + + VOICE_PREROLL_SAMPLES + + VOICE_WINDOW_PADDING_SAMPLES + } else { + EXPORT_AUDIO_BLOCK_SAMPLES + }; + assert!( + track.samples.len() <= limit * track.source.channels() as usize + ); + } + if written.is_none() { + break; + } + } + candidate.validate_to_end().unwrap(); + } + } + } + } + #[test] fn fullscreen_holds_match_full_renderer_without_retaining_whole_tracks() { assert_bounded_sink_variants(26..32, &[1, 7, 997, 4_096, 48_001]); @@ -1804,7 +1892,7 @@ mod tests { assert!( track.samples.len() <= (if track.enhancer.is_some() { - MAX_ENHANCED_WINDOW_SAMPLES + MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES } else { EXPORT_AUDIO_BLOCK_SAMPLES }) * track.source.channels() as usize @@ -2045,12 +2133,12 @@ mod tests { let cancellation = Arc::new(AtomicBool::new(false)); let mut track = ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap(); - track.prepare(8_000, 9_000).unwrap(); - track.prepare(8_500, 9_501).unwrap(); + track.prepare(8_000, 9_000, 0).unwrap(); + track.prepare(8_500, 9_501, 0).unwrap(); let full = AudioData::from_file(&path).unwrap(); assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]); assert!(matches!( - track.prepare(0, 1), + track.prepare(0, 1, 0), Err(ExportAudioError::InvalidWindow) )); let mut candidate = ExportAudioRenderer { @@ -2077,6 +2165,55 @@ mod tests { ); } + #[test] + fn source_history_reuses_decoded_samples_and_stays_bounded() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source.wav"); + write_wav(&path, 144_013, 2); + let cancellation = Arc::new(AtomicBool::new(false)); + let mut track = + ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap(); + let full = AudioData::from_file(&path).unwrap(); + track + .prepare(50_000, 52_000, STUDIO_SOUND_HISTORY_SAMPLES) + .unwrap(); + let position = track.source.position(); + let oldest = track.source_start; + track + .prepare(25_000, 27_000, STUDIO_SOUND_HISTORY_SAMPLES) + .unwrap(); + assert_eq!(track.source.position(), position); + assert_eq!(track.source_start, oldest); + assert_eq!( + track.view().sample_slice(25_000 * 2..27_000 * 2).unwrap(), + &full.samples()[25_000 * 2..27_000 * 2] + ); + assert!(matches!( + track.prepare( + oldest as i128 - 1, + oldest as i128, + STUDIO_SOUND_HISTORY_SAMPLES + ), + Err(ExportAudioError::InvalidWindow) + )); + for start in (60_000..150_000).step_by(997) { + track + .prepare(start, start + 1_632, STUDIO_SOUND_HISTORY_SAMPLES) + .unwrap(); + let end = track.source.position() as usize; + assert_eq!( + track.samples, + full.samples()[track.source_start * 2..end * 2] + ); + assert!( + track.samples.len() + <= (MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES) * 2 + ); + } + track.prepare(144_013, 144_014, 0).unwrap(); + assert!(track.samples.is_empty()); + } + #[test] fn sink_failure_stops_before_later_source_reads() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/export/examples/studio-sound-cut-benchmark.rs b/crates/export/examples/studio-sound-cut-benchmark.rs new file mode 100644 index 0000000000..8d4abab753 --- /dev/null +++ b/crates/export/examples/studio-sound-cut-benchmark.rs @@ -0,0 +1,118 @@ +use cap_export::{ + ExporterBase, + mp4::{ExportCompression, Mp4ExportSettings}, +}; +use cap_project::{ClipConfiguration, ProjectConfiguration, RecordingMeta, TimelineSegment, XY}; +use std::{ + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::Instant, +}; + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() { + let root = PathBuf::from(std::env::args_os().nth(1).expect("fixture directory")); + let output = PathBuf::from(std::env::args_os().nth(2).expect("output MP4 path")); + let gap: f64 = std::env::var("CAP_EXPORT_CUT_GAP").map_or(0.1, |v| v.parse().unwrap()); + let mut project = ProjectConfiguration { + timeline: Some( + serde_json::from_value(serde_json::json!({"segments":[],"zoomSegments":[]})).unwrap(), + ), + ..Default::default() + }; + project.audio.improve = true; + project.camera.hide = true; + project.cursor.hide = true; + project.cursor.raw = true; + project.clips = vec![ClipConfiguration { + index: 0, + ..Default::default() + }]; + project.timeline.as_mut().unwrap().segments = vec![ + TimelineSegment { + recording_clip: 0, + start: 0.0, + end: 4.0, + timescale: 1.0, + ..Default::default() + }, + TimelineSegment { + recording_clip: 0, + start: 4.0 + gap, + end: 8.0 + gap, + timescale: 1.0, + ..Default::default() + }, + ]; + project.write(&root).unwrap(); + let mut meta: RecordingMeta = serde_json::from_value(serde_json::json!({ + "platform":"Linux", "pretty_name":"Studio Sound cut benchmark", "sharing":null, + "segments":[{ + "display":{"path":"display.mp4","fps":1,"start_time":0.0}, + "mic":{"path":"mic.wav","start_time":0.0} + }], + "cursors":{},"status":{"status":"Complete"} + })) + .unwrap(); + meta.project_path = root.clone(); + meta.save_for_project().unwrap(); + let first_progress = Arc::new(AtomicU64::new(u64::MAX)); + let recorded_progress = first_progress.clone(); + let start = Instant::now(); + let base = ExporterBase::builder(root) + .with_output_path(output.clone()) + .with_force_ffmpeg_decoder(true) + .build_for_mp4(Arc::new(AtomicBool::new(false))) + .await + .expect("export preparation"); + let build_seconds = start.elapsed().as_secs_f64(); + let streaming = base.uses_streaming_audio(); + let settings = Mp4ExportSettings { + fps: 15, + resolution_base: XY::new(640, 360), + compression: ExportCompression::Web, + custom_bpp: None, + force_ffmpeg_decoder: true, + optimize_filesize: false, + }; + let export_start = Instant::now(); + settings + .export_prepared(base, move |_| { + recorded_progress + .compare_exchange( + u64::MAX, + start.elapsed().as_micros() as u64, + Ordering::Relaxed, + Ordering::Relaxed, + ) + .ok(); + true + }) + .await + .expect("complete export"); + let export_seconds = export_start.elapsed().as_secs_f64(); + let total_seconds = start.elapsed().as_secs_f64(); + let maxrss_kib = std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|status| { + status.lines().find_map(|line| { + line.strip_prefix("VmHWM:")? + .split_whitespace() + .next()? + .parse::() + .ok() + }) + }); + println!( + "EXPORT_BENCH {}", + serde_json::json!({ + "gap":gap,"streaming_audio":streaming,"build_seconds":build_seconds, + "first_progress_seconds":first_progress.load(Ordering::Relaxed) as f64 / 1_000_000.0, + "export_seconds":export_seconds,"total_seconds":total_seconds,"maxrss_kib":maxrss_kib, + "output":output + }) + ); +}