Skip to content

perf(editor): stream short Studio Sound cuts with bounded history #3254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richiemcilroy merged 1 commit into from
Oct 25, 2026
Merged

perf(editor): stream short Studio Sound cuts with bounded history #3254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 166 additions & 29 deletions crates/editor/src/export_audio.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096;
const MAX_PARALLEL_AUDIO_SOURCES: usize = 4;
const MAX_ENHANCED_WINDOW_SAMPLES: usize =
EXPORT_AUDIO_BLOCK_SAMPLES + VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;
const STUDIO_SOUND_HISTORY_SAMPLES: usize = VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES;

#[derive(Clone, Debug)]
pub enum ExportAudioError {
Expand Down Expand Up @@ -325,17 +326,6 @@ impl ExportAudioRenderer {
{
return false;
}
// Studio Sound re-reads up to VOICE_PREROLL_SAMPLES before each cut to warm up the
// enhancer. The streaming reader only moves forward, so a cut shorter than that
// within one clip has to use the in-memory export path instead. An exact split
// (nothing removed) keeps the enhancer contiguous and still streams.
let studio_sound =
project.audio.improve && !project.audio.mute && project.audio.mic_volume_db > -30.0;
let min_gap = if studio_sound {
(VOICE_PREROLL_SAMPLES + VOICE_WINDOW_PADDING_SAMPLES) as f64 / 48_000.0
} else {
0.0
};
let mut ends = HashMap::new();
let mut duration = holds.iter().map(|(start, end)| end - start).sum::<f64>();
if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 {
Expand All @@ -356,11 +346,7 @@ impl ExportAudioRenderer {
return false;
}
if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end)
&& (!timeline.transitions.is_empty()
|| segment.start < previous_end
|| (min_gap > 0.0
&& segment.start > previous_end
&& segment.start < previous_end + min_gap))
&& (!timeline.transitions.is_empty() || segment.start < previous_end)
{
return false;
}
Expand Down Expand Up @@ -532,12 +518,16 @@ impl ExportAudioSources {
)
});
let range = enhancer.source_range(start, end.saturating_sub(start));
track.prepare(range.start as i128, range.end as i128)?;
track.prepare(
range.start as i128,
range.end as i128,
STUDIO_SOUND_HISTORY_SAMPLES,
)?;
track.enhanced =
Some(enhancer.render(&track.view(), start, end.saturating_sub(start)));
track.enhancer = Some(enhancer);
} else {
track.prepare(start, end)?;
track.prepare(start, end, 0)?;
track.enhanced = None;
track.enhancer = None;
}
Expand Down Expand Up @@ -657,20 +647,21 @@ impl ExportAudioTrack {
((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize
}

fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> {
fn prepare(&mut self, start: i128, end: i128, history: usize) -> Result<(), ExportAudioError> {
let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?;
if end < start || end - start > MAX_ENHANCED_WINDOW_SAMPLES || start < self.source_start {
return Err(ExportAudioError::InvalidWindow);
}
let channels = self.source.channels() as usize;
let discard = start
let retained_start = start.saturating_sub(history).max(self.source_start);
let discard = retained_start
.saturating_sub(self.source_start)
.min(self.samples.len() / channels)
* channels;
self.samples.copy_within(discard.., 0);
self.samples.truncate(self.samples.len() - discard);
self.source_start = start;
self.source_start = retained_start;
while self.eof.is_none() && self.source.position() < end as u64 {
let wanted = (end as u64 - self.source.position()).min(12_000) as usize;
match self
Expand All @@ -684,7 +675,7 @@ impl ExportAudioTrack {
ChunkRead::Chunk(chunk) => {
let chunk_start = usize::try_from(chunk.source_start_sample)
.map_err(|_| ExportAudioError::InvalidWindow)?;
let discard = start
let discard = retained_start
.saturating_sub(chunk_start)
.min(chunk.samples.len() / channels)
* channels;
Expand Down Expand Up @@ -1240,7 +1231,7 @@ mod tests {
.unwrap(),
);
for track in candidate.sources.tracks.iter_mut().flatten() {
track.prepare(10, 100).unwrap();
track.prepare(10, 100, 0).unwrap();
}
let identity = |track: &ExportAudioTrack| {
(
Expand Down Expand Up @@ -1498,7 +1489,7 @@ mod tests {
}

#[test]
fn eligibility_rejects_studio_sound_cuts_shorter_than_voice_preroll() {
fn eligibility_accepts_studio_sound_short_forward_cuts() {
let mut short_cut = project();
short_cut
.timeline
Expand All @@ -1508,7 +1499,7 @@ mod tests {
.push(segment(0, 0.34, 0.36));
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.improve = true;
assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));
short_cut.audio.mute = true;
assert!(ExportAudioRenderer::eligible_sources(&short_cut, 3));

Expand All @@ -1531,13 +1522,110 @@ mod tests {
.segments
.push(segment(0, 0.33, 0.9));
assert!(ExportAudioRenderer::eligible_sources(&exact_split, 3));

short_cut.timeline.as_mut().unwrap().segments[2].start = 0.32;
assert!(!ExportAudioRenderer::eligible_sources(&short_cut, 3));
}

#[test]
fn studio_sound_transition_with_single_sample_requests_keeps_lookahead() {
assert_bounded_sink_variants([19], &[1]);
}

#[test]
fn short_studio_sound_cuts_match_preview_after_history_rolls() {
ffmpeg::init().unwrap();
let directory = tempfile::tempdir().unwrap();
let paths =
["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name));
for ((path, frames), channels) in
paths.iter().zip([144_013, 96_007, 168_013]).zip([1, 2, 2])
{
write_wav(path, frames, channels);
}
let data = paths
.iter()
.map(|path| Arc::new(AudioData::from_file(path).unwrap()))
.collect::<Vec<_>>();
for (tier, isolation) in [
cap_project::VoiceIsolation::Light,
cap_project::VoiceIsolation::Balanced,
cap_project::VoiceIsolation::Strong,
]
.into_iter()
.enumerate()
{
for offset in [0, 1] {
let mut project = project();
project.audio.improve = true;
project.audio.isolation = isolation;
if offset == 1 {
project.clips[0].offsets.mic = 0.137_125;
project.clips[0].offsets.system_audio = -0.031_25;
project.clips[1].offsets.mic = -0.117_125;
project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR;
}
project.timeline.as_mut().unwrap().segments = vec![
segment(0, 0.001_01, 0.72),
segment(0, 0.73, 0.8),
segment(0, 0.800_02, 0.92),
segment(0, 1.0, 1.63),
segment(1, 0.35, 1.4),
segment(1, 1.5, 2.2),
segment(0, 1.8, 2.5),
segment(0, 2.500_02, 3.05),
];
let requests = [997, 4_096, 48_001]
.into_iter()
.chain((tier == 1 && offset == 0).then_some(1));
for request in requests {
let (mut reference, mut candidate) = fixture(&paths, &data);
reference.set_playhead(0.0, &project);
loop {
let expected = reference.render_frame_raw(request, &project);
let mut actual = Vec::new();
let written = candidate
.render_chunks(request, &project, |offset, samples| {
assert_eq!(offset * 2, actual.len());
assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2);
actual.extend_from_slice(samples);
Ok(())
})
.unwrap_or_else(|error| {
panic!("{isolation:?}, offset {offset}, request {request}: {error}")
});
assert_eq!(written, expected.as_ref().map(|(count, _)| *count));
let expected = expected.as_ref().map_or(&[][..], |(_, data)| data);
assert_eq!(actual.len(), expected.len());
assert!(
actual
.iter()
.map(|sample| sample.to_bits())
.eq(expected.iter().map(|sample| sample.to_bits())),
"{isolation:?}, offset {offset}, request {request}"
);
for track in candidate.sources.tracks.iter().flatten() {
let limit = if track.enhancer.is_some() {
MAX_ENHANCED_WINDOW_SAMPLES
+ VOICE_PREROLL_SAMPLES
+ VOICE_WINDOW_PADDING_SAMPLES
} else {
EXPORT_AUDIO_BLOCK_SAMPLES
};
assert!(
track.samples.len() <= limit * track.source.channels() as usize
);
}
if written.is_none() {
break;
}
}
candidate.validate_to_end().unwrap();
}
}
}
}

#[test]
fn fullscreen_holds_match_full_renderer_without_retaining_whole_tracks() {
assert_bounded_sink_variants(26..32, &[1, 7, 997, 4_096, 48_001]);
Expand Down Expand Up @@ -1804,7 +1892,7 @@ mod tests {
assert!(
track.samples.len()
<= (if track.enhancer.is_some() {
MAX_ENHANCED_WINDOW_SAMPLES
MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES
} else {
EXPORT_AUDIO_BLOCK_SAMPLES
}) * track.source.channels() as usize
Expand Down Expand Up @@ -2045,12 +2133,12 @@ mod tests {
let cancellation = Arc::new(AtomicBool::new(false));
let mut track =
ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
track.prepare(8_000, 9_000).unwrap();
track.prepare(8_500, 9_501).unwrap();
track.prepare(8_000, 9_000, 0).unwrap();
track.prepare(8_500, 9_501, 0).unwrap();
let full = AudioData::from_file(&path).unwrap();
assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]);
assert!(matches!(
track.prepare(0, 1),
track.prepare(0, 1, 0),
Err(ExportAudioError::InvalidWindow)
));
let mut candidate = ExportAudioRenderer {
Expand All @@ -2077,6 +2165,55 @@ mod tests {
);
}

#[test]
fn source_history_reuses_decoded_samples_and_stays_bounded() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("source.wav");
write_wav(&path, 144_013, 2);
let cancellation = Arc::new(AtomicBool::new(false));
let mut track =
ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap();
let full = AudioData::from_file(&path).unwrap();
track
.prepare(50_000, 52_000, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
let position = track.source.position();
let oldest = track.source_start;
track
.prepare(25_000, 27_000, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
assert_eq!(track.source.position(), position);
assert_eq!(track.source_start, oldest);
assert_eq!(
track.view().sample_slice(25_000 * 2..27_000 * 2).unwrap(),
&full.samples()[25_000 * 2..27_000 * 2]
);
assert!(matches!(
track.prepare(
oldest as i128 - 1,
oldest as i128,
STUDIO_SOUND_HISTORY_SAMPLES
),
Err(ExportAudioError::InvalidWindow)
));
for start in (60_000..150_000).step_by(997) {
track
.prepare(start, start + 1_632, STUDIO_SOUND_HISTORY_SAMPLES)
.unwrap();
let end = track.source.position() as usize;
assert_eq!(
track.samples,
full.samples()[track.source_start * 2..end * 2]
);
assert!(
track.samples.len()
<= (MAX_ENHANCED_WINDOW_SAMPLES + STUDIO_SOUND_HISTORY_SAMPLES) * 2
);
}
track.prepare(144_013, 144_014, 0).unwrap();
assert!(track.samples.is_empty());
}

#[test]
fn sink_failure_stops_before_later_source_reads() {
let directory = tempfile::tempdir().unwrap();
Expand Down
Loading
Loading