From 6c00b3f29a095adbff570c7e0ba9ab73fdd4842a Mon Sep 17 00:00:00 2001 From: Yoav Levavi Date: Wed, 21 Oct 2026 18:21:02 +0300 Subject: [PATCH 1/2] gpui: audio scrub while dragging the playhead on the ruler With playback stopped, each ruler move plays a 90ms slice of the mix under the playhead (recording audio plus music) on a small output stream of its own. EditorInstance::audio_snippet renders the slice with the regular AudioRenderer. Co-Authored-By: Claude Opus 5.5 --- apps/desktop-gpui/src/editor_window.rs | 45 ++++++++ apps/desktop-gpui/src/main.rs | 1 + apps/desktop-gpui/src/scrub_audio.rs | 144 +++++++++++++++++++++++++ crates/editor/src/editor_instance.rs | 28 +++++ 4 files changed, 218 insertions(+) create mode 100644 apps/desktop-gpui/src/scrub_audio.rs diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index d19e1cd05c6..2b2ce2df6f4 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -1355,6 +1355,10 @@ pub struct EditorWindow { /// Kept alive for the window's lifetime: dropping the last `Arc` is what /// tears the decoders down, and `dispose()` on close does it explicitly. pub(crate) instance: Option>, + /// Audio scrub: a slice request is in flight, and the newest position + /// asked for meanwhile (played once it lands). + scrub_audio_busy: bool, + scrub_audio_next: Option, focus: FocusHandle, // -- Transport ---------------------------------------------------------- @@ -2048,6 +2052,8 @@ impl EditorWindow { total: 0.0, transport: None, scrub: None, + scrub_audio_busy: false, + scrub_audio_next: None, stats: None, play_mark: None, timeline: TimelineModel::default(), @@ -3877,6 +3883,7 @@ impl EditorWindow { // `applyScrub()` runs immediately on mousedown. self.scrub = Some(Scrub::Ruler); self.seek_to_time(time, cx); + self.scrub_audio_at(time, cx); } else { self.scrub = Some(Scrub::Press { time }); } @@ -3905,6 +3912,44 @@ impl EditorWindow { // in-flight work when it sees a newer instruction // (`editor_instance.rs:563-575`). self.seek_to_time(time, cx); + self.scrub_audio_at(time, cx); + } + + /// Audio scrub: while the ruler is dragged with playback stopped, play a + /// short slice of the mix under the playhead. One request at a time; the + /// newest position waits and plays next, spaced by about a slice so they + /// run back to back. + fn scrub_audio_at(&mut self, time: f64, cx: &mut Context) { + if self.playing { + return; + } + if self.scrub_audio_busy { + self.scrub_audio_next = Some(time); + return; + } + let Some(instance) = self.instance.clone() else { + return; + }; + self.scrub_audio_busy = true; + let slice = gpui_tokio::Tokio::spawn(cx, async move { + let samples = instance + .audio_snippet(time, crate::scrub_audio::SNIPPET_SECS) + .await; + crate::scrub_audio::play(samples); + }); + cx.spawn(async move |this, cx| { + let _ = slice.await; + cx.background_executor() + .timer(Duration::from_millis(60)) + .await; + let _ = this.update(cx, |this, cx| { + this.scrub_audio_busy = false; + if let Some(next) = this.scrub_audio_next.take() { + this.scrub_audio_at(next, cx); + } + }); + }) + .detach(); } /// The **timeline's own** mouseup, the one the source registers on diff --git a/apps/desktop-gpui/src/main.rs b/apps/desktop-gpui/src/main.rs index 9e886003506..b3205acbc30 100644 --- a/apps/desktop-gpui/src/main.rs +++ b/apps/desktop-gpui/src/main.rs @@ -68,6 +68,7 @@ mod screenshot_crop; mod screenshot_editor; mod screenshot_export; mod script_text; +mod scrub_audio; mod session; mod settings_pages; mod settings_window; diff --git a/apps/desktop-gpui/src/scrub_audio.rs b/apps/desktop-gpui/src/scrub_audio.rs new file mode 100644 index 00000000000..591fede5c5b --- /dev/null +++ b/apps/desktop-gpui/src/scrub_audio.rs @@ -0,0 +1,144 @@ +//! Audio scrubbing: while the playhead is dragged, the editor asks for a +//! short slice of the mix under it and this plays it on a small output +//! stream of its own, so it never disturbs the playback pipeline. + +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::Duration; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use tracing::warn; + +/// Length of each scrub slice; the next one replaces whatever is left. +pub const SNIPPET_SECS: f64 = 0.09; +/// The output stream is paused after this long without a slice. +const IDLE_PAUSE: Duration = Duration::from_secs(2); + +/// Interleaved stereo waiting to be played, and the read position in frames. +#[derive(Default)] +struct ScrubBuffer { + samples: Vec, + position: f64, +} + +static PLAYER: LazyLock>>>> = LazyLock::new(Default::default); + +/// Plays one scrub slice (interleaved stereo at +/// [`cap_editor::AudioRenderer::SAMPLE_RATE`]), replacing whatever is left of +/// the previous one. +pub fn play(samples: Vec) { + if samples.is_empty() { + return; + } + let mut player = PLAYER.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(tx) = player.as_ref() + && tx.send(samples.clone()).is_ok() + { + return; + } + match spawn_player() { + Ok(tx) => { + let _ = tx.send(samples); + *player = Some(tx); + } + Err(error) => warn!("Scrub audio unavailable: {error}"), + } +} + +/// Starts the thread that owns the output stream (cpal streams are not Send). +fn spawn_player() -> Result>, String> { + let (tx, rx) = mpsc::channel::>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + std::thread::Builder::new() + .name("scrub-audio".into()) + .spawn(move || { + let queue = Arc::new(Mutex::new(ScrubBuffer::default())); + let stream = match build_stream(queue.clone()) { + Ok(stream) => { + let _ = ready_tx.send(Ok(())); + stream + } + Err(error) => { + let _ = ready_tx.send(Err(error)); + return; + } + }; + let mut playing = false; + loop { + match rx.recv_timeout(IDLE_PAUSE) { + Ok(slice) => { + *queue.lock().unwrap_or_else(|p| p.into_inner()) = ScrubBuffer { + samples: slice, + position: 0.0, + }; + if !playing { + playing = stream.play().is_ok(); + } + } + Err(RecvTimeoutError::Timeout) => { + if playing { + let _ = stream.pause(); + playing = false; + } + } + Err(RecvTimeoutError::Disconnected) => break, + } + } + }) + .map_err(|e| e.to_string())?; + ready_rx + .recv_timeout(Duration::from_secs(2)) + .map_err(|e| e.to_string())??; + Ok(tx) +} + +/// Builds an f32 output stream fed from `queue`, which holds interleaved +/// stereo at the renderer's sample rate; it is resampled (nearest frame) and +/// mapped to the device's channel count on the way out. +fn build_stream(queue: Arc>) -> Result { + let device = cpal::default_host() + .default_output_device() + .ok_or("No audio output device")?; + let supported = device.default_output_config().map_err(|e| e.to_string())?; + if supported.sample_format() != cpal::SampleFormat::F32 { + return Err(format!( + "Unsupported scrub output format {:?}", + supported.sample_format() + )); + } + let config = supported.config(); + let out_channels = usize::from(config.channels); + let step = f64::from(cap_editor::AudioRenderer::SAMPLE_RATE) / f64::from(config.sample_rate.0); + + let stream = device + .build_output_stream( + &config, + move |buffer: &mut [f32], _| { + let mut queue = queue.lock().unwrap_or_else(|p| p.into_inner()); + let ScrubBuffer { samples, position } = &mut *queue; + for frame in buffer.chunks_mut(out_channels) { + let index = *position as usize * 2; + let (Some(&left), Some(&right)) = (samples.get(index), samples.get(index + 1)) + else { + frame.fill(0.0); + continue; + }; + for (channel, sample) in frame.iter_mut().enumerate() { + *sample = match channel { + 0 => left, + 1 => right, + _ => 0.0, + }; + } + if out_channels == 1 { + frame[0] = (left + right) * 0.5; + } + *position += step; + } + }, + |error| warn!("Scrub audio stream error: {error}"), + None, + ) + .map_err(|e| e.to_string())?; + Ok(stream) +} diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index cecbe1db9ce..5409daa72b5 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -860,6 +860,34 @@ impl EditorInstance { (self.on_state_change)(&state); } + /// A short slice of the mixed project audio at `time` (output seconds), + /// interleaved stereo at [`crate::AudioRenderer::SAMPLE_RATE`], for audio + /// scrubbing. Edges are faded so back-to-back slices don't click. + pub async fn audio_snippet(&self, time: f64, secs: f64) -> Vec { + let segments = crate::get_audio_segments(&self.segment_medias).await; + let music = self.load_music_tracks().await; + let project = self.project_config.1.borrow().clone(); + let mut renderer = crate::AudioRenderer::new(segments).with_music(music); + renderer.set_playhead(time.max(0.0), &project); + let frames = + (secs.clamp(0.01, 0.5) * f64::from(crate::AudioRenderer::SAMPLE_RATE)) as usize; + let Some((_, mut samples)) = renderer.render_frame_raw(frames, &project) else { + return Vec::new(); + }; + let channels = usize::from(crate::AudioRenderer::CHANNELS); + let total = samples.len() / channels; + let fade = (crate::AudioRenderer::SAMPLE_RATE as usize / 200) + .min(total / 2) + .max(1); + for frame in 0..total { + let gain = (frame.min(total - 1 - frame) as f32 / fade as f32).min(1.0); + for sample in &mut samples[frame * channels..(frame + 1) * channels] { + *sample *= gain; + } + } + samples + } + /// Decodes (and caches) the music tracks referenced by the current project /// config off the async runtime so playback start isn't blocked by ffmpeg. async fn load_music_tracks(&self) -> crate::MusicTracks { From a637871267d6e824ad6aa6550c2d985da3120e52 Mon Sep 17 00:00:00 2001 From: Yoav Levavi Date: Tue, 27 Oct 2026 21:04:15 +0200 Subject: [PATCH 2/2] Hand scrub slices through preallocated rings; render them off the async workers Co-Authored-By: Claude Opus 5.5 --- crates/editor/src/audio_output.rs | 65 ++++++++++++++++++++-------- crates/editor/src/editor_instance.rs | 42 ++++++++++-------- 2 files changed, 71 insertions(+), 36 deletions(-) diff --git a/crates/editor/src/audio_output.rs b/crates/editor/src/audio_output.rs index 889d8c07e97..491daa76b69 100644 --- a/crates/editor/src/audio_output.rs +++ b/crates/editor/src/audio_output.rs @@ -30,6 +30,10 @@ use cpal::{ SampleFormat, traits::{DeviceTrait, HostTrait, StreamTrait}, }; +use ringbuf::{ + HeapProd, HeapRb, + traits::{Consumer, Observer, Producer, Split}, +}; use tokio::sync::watch; use tracing::{error, info}; @@ -1574,9 +1578,10 @@ where let mut active: Option> = None; // Scrub slices arrive already in the device's format; played-out buffers - // go back to the output thread so the callback never frees memory. - let (snippet_tx, snippet_rx) = std_mpsc::channel::>(); - let (spent_tx, spent_rx) = std_mpsc::channel::>(); + // go back to the output thread so the callback never frees memory. Both + // rings are preallocated, so neither side allocates to hand one over. + let (snippet_tx, mut snippet_rx) = HeapRb::>::new(SCRUB_QUEUE).split(); + let (mut spent_tx, spent_rx) = HeapRb::>::new(SCRUB_QUEUE * 2).split(); let mut snippet: Option<(Vec, usize)> = None; let callback_source_active = source_active.clone(); let stream = device @@ -1584,15 +1589,14 @@ where &config, move |buffer: &mut [T], info| { drain_source_commands(&mut active, &source_rx); - while let Ok(next) = snippet_rx.try_recv() { - if let Some((spent, _)) = snippet.replace((next, 0)) { - let _ = spent_tx.send(spent); - } - } - if active.is_some() - && let Some((spent, _)) = snippet.take() + while spent_tx.vacant_len() > usize::from(snippet.is_some()) + && let Some(next) = snippet_rx.try_pop() { - let _ = spent_tx.send(spent); + retire_snippet(&mut spent_tx, &mut snippet); + snippet = Some((next, 0)); + } + if active.is_some() { + retire_snippet(&mut spent_tx, &mut snippet); } callback_source_active .store(active.is_some() || snippet.is_some(), Ordering::Release); @@ -1606,10 +1610,8 @@ where buffer[..count].copy_from_slice(&samples[*position..*position + count]); buffer[count..].fill(T::EQUILIBRIUM); *position += count; - if *position >= samples.len() - && let Some((spent, _)) = snippet.take() - { - let _ = spent_tx.send(spent); + if *position >= samples.len() { + retire_snippet(&mut spent_tx, &mut snippet); } return; }; @@ -1674,9 +1676,11 @@ where let _ = source_tx.send(SourceCommand::Remove { generation }); }); + let spent_rx = std::cell::RefCell::new(spent_rx); + let snippet_tx = std::cell::RefCell::new(snippet_tx); let play_snippet = Box::new(move |samples: Vec| { - while spent_rx.try_recv().is_ok() {} - let _ = snippet_tx.send(convert_snippet::( + spent_rx.borrow_mut().clear(); + let _ = snippet_tx.borrow_mut().try_push(convert_snippet::( &samples, crate::AudioRenderer::SAMPLE_RATE, output_info.sample_rate, @@ -1694,6 +1698,20 @@ where }) } +/// Hands a scrub slice back to the output thread. It is only let go when the +/// spent ring has room, otherwise it stays (finished, so silent) until there +/// is, so the audio callback never frees it. +fn retire_snippet(spent: &mut HeapProd>, slot: &mut Option<(Vec, usize)>) { + if !spent.is_full() + && let Some((samples, _)) = slot.take() + { + let _ = spent.try_push(samples); + } +} + +/// Scrub slices waiting for the callback; a drag sends one every ~100 ms. +const SCRUB_QUEUE: usize = 4; + /// How long scrub slices are dropped after the stream fails to open. const SCRUB_RETRY_AFTER_FAILURE: Duration = Duration::from_secs(2); @@ -1745,7 +1763,18 @@ mod native_tests; #[cfg(test)] mod scrub_snippet_tests { - use super::convert_snippet; + use super::{HeapRb, Split, convert_snippet, retire_snippet}; + + #[test] + fn a_full_spent_ring_keeps_the_slice_instead_of_freeing_it() { + let (mut spent, _rx) = HeapRb::>::new(1).split(); + let mut slot = Some((vec![1.0], 1)); + retire_snippet(&mut spent, &mut slot); + assert!(slot.is_none()); + let mut slot = Some((vec![2.0], 1)); + retire_snippet(&mut spent, &mut slot); + assert_eq!(slot, Some((vec![2.0], 1))); + } #[test] fn snippets_are_resampled_and_mapped_to_the_device_layout() { diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 4cdbc74d5e1..7f505ccb730 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -878,25 +878,31 @@ impl EditorInstance { let segments = crate::get_audio_segments(&self.segment_medias).await; let music = self.load_music_tracks().await; let project = self.project_config.1.borrow().clone(); - let mut renderer = crate::AudioRenderer::new(segments).with_music(music); - renderer.set_playhead(time.max(0.0), &project); - let frames = - (secs.clamp(0.01, 0.5) * f64::from(crate::AudioRenderer::SAMPLE_RATE)) as usize; - let Some((_, mut samples)) = renderer.render_frame_raw(frames, &project) else { - return Vec::new(); - }; - let channels = usize::from(crate::AudioRenderer::CHANNELS); - let total = samples.len() / channels; - let fade = (crate::AudioRenderer::SAMPLE_RATE as usize / 200) - .min(total / 2) - .max(1); - for frame in 0..total { - let gain = (frame.min(total - 1 - frame) as f32 / fade as f32).min(1.0); - for sample in &mut samples[frame * channels..(frame + 1) * channels] { - *sample *= gain; + // Rendering speed-changed clips builds audio filters, so it runs off + // the async workers. + tokio::task::spawn_blocking(move || { + let mut renderer = crate::AudioRenderer::new(segments).with_music(music); + renderer.set_playhead(time.max(0.0), &project); + let frames = + (secs.clamp(0.01, 0.5) * f64::from(crate::AudioRenderer::SAMPLE_RATE)) as usize; + let Some((_, mut samples)) = renderer.render_frame_raw(frames, &project) else { + return Vec::new(); + }; + let channels = usize::from(crate::AudioRenderer::CHANNELS); + let total = samples.len() / channels; + let fade = (crate::AudioRenderer::SAMPLE_RATE as usize / 200) + .min(total / 2) + .max(1); + for frame in 0..total { + let gain = (frame.min(total - 1 - frame) as f32 / fade as f32).min(1.0); + for sample in &mut samples[frame * channels..(frame + 1) * channels] { + *sample *= gain; + } } - } - samples + samples + }) + .await + .unwrap_or_default() } /// Decodes (and caches) the music tracks referenced by the current project