diff --git a/apps/desktop-gpui/src/assets.rs b/apps/desktop-gpui/src/assets.rs index 638020d001e..e21b5b9d7da 100644 --- a/apps/desktop-gpui/src/assets.rs +++ b/apps/desktop-gpui/src/assets.rs @@ -167,6 +167,7 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "ratio.svg", "refresh-cw.svg", "rotate-ccw.svg", + "volume-2.svg", "volume-x.svg", "gem.svg", "sliders-horizontal.svg", diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index ecbc4b4254b..97572a39231 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -203,6 +203,8 @@ impl EditorVerticalLayout { } } +/// Length of each audio-scrub slice; the next one replaces what is left. +const SCRUB_SLICE_SECS: f64 = 0.09; const PRESETS_MENU_WIDTH: f32 = 288.; const PRESET_SUBMENU_WIDTH: f32 = 208.; const PRESET_SUBMENU_HEIGHT: f32 = 5. * 32. + 4. * 2. + 8.; @@ -1367,6 +1369,12 @@ pub struct EditorWindow { /// Kept alive for the window's lifetime: dropping the last `Arc` is what /// tears the decoders down, and `dispose()` on close does it explicitly. pub(crate) instance: Option>, + /// Audio scrub: a slice request is in flight, and the newest position + /// asked for meanwhile (played once it lands). + /// The toolbar's speaker toggle ([`crate::store::audio_scrub_enabled`]). + audio_scrub: bool, + scrub_audio_busy: bool, + scrub_audio_next: Option, focus: FocusHandle, // -- Transport ---------------------------------------------------------- @@ -2070,6 +2078,9 @@ impl EditorWindow { total: 0.0, transport: None, scrub: None, + audio_scrub: crate::store::audio_scrub_enabled(), + scrub_audio_busy: false, + scrub_audio_next: None, stats: None, play_mark: None, timeline: TimelineModel::default(), @@ -3989,6 +4000,7 @@ impl EditorWindow { // `applyScrub()` runs immediately on mousedown. self.scrub = Some(Scrub::Ruler); self.seek_to_time(time, cx); + self.scrub_audio_at(time, cx); } else { self.scrub = Some(Scrub::Press { time }); } @@ -4017,6 +4029,41 @@ impl EditorWindow { // in-flight work when it sees a newer instruction // (`editor_instance.rs:563-575`). self.scrub_to_time(time, cx); + self.scrub_audio_at(time, cx); + } + + /// Audio scrub: while the ruler is dragged with playback stopped, play a + /// short slice of the mix under the playhead. One request at a time; the + /// newest position waits and plays next, spaced by about a slice so they + /// run back to back. + fn scrub_audio_at(&mut self, time: f64, cx: &mut Context) { + if self.playing || !self.audio_scrub { + return; + } + if self.scrub_audio_busy { + self.scrub_audio_next = Some(time); + return; + } + let Some(instance) = self.instance.clone() else { + return; + }; + self.scrub_audio_busy = true; + let slice = gpui_tokio::Tokio::spawn(cx, async move { + instance.scrub_audio(time, SCRUB_SLICE_SECS).await; + }); + cx.spawn(async move |this, cx| { + let _ = slice.await; + cx.background_executor() + .timer(Duration::from_millis(60)) + .await; + let _ = this.update(cx, |this, cx| { + this.scrub_audio_busy = false; + if let Some(next) = this.scrub_audio_next.take() { + this.scrub_audio_at(next, cx); + } + }); + }) + .detach(); } /// The **timeline's own** mouseup, the one the source registers on @@ -10538,6 +10585,31 @@ impl EditorWindow { )), ), ) + .child( + ui::EditorButton::plain(&theme, "transport-audio-scrub") + .left_icon(if self.audio_scrub { + "icons/volume-2.svg" + } else { + "icons/volume-x.svg" + }) + .tooltip( + &theme, + if self.audio_scrub { + "Audio while scrubbing: on" + } else { + "Audio while scrubbing: off" + }, + ) + .pressed(self.audio_scrub) + .on_click(cx.listener(|this, _, window, cx| { + this.audio_scrub = !this.audio_scrub; + if !crate::store::set_audio_scrub_enabled(this.audio_scrub) { + tracing::warn!("could not save the audio scrub preference"); + } + cx.notify(); + window.refresh(); + })), + ) .child(self.header_divider()) // `IconCapZoomOut` -> `updateZoom(zoom * 1.1, // playbackTime)`; `IconCapZoomIn` -> `zoom / 1.1` diff --git a/apps/desktop-gpui/src/store.rs b/apps/desktop-gpui/src/store.rs index 01fa3a5088e..f7f8b306d2d 100644 --- a/apps/desktop-gpui/src/store.rs +++ b/apps/desktop-gpui/src/store.rs @@ -1096,6 +1096,20 @@ pub fn set_studio_sound_defaults(enabled: bool, isolation: cap_project::VoiceIso section.insert("isolation".into(), serde_json::to_value(isolation).unwrap()); set_store_value(AUDIO_ENHANCEMENT, Value::Object(section)) } +/// Editor preferences that belong to the user rather than a project. +pub const EDITOR_PREFERENCES: &str = "editor_preferences"; +pub const AUDIO_SCRUB_KEY: &str = "audioScrub"; + +/// Whether dragging the playhead on the ruler plays the audio under it. +/// Defaults on; the timeline toolbar's speaker toggle writes it. +pub fn audio_scrub_enabled() -> bool { + bool_at(&store_section(EDITOR_PREFERENCES), AUDIO_SCRUB_KEY, true) +} + +pub fn set_audio_scrub_enabled(enabled: bool) -> bool { + set_store_setting(EDITOR_PREFERENCES, AUDIO_SCRUB_KEY, Value::Bool(enabled)) +} + /// `RecordingSettingsStore::KEY` (`src-tauri/src/recording_settings.rs:37`) -- /// the section the tray's Select Mode submenu reads and writes. pub const RECORDING_SETTINGS: &str = "recording_settings"; diff --git a/crates/editor/src/audio_output.rs b/crates/editor/src/audio_output.rs index 3de0d2f2be0..491daa76b69 100644 --- a/crates/editor/src/audio_output.rs +++ b/crates/editor/src/audio_output.rs @@ -30,6 +30,10 @@ use cpal::{ SampleFormat, traits::{DeviceTrait, HostTrait, StreamTrait}, }; +use ringbuf::{ + HeapProd, HeapRb, + traits::{Consumer, Observer, Producer, Split}, +}; use tokio::sync::watch; use tracing::{error, info}; @@ -95,6 +99,12 @@ enum ControlMsg { StopPlayback { generation: u64, }, + /// A short scrub slice (interleaved stereo at + /// [`crate::AudioRenderer::SAMPLE_RATE`]) to play while playback is + /// stopped, replacing whatever is left of the previous one. + Scrub { + samples: Vec, + }, Shutdown, } @@ -516,6 +526,18 @@ impl AudioOutput { .send(ControlMsg::StopPlayback { generation }); } + /// Plays a short scrub slice (interleaved stereo at + /// [`crate::AudioRenderer::SAMPLE_RATE`]) on the shared stream, replacing + /// whatever is left of the previous slice. Ignored while playback owns the + /// stream. Non-blocking: the slice is converted to the device's format on + /// the output thread. + pub fn play_snippet(&self, samples: Vec) { + if samples.is_empty() { + return; + } + let _ = self.control_tx.send(ControlMsg::Scrub { samples }); + } + /// Tears down the stream and thread. Also happens on drop. pub fn shutdown(&self) { let _ = self.control_tx.send(ControlMsg::Shutdown); @@ -596,6 +618,7 @@ struct TypedStreamHandle { install: Box, u64, SourceAcknowledgement) -> Result<(), String>>, install_progressive: Box, remove: Box)>, + play_snippet: Box)>, } struct StreamState { @@ -659,6 +682,9 @@ fn control_thread(control_rx: std_mpsc::Receiver) { let mut state: Option = None; let mut idle = IdleTimer::new(Instant::now()); let mut awake_holds = 0usize; + // After a stream fails to open for a scrub slice, slices are dropped + // until this passes instead of reopening the device every ~100 ms. + let mut scrub_retry_at: Option = None; loop { let msg = if state.as_ref().is_some_and(StreamState::idle_pausable) { @@ -770,6 +796,19 @@ fn control_thread(control_rx: std_mpsc::Receiver) { |stream, generation| (stream.handle.remove)(Some(generation)), ); } + ControlMsg::Scrub { samples } => { + let now = Instant::now(); + if scrub_retry_at.is_none_or(|retry_at| now >= retry_at) { + if ensure_stream(&mut state) { + scrub_retry_at = None; + if let Some(stream) = &state { + (stream.handle.play_snippet)(samples); + } + } else { + scrub_retry_at = Some(now + SCRUB_RETRY_AFTER_FAILURE); + } + } + } ControlMsg::Retire(source) => drop(source), ControlMsg::Realign(realign) => realign(), ControlMsg::Shutdown => break, @@ -1275,6 +1314,7 @@ fn control_thread_headless(control_rx: std_mpsc::Receiver, mut tap: generation: Some(generation), }); } + ControlMsg::Scrub { .. } => {} ControlMsg::Retire(source) => drop(source), ControlMsg::Realign(realign) => realign(), ControlMsg::Shutdown => break, @@ -1408,6 +1448,7 @@ fn control_thread_offline( }); } ControlMsg::EnsureStream | ControlMsg::HoldAwake | ControlMsg::ReleaseAwake => {} + ControlMsg::Scrub { .. } => {} ControlMsg::Retire(source) => drop(source), ControlMsg::Realign(realign) => realign(), ControlMsg::Shutdown => break, @@ -1536,16 +1577,42 @@ where let source_active = Arc::new(AtomicBool::new(false)); let mut active: Option> = None; + // Scrub slices arrive already in the device's format; played-out buffers + // go back to the output thread so the callback never frees memory. Both + // rings are preallocated, so neither side allocates to hand one over. + let (snippet_tx, mut snippet_rx) = HeapRb::>::new(SCRUB_QUEUE).split(); + let (mut spent_tx, spent_rx) = HeapRb::>::new(SCRUB_QUEUE * 2).split(); + let mut snippet: Option<(Vec, usize)> = None; let callback_source_active = source_active.clone(); let stream = device .build_output_stream( &config, move |buffer: &mut [T], info| { drain_source_commands(&mut active, &source_rx); - callback_source_active.store(active.is_some(), Ordering::Release); + while spent_tx.vacant_len() > usize::from(snippet.is_some()) + && let Some(next) = snippet_rx.try_pop() + { + retire_snippet(&mut spent_tx, &mut snippet); + snippet = Some((next, 0)); + } + if active.is_some() { + retire_snippet(&mut spent_tx, &mut snippet); + } + callback_source_active + .store(active.is_some() || snippet.is_some(), Ordering::Release); let Some(source) = active.as_mut() else { - buffer.fill(T::EQUILIBRIUM); + let Some((samples, position)) = snippet.as_mut() else { + buffer.fill(T::EQUILIBRIUM); + return; + }; + let count = (samples.len() - *position).min(buffer.len()); + buffer[..count].copy_from_slice(&samples[*position..*position + count]); + buffer[count..].fill(T::EQUILIBRIUM); + *position += count; + if *position >= samples.len() { + retire_snippet(&mut spent_tx, &mut snippet); + } return; }; @@ -1609,18 +1676,136 @@ where let _ = source_tx.send(SourceCommand::Remove { generation }); }); + let spent_rx = std::cell::RefCell::new(spent_rx); + let snippet_tx = std::cell::RefCell::new(snippet_tx); + let play_snippet = Box::new(move |samples: Vec| { + spent_rx.borrow_mut().clear(); + let _ = snippet_tx.borrow_mut().try_push(convert_snippet::( + &samples, + crate::AudioRenderer::SAMPLE_RATE, + output_info.sample_rate, + output_info.channels, + )); + }); + Ok(TypedStreamHandle { stream, source_active, install, install_progressive, remove, + play_snippet, }) } +/// Hands a scrub slice back to the output thread. It is only let go when the +/// spent ring has room, otherwise it stays (finished, so silent) until there +/// is, so the audio callback never frees it. +fn retire_snippet(spent: &mut HeapProd>, slot: &mut Option<(Vec, usize)>) { + if !spent.is_full() + && let Some((samples, _)) = slot.take() + { + let _ = spent.try_push(samples); + } +} + +/// Scrub slices waiting for the callback; a drag sends one every ~100 ms. +const SCRUB_QUEUE: usize = 4; + +/// How long scrub slices are dropped after the stream fails to open. +const SCRUB_RETRY_AFTER_FAILURE: Duration = Duration::from_secs(2); + +/// Converts an interleaved stereo slice at `source_rate` to the device's +/// rate (linear interpolation), channel count (left/right on the first two +/// channels, their mix on a mono device, silence on any others) and sample +/// type. +fn convert_snippet>( + samples: &[f32], + source_rate: u32, + output_rate: u32, + output_channels: usize, +) -> Vec { + let output_channels = output_channels.max(1); + let source_frames = samples.len() / 2; + if source_frames == 0 || source_rate == 0 || output_rate == 0 { + return Vec::new(); + } + let output_frames = + (source_frames as u64 * u64::from(output_rate) / u64::from(source_rate)).max(1) as usize; + let step = f64::from(source_rate) / f64::from(output_rate); + let frame = |index: usize| { + let index = index.min(source_frames - 1); + (samples[index * 2], samples[index * 2 + 1]) + }; + let mut out = Vec::with_capacity(output_frames * output_channels); + for n in 0..output_frames { + let position = n as f64 * step; + let base = position.floor() as usize; + let t = (position - base as f64) as f32; + let (l0, r0) = frame(base); + let (l1, r1) = frame(base + 1); + let (left, right) = (l0 + (l1 - l0) * t, r0 + (r1 - r0) * t); + for channel in 0..output_channels { + let value = match (output_channels, channel) { + (1, _) => (left + right) * 0.5, + (_, 0) => left, + (_, 1) => right, + _ => 0.0, + }; + out.push(T::from_sample_(value)); + } + } + out +} + #[cfg(test)] mod native_tests; +#[cfg(test)] +mod scrub_snippet_tests { + use super::{HeapRb, Split, convert_snippet, retire_snippet}; + + #[test] + fn a_full_spent_ring_keeps_the_slice_instead_of_freeing_it() { + let (mut spent, _rx) = HeapRb::>::new(1).split(); + let mut slot = Some((vec![1.0], 1)); + retire_snippet(&mut spent, &mut slot); + assert!(slot.is_none()); + let mut slot = Some((vec![2.0], 1)); + retire_snippet(&mut spent, &mut slot); + assert_eq!(slot, Some((vec![2.0], 1))); + } + + #[test] + fn snippets_are_resampled_and_mapped_to_the_device_layout() { + let stereo: Vec = (0..480).flat_map(|i| [i as f32, -(i as f32)]).collect(); + let same: Vec = convert_snippet(&stereo, 48_000, 48_000, 2); + assert_eq!(same, stereo); + + let down: Vec = convert_snippet(&stereo, 48_000, 24_000, 2); + assert_eq!(down.len(), 480); + assert_eq!(&down[..4], &[0.0, 0.0, 2.0, -2.0]); + + let mono: Vec = convert_snippet(&stereo, 48_000, 48_000, 1); + assert_eq!(mono.len(), 480); + assert!( + mono.iter().all(|sample| *sample == 0.0), + "L and -L mix to silence" + ); + + let surround: Vec = convert_snippet(&stereo, 48_000, 44_100, 6); + assert_eq!(surround.len(), 441 * 6); + assert!( + surround + .chunks(6) + .all(|frame| frame[2..].iter().all(|s| *s == 0.0)) + ); + + let ints: Vec = convert_snippet(&[1.0, -1.0], 48_000, 48_000, 2); + assert_eq!(ints, vec![i16::MAX, i16::MIN]); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index c3c44bec553..7f505ccb730 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -863,6 +863,48 @@ impl EditorInstance { (self.on_state_change)(&state); } + /// Audio scrubbing: plays `secs` of the mixed project audio at `time` + /// (output seconds) on this editor's own audio output, so it follows + /// device changes and closes with the editor. Ignored while playing. + pub async fn scrub_audio(&self, time: f64, secs: f64) { + let samples = self.audio_snippet(time, secs).await; + self.audio_output.play_snippet(samples); + } + + /// A short slice of the mixed project audio at `time` (output seconds), + /// interleaved stereo at [`crate::AudioRenderer::SAMPLE_RATE`]. Edges are + /// faded so back-to-back slices don't click. + async fn audio_snippet(&self, time: f64, secs: f64) -> Vec { + let segments = crate::get_audio_segments(&self.segment_medias).await; + let music = self.load_music_tracks().await; + let project = self.project_config.1.borrow().clone(); + // Rendering speed-changed clips builds audio filters, so it runs off + // the async workers. + tokio::task::spawn_blocking(move || { + let mut renderer = crate::AudioRenderer::new(segments).with_music(music); + renderer.set_playhead(time.max(0.0), &project); + let frames = + (secs.clamp(0.01, 0.5) * f64::from(crate::AudioRenderer::SAMPLE_RATE)) as usize; + let Some((_, mut samples)) = renderer.render_frame_raw(frames, &project) else { + return Vec::new(); + }; + let channels = usize::from(crate::AudioRenderer::CHANNELS); + let total = samples.len() / channels; + let fade = (crate::AudioRenderer::SAMPLE_RATE as usize / 200) + .min(total / 2) + .max(1); + for frame in 0..total { + let gain = (frame.min(total - 1 - frame) as f32 / fade as f32).min(1.0); + for sample in &mut samples[frame * channels..(frame + 1) * channels] { + *sample *= gain; + } + } + samples + }) + .await + .unwrap_or_default() + } + /// Decodes (and caches) the music tracks referenced by the current project /// config off the async runtime so playback start isn't blocked by ffmpeg. async fn load_music_tracks(&self) -> crate::MusicTracks {