diff --git a/crates/mix/src/lib.rs b/crates/mix/src/lib.rs index 74238609..0d84d438 100644 --- a/crates/mix/src/lib.rs +++ b/crates/mix/src/lib.rs @@ -45,6 +45,53 @@ struct Strip { /// Per-clip effect chains (Clip Effects) and a scratch buffer to render clips into. clip_fx: HashMap, clipbuf: Vec>, + /// Total latency of the active inserts (samples). + latency: usize, + /// Delay compensation: align this strip with the slowest one (post- and pre-fader paths), + /// plus extra delay for direct-to-main outputs so they line up with aux returns. + align: DelayLine, + align_pre: DelayLine, + out_extra: DelayLine, + outbuf: Vec>, +} + +/// A multichannel delay line (samples), resized only when the delay changes. +#[derive(Default)] +struct DelayLine { + bufs: Vec>, + idx: usize, + len: usize, +} + +impl DelayLine { + fn set(&mut self, len: usize, ch: usize) { + let len = len.min(1 << 20); + if len != self.len || self.bufs.len() != ch { + self.len = len; + self.idx = 0; + self.bufs = vec![vec![0.0; len]; ch]; + } + } + fn process(&mut self, io: &mut [Vec], frames: usize) { + if self.len == 0 { + return; + } + let start = self.idx; + for (c, ch) in io.iter_mut().enumerate() { + let Some(ring) = self.bufs.get_mut(c) else { continue }; + let mut i = start; + for x in ch.iter_mut().take(frames) { + if let Some(r) = ring.get_mut(i) { + std::mem::swap(r, x); + } + i += 1; + if i >= self.len { + i = 0; + } + } + } + self.idx = (start + frames) % self.len.max(1); + } } /// Clip Effects: an EQ and a compressor per clip, configured from `edit.values` @@ -70,6 +117,11 @@ impl Strip { gr: 0.0, clip_fx: HashMap::new(), clipbuf: Vec::new(), + latency: 0, + align: DelayLine::default(), + align_pre: DelayLine::default(), + out_extra: DelayLine::default(), + outbuf: Vec::new(), } } } @@ -89,6 +141,14 @@ pub struct MixEngine { /// Snapshot key of the session the strips/order were synced to. synced: (usize, usize, usize), order: Vec, + /// Current total compensation delay (samples) of the mix. + pub latency: usize, + /// Live input for the next block (planar), set by the audio host for input monitoring. + pub input: Vec>, + /// Transport stopped: only monitored inputs sound (no clips). + pub monitor_only: bool, + /// Recording: record-armed tracks hear their input (auto input). + pub recording: bool, } impl MixEngine { @@ -106,6 +166,10 @@ impl MixEngine { last_end: 0, synced: (0, 0, 0), order: Vec::new(), + latency: 0, + input: Vec::new(), + monitor_only: false, + recording: false, } } @@ -216,22 +280,49 @@ impl MixEngine { } } let busses = std::mem::take(&mut self.busses); - run_strips(&mut work, |(t, st)| process_strip(s, t, st, &busses, pos, frames, any_solo)); + let live_in = LiveInput { input: &self.input, monitor_only: self.monitor_only, recording: self.recording }; + run_strips(&mut work, |(t, st)| process_strip(s, t, st, &busses, &live_in, pos, frames, any_solo)); self.busses = busses; - for (t, st) in &work { + // Delay compensation: align every strip to the slowest; direct-to-main outputs also wait + // for the slowest aux return (aux latencies are from the previous block; they are stable). + let pdc = s.edit.delay_compensation; + let lmax_t = if pdc { work.iter().map(|(_, st)| st.latency).max().unwrap_or(0) } else { 0 }; + let lmax_a = if pdc { + self.strips.iter().filter(|(id, _)| s.track(**id).is_some_and(|t| is_aux(t))).map(|(_, st)| st.latency).max().unwrap_or(0) + } else { + 0 + }; + self.latency = lmax_t + lmax_a; + for (t, st) in work.iter_mut() { + let ch = st.buf.len(); + st.align.set(lmax_t.saturating_sub(st.latency), ch); + st.align.process(&mut st.buf, frames); + if t.mixer.sends.iter().flatten().any(|x| x.pre_fader) { + st.align_pre.set(lmax_t.saturating_sub(st.latency), ch); + st.align_pre.process(&mut st.pre, frames); + } + st.out_extra.set(if t.mixer.output == Route::Main { lmax_a } else { 0 }, ch); + } + for (t, st) in work.iter_mut() { self.route_strip(t, st, pos, frames); } for (t, st) in work { self.strips.insert(t.id, st); } + let is_aux = |t: &Track| matches!(t.kind, TrackKind::Aux | TrackKind::Folder); // Phase 2: auxes in dependency order (they read busses). for t in order.iter().filter_map(|&i| s.tracks.get(i)).filter(live).filter(|t| is_aux(t)) { let Some(mut st) = self.strips.remove(&t.id) else { continue }; if st.scratch.len() < self.max_block { st.scratch.resize(self.max_block, 0.0); } - process_strip(s, t, &mut st, &self.busses, pos, frames, any_solo); - self.route_strip(t, &st, pos, frames); + let live_in = LiveInput { input: &self.input, monitor_only: self.monitor_only, recording: self.recording }; + process_strip(s, t, &mut st, &self.busses, &live_in, pos, frames, any_solo); + let ch = st.buf.len(); + st.align.set(if pdc { lmax_a.saturating_sub(st.latency) } else { 0 }, ch); + st.align.process(&mut st.buf, frames); + st.out_extra.set(0, ch); + self.route_strip(t, &mut st, pos, frames); self.strips.insert(t.id, st); } self.order = order; @@ -288,7 +379,7 @@ impl MixEngine { } /// Send, meter and pan a processed strip into the busses / main mix. - fn route_strip(&mut self, t: &Track, strip: &Strip, pos: Samples, frames: usize) { + fn route_strip(&mut self, t: &Track, strip: &mut Strip, pos: Samples, frames: usize) { let mut pk = [0.0f32; 2]; for (c, p) in pk.iter_mut().enumerate() { let ch = c.min(strip.buf.len().saturating_sub(1)); @@ -306,6 +397,17 @@ impl MixEngine { } let pans: [f32; 2] = [pan_at(t, 0, pos), pan_at(t, 1, pos)]; match &t.mixer.output { + Route::Main if strip.out_extra.len > 0 => { + if strip.outbuf.len() != strip.buf.len() { + strip.outbuf = vec![Vec::new(); strip.buf.len()]; + } + for (d, src) in strip.outbuf.iter_mut().zip(strip.buf.iter()) { + d.clear(); + d.extend_from_slice(src.get(..frames).unwrap_or(&[])); + } + strip.out_extra.process(&mut strip.outbuf, frames); + pan_into(&strip.outbuf, frames, &mut self.main, &pans, self.pan_law); + } Route::Main => pan_into(&strip.buf, frames, &mut self.main, &pans, self.pan_law), Route::Bus(b) => { if let Some(dst) = self.busses.get_mut(b) { @@ -319,11 +421,46 @@ impl MixEngine { /// Source → trim → inserts → (pre-fader copy) → mute/fader for one strip. Independent of other /// strips except that aux inputs read `busses`, so non-aux strips can run in parallel. -fn process_strip(s: &Session, t: &Track, strip: &mut Strip, busses: &HashMap>>, pos: Samples, frames: usize, any_solo: bool) { +/// Live input routed to monitored tracks. +struct LiveInput<'a> { + input: &'a [Vec], + monitor_only: bool, + recording: bool, +} + +fn process_strip( + s: &Session, + t: &Track, + strip: &mut Strip, + busses: &HashMap>>, + live: &LiveInput<'_>, + pos: Samples, + frames: usize, + any_solo: bool, +) { for c in strip.buf.iter_mut() { c.iter_mut().take(frames).for_each(|x| *x = 0.0); } match t.kind { + TrackKind::Audio if !live.input.is_empty() && (t.mixer.input_monitor || (t.mixer.record_arm && (live.monitor_only || live.recording))) => { + // Input monitoring: the track hears its hardware input instead of its clips. + let first = match &t.mixer.input { + Route::Hardware(h) => { + h.trim_start_matches("In ").split(['-', ' ']).next().and_then(|n| n.parse::().ok()).map_or(0, |n| n.saturating_sub(1)) + } + _ => 0, + }; + let n_in = live.input.len().max(1); + for (k, dst) in strip.buf.iter_mut().enumerate() { + if let Some(src) = live.input.get((first + k) % n_in) { + for (d, x) in dst.iter_mut().zip(src.iter()).take(frames) { + *d = *x; + } + } + } + } + TrackKind::Audio if live.monitor_only => {} + TrackKind::Instrument | TrackKind::Midi if live.monitor_only => {} TrackKind::Audio => { let block = Range { start: pos, end: pos.saturating_add(frames as i64) }; for clip in t.clips() { @@ -400,6 +537,7 @@ fn process_strip(s: &Session, t: &Track, strip: &mut Strip, busses: &HashMap= 0.5); let soloed_out = any_solo && !t.mixer.solo && !t.mixer.solo_safe && !receives_solo(s, t); strip.muted = t.mixer.mute || auto_mute || soloed_out; @@ -864,16 +1004,26 @@ fn mix_to_bus( } } -/// Offline render of the main mix over `range` (stereo, planar). +/// Offline render of the main mix over `range` (stereo, planar). The mix's delay-compensation +/// latency is removed, so the result lines up with the timeline. pub fn render_range(s: &Session, range: Range, block: usize) -> Vec> { let len = usize::try_from(range.len().max(0)).unwrap_or(0); - let mut out = vec![vec![0.0f32; len]; MAIN_CHANNELS]; + // Probe the latency with one block, then render `len + latency` and drop the head. + let latency = { + let mut probe = MixEngine::new(s.sample_rate.as_f64() as f32, block); + let b = probe.max_block(); + let mut tmp = vec![vec![0.0f32; b]; MAIN_CHANNELS]; + probe.render(s, range.start, b.min(len.max(1)), &mut tmp); + probe.latency + }; + let total = len.saturating_add(latency); + let mut out = vec![vec![0.0f32; total]; MAIN_CHANNELS]; let mut eng = MixEngine::new(s.sample_rate.as_f64() as f32, block); let b = eng.max_block(); let mut tmp = vec![vec![0.0f32; b]; MAIN_CHANNELS]; let mut done = 0usize; - while done < len { - let n = (len - done).min(b); + while done < total { + let n = (total - done).min(b); eng.render(s, range.start + done as i64, n, &mut tmp); for (o, t) in out.iter_mut().zip(tmp.iter()) { if let (Some(d), Some(sr)) = (o.get_mut(done..done + n), t.get(..n)) { @@ -882,6 +1032,11 @@ pub fn render_range(s: &Session, range: Range, block: usize) -> Vec> { } done += n; } + if latency > 0 { + for c in &mut out { + c.drain(..latency.min(c.len())); + } + } out } @@ -1081,6 +1236,55 @@ mod tests { assert!(out[0][2000] < 0.2, "{}", out[0][2000]); } + #[test] + fn delay_compensation_aligns_latent_tracks() { + let mut s = Session::default(); + let mut imp = vec![0.0f32; 48_000]; + imp[1000] = 1.0; + s.pool.insert(SourceId(900), Arc::new(SourceAudio::new(AudioBuffer { sample_rate: 48_000, channels: vec![imp] }))); + let mut ids = Vec::new(); + for (pan, fx) in [(-1.0f32, true), (1.0, false)] { + let t = s.add_track(TrackKind::Audio, ChannelFormat::Mono, None); + let cid = s.new_clip_id(); + let tr = s.track_mut(t).unwrap(); + tr.playlist_mut().unwrap().clips.push(Clip::audio(cid, "i", SourceId(900), 0, 0, 48_000)); + tr.mixer.pan = vec![pan]; + if fx { + tr.mixer.inserts[0] = Some(Insert::new("maximizer")); + } + ids.push(t); + } + let argmax = |c: &[f32]| c.iter().enumerate().max_by(|a, b| a.1.abs().total_cmp(&b.1.abs())).map(|(i, _)| i).unwrap(); + let lat = soundcraft_dsp::create("maximizer").unwrap().latency(); + assert!(lat > 0, "test needs a latent plugin"); + let out = render_range(&s, Range::new(0, 8000), 512); + assert_eq!(argmax(&out[0]), argmax(&out[1]), "PDC should align L and R"); + s.edit.delay_compensation = false; + let out = render_range(&s, Range::new(0, 8000), 512); + assert_ne!(argmax(&out[0]), argmax(&out[1])); + } + + #[test] + fn input_monitoring_replaces_clips() { + let (mut s, t) = session_with_dc(0.5, 4800); + let mut eng = MixEngine::new(48_000.0, 512); + eng.input = vec![vec![0.25; 512]]; + let mut out = vec![vec![0.0; 512]; 2]; + // Not monitored: the clip plays. + eng.render(&s, 0, 512, &mut out); + assert!((out[0][100] - 0.5 * std::f32::consts::FRAC_1_SQRT_2).abs() < 1e-3); + // Monitored: the input replaces it, and it still sounds when stopped. + s.track_mut(t).unwrap().mixer.input_monitor = true; + eng.render(&s, 512, 512, &mut out); + assert!((out[0][100] - 0.25 * std::f32::consts::FRAC_1_SQRT_2).abs() < 1e-3, "{}", out[0][100]); + eng.monitor_only = true; + eng.render(&s, 1024, 512, &mut out); + assert!(out[0][100] > 0.1); + s.track_mut(t).unwrap().mixer.input_monitor = false; + eng.render(&s, 1024, 512, &mut out); + assert!(out[0][100].abs() < 1e-6, "stopped, unmonitored: silence"); + } + #[test] fn empty_session_is_silent() { let s = Session::default(); diff --git a/crates/playback/src/lib.rs b/crates/playback/src/lib.rs index 32e3b443..466ae231 100644 --- a/crates/playback/src/lib.rs +++ b/crates/playback/src/lib.rs @@ -29,6 +29,8 @@ pub struct MeterSnapshot { enum Cmd { Session(Arc), + Input(Arc), + Recording(bool), Play { from: Samples, end: Option, looped: Option }, Stop, Locate(Samples), @@ -58,6 +60,8 @@ struct AudioState { pending_len: usize, read: f64, device_rate: f64, + input: Option>, + recording: bool, } impl AudioState { @@ -77,12 +81,16 @@ impl AudioState { pending_len: 0, read: 0.0, device_rate, + input: None, + recording: false, } } fn drain(&mut self) { while let Ok(c) = self.rx.try_recv() { match c { + Cmd::Input(r) => self.input = Some(r), + Cmd::Recording(r) => self.recording = r, Cmd::Session(s) => { if (s.sample_rate.as_f64() - self.session.sample_rate.as_f64()).abs() > 0.5 { self.mix = MixEngine::new(s.sample_rate.as_f64() as f32, BLOCK); @@ -138,6 +146,15 @@ impl AudioState { self.pending_len = BLOCK; return; } + if let Some(ring) = &self.input { + let ch = ring.channels.load(Ordering::Relaxed).max(1); + if self.mix.input.len() != ch { + self.mix.input = vec![vec![0.0; BLOCK]; ch]; + } + ring.pop_into(&mut self.mix.input, n); + } + self.mix.monitor_only = !self.playing; + self.mix.recording = self.recording; self.mix.render(&self.session, self.pos, n, &mut self.block); for (d, s) in self.pending.iter_mut().zip(self.block.iter()) { if let (Some(dd), Some(ss)) = (d.get_mut(..n), s.get(..n)) { @@ -145,6 +162,9 @@ impl AudioState { } } self.pending_len = n; + if !self.playing { + return; + } self.pos += n as i64; self.shared.position.store(self.pos, Ordering::Relaxed); if let Ok(mut m) = self.shared.meters.try_lock() { @@ -159,7 +179,8 @@ impl AudioState { fn fill(&mut self, out: &mut [f32], channels: usize) { self.drain(); let channels = channels.max(1); - if !self.playing { + let monitoring = self.input.is_some() && self.session.tracks.iter().any(|t| t.mixer.input_monitor || t.mixer.record_arm); + if !self.playing && !monitoring { out.iter_mut().for_each(|x| *x = 0.0); if let Ok(mut m) = self.shared.meters.try_lock() { m.main = StripMeter::default(); @@ -172,7 +193,7 @@ impl AudioState { for frame in out.chunks_mut(channels) { while self.read >= self.pending_len as f64 { self.read -= self.pending_len as f64; - if !self.playing { + if !self.playing && !monitoring { break; } self.render_next(); @@ -186,7 +207,7 @@ impl AudioState { let ch = self.pending.get(c.min(1)); let a = ch.and_then(|v| v.get(i0)).copied().unwrap_or(0.0); let b = ch.and_then(|v| v.get(i0 + 1)).copied().unwrap_or(a); - *o = if c < 2 && self.playing { (a + (b - a) * fr).clamp(-1.0, 1.0) } else { 0.0 }; + *o = if c < 2 && (self.playing || monitoring) { (a + (b - a) * fr).clamp(-1.0, 1.0) } else { 0.0 }; } self.read += step; } @@ -302,6 +323,15 @@ impl Player { let _ = self.tx.send(Cmd::Play { from, end, looped }); } + /// Feed live input (from a `record::Recorder`) into the mix for input monitoring. + pub fn set_input(&self, ring: Arc) { + let _ = self.tx.send(Cmd::Input(ring)); + } + + pub fn set_recording(&self, on: bool) { + let _ = self.tx.send(Cmd::Recording(on)); + } + pub fn stop(&self) { self.shared.playing.store(false, Ordering::Relaxed); let _ = self.tx.send(Cmd::Stop); diff --git a/crates/playback/src/record.rs b/crates/playback/src/record.rs index e6154eb9..d9def4c0 100644 --- a/crates/playback/src/record.rs +++ b/crates/playback/src/record.rs @@ -21,6 +21,8 @@ struct Shared { pub struct Recorder { shared: Arc, + /// Live input for monitoring (always fed while the input stream runs). + pub monitor: Arc, #[cfg(not(target_os = "freebsd"))] _stream: Option, pub device_name: String, @@ -48,11 +50,14 @@ impl Recorder { let config = cfg.config(); let shared = Arc::new(Shared { armed: AtomicBool::new(false), data: Mutex::new(Vec::new()) }); let sh = Arc::clone(&shared); + let monitor = InputRing::new(usize::from(config.channels)); + let mon = Arc::clone(&monitor); let mut backlog: Vec = Vec::new(); let stream = device .build_input_stream( &config, move |data: &[f32], _: &cpal::InputCallbackInfo| { + mon.push(data); if !sh.armed.load(Ordering::Relaxed) { backlog.clear(); return; @@ -73,6 +78,7 @@ impl Recorder { .map_err(|e| e.to_string())?; stream.play().map_err(|e| e.to_string())?; Ok(Recorder { + monitor, shared, _stream: Some(stream), device_name: name, @@ -133,3 +139,48 @@ mod tests { assert_eq!(t.channels.len(), 1); } } + +/// A small interleaved FIFO from the input callback to the output callback, for input +/// monitoring. Both sides only `try_lock`; it holds at most ~0.5 s and drops the oldest frames. +#[derive(Default)] +pub struct InputRing { + data: Mutex>, + pub channels: std::sync::atomic::AtomicUsize, +} + +impl InputRing { + pub fn new(channels: usize) -> Arc { + Arc::new(InputRing { + data: Mutex::new(std::collections::VecDeque::with_capacity(48_000 * channels.max(1))), + channels: std::sync::atomic::AtomicUsize::new(channels.max(1)), + }) + } + + pub fn push(&self, samples: &[f32]) { + if let Ok(mut d) = self.data.try_lock() { + let cap = 24_000 * self.channels.load(Ordering::Relaxed).max(1); + d.extend(samples.iter().copied()); + while d.len() > cap { + d.pop_front(); + } + } + } + + /// Fill `out` (planar) with up to `frames` frames; missing frames are silence. + pub fn pop_into(&self, out: &mut [Vec], frames: usize) { + let ch = self.channels.load(Ordering::Relaxed).max(1); + for c in out.iter_mut() { + c.iter_mut().take(frames).for_each(|x| *x = 0.0); + } + let Ok(mut d) = self.data.try_lock() else { return }; + let avail = (d.len() / ch).min(frames); + for f in 0..avail { + for c in 0..ch { + let v = d.pop_front().unwrap_or(0.0); + if let Some(x) = out.get_mut(c).and_then(|o| o.get_mut(f)) { + *x = v; + } + } + } + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d3eb9da4..f9382312 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -155,6 +155,7 @@ pub struct SoundApp { /// Input capture, opened on first record. pub recorder: Option, record_start: Samples, + recorder_failed: bool, pub ui: UiState, pub services: Services, pub dialogs: dialogs::Dialogs, @@ -188,6 +189,7 @@ impl SoundApp { player, recorder: None, record_start: 0, + recorder_failed: false, ui: UiState::default(), services, dialogs: dialogs::Dialogs::default(), @@ -280,14 +282,13 @@ impl SoundApp { self.ui.status = "Record-enable a track first (the red button in its header).".into(); return; } - if self.recorder.is_none() && self.player.is_some() { - match soundcraft_playback::record::Recorder::open() { - Ok(r) => self.recorder = Some(r), - Err(e) => { - self.ui.status = format!("Cannot record: {e}"); - return; - } - } + self.ensure_input(); + if self.recorder.is_none() { + self.ui.status = "Cannot record: no audio input device".into(); + return; + } + if let Some(p) = &self.player { + p.set_recording(true); } self.record_start = self.engine.session().edit.selection.start; if let Some(r) = &self.recorder { @@ -306,8 +307,30 @@ impl SoundApp { } } + /// Open the input device once (for recording and input monitoring) and feed the player. + fn ensure_input(&mut self) { + if self.recorder.is_some() || self.recorder_failed || self.player.is_none() { + return; + } + match soundcraft_playback::record::Recorder::open() { + Ok(r) => { + if let Some(p) = &self.player { + p.set_input(std::sync::Arc::clone(&r.monitor)); + } + self.recorder = Some(r); + } + Err(e) => { + self.recorder_failed = true; + self.ui.status = format!("No audio input: {e}"); + } + } + } + fn finish_recording(&mut self) { self.engine.transport.recording = false; + if let Some(p) = &self.player { + p.set_recording(false); + } let Some(r) = &self.recorder else { return }; let take = r.take(); let rate = take.sample_rate; @@ -534,6 +557,12 @@ impl SoundApp { } self.handle_transport(dt); self.write_automation(ctx); + if self.recorder.is_none() + && !self.recorder_failed + && self.engine.session().tracks.iter().any(|t| t.mixer.input_monitor || t.mixer.record_arm) + { + self.ensure_input(); + } if self.is_playing() || self.meters.values().any(|m| m.level[0] > 0.0001) || !self.pending_shots.is_empty() { ctx.request_repaint(); }