Merge pull request #132 from punkhop/fix/playback-rate-conversion

Audio: play a sequence on a device at another rate without phase jumps

crates/audio-dsp/src/lib.rs

@@ -7,6 +7,7 @@
 //!   engine and UI can build controls generically.
 //! * [`biquad`] — RBJ-cookbook biquads (TDF-II) with analytic magnitude response.
 //! * [`sync`] — offset between two recordings of one event (GCC-PHAT, sample-accurate).
+//! * [`resample`] — streaming, block-invariant rate conversion for playback on a device at another rate.
 //! * [`channels`] — channel layouts, ITU-R BS.775 up/downmix, 5.1 mixdown types, the 5.1 panner.
 //!
 //! Conventions: audio is **planar f32** (`&mut [&mut [f32]]`, one slice per channel, all the same
@@ -27,6 +28,7 @@ pub mod fft;
 pub mod loudness;
 pub mod oversample;
 pub mod remix;
+pub mod resample;
 mod smooth;
 pub mod sync;
 

crates/audio-dsp/src/resample.rs

@@ -0,0 +1,170 @@
+//! Streaming sample-rate conversion for playback, when the output device runs at another rate
+//! than the sequence (a 44.1 kHz device playing a 48 kHz sequence).
+//!
+//! Every output sample's source position comes from its absolute output frame, as an exact
+//! rational (`frame × from / to`), and is linearly interpolated, so the output is identical however
+//! the stream is cut into blocks: no phase jump at block boundaries. Source frames are pulled in
+//! contiguous, non-overlapping spans, so whatever produces them (the mixer, with its effect state)
+//! sees one continuous stream. Output lags the source by one source sample (≈21 µs at 48 kHz),
+//! which is what lets each block interpolate without reading ahead.
+
+/// Converts a planar stream from `from` Hz to `to` Hz, block by block.
+#[derive(Debug, Clone)]
+pub struct StreamResampler {
+    from: u32,
+    to: u32,
+    /// Source frames `[held_start, held_start + held[c].len())` kept from the last pull.
+    held_start: i64,
+    held: Vec<Vec<f32>>,
+    /// The output frame the next block is expected to start at (contiguous playback).
+    next_out: Option<i64>,
+}
+
+impl StreamResampler {
+    /// Rates of zero are treated as 1 Hz rather than dividing by zero.
+    pub fn new(from: u32, to: u32) -> Self {
+        Self { from: from.max(1), to: to.max(1), held_start: 0, held: Vec::new(), next_out: None }
+    }
+
+    pub fn rates(&self) -> (u32, u32) {
+        (self.from, self.to)
+    }
+
+    /// The source frame whose sample lands at (or just before) output frame `out`.
+    fn source_index(&self, out: i64) -> (i64, f32) {
+        let p = out as i128 * self.from as i128;
+        let to = self.to as i128;
+        (p.div_euclid(to) as i64, (p.rem_euclid(to) as f64 / to as f64) as f32)
+    }
+
+    /// Output frames `[out, out + n)`, planar. `pull(start, frames)` returns source frames
+    /// `[start, start + frames)`, planar (missing samples read as silence). A block that does not
+    /// continue the previous one (a seek) starts the stream afresh.
+    pub fn process(&mut self, out: i64, n: usize, mut pull: impl FnMut(i64, usize) -> Vec<Vec<f32>>) -> Vec<Vec<f32>> {
+        if n == 0 {
+            return vec![Vec::new(); self.held.len().max(1)];
+        }
+        let (first, _) = self.source_index(out);
+        let (last, _) = self.source_index(out.saturating_add(n as i64 - 1));
+        // interpolation reads source frames first - 1 ..= last
+        if self.next_out != Some(out) || self.held.is_empty() {
+            self.held_start = first - 1;
+            self.held.clear();
+        }
+        let held_end = self.held_start + self.held.first().map_or(0, Vec::len) as i64;
+        if last + 1 > held_end {
+            let want = (last + 1 - held_end) as usize;
+            let got = pull(held_end, want);
+            if self.held.is_empty() {
+                self.held = vec![Vec::new(); got.len().max(1)];
+            }
+            for (c, h) in self.held.iter_mut().enumerate() {
+                let src = got.get(c).map(Vec::as_slice).unwrap_or_default();
+                h.extend((0..want).map(|i| src.get(i).copied().unwrap_or(0.0)));
+            }
+        }
+        let mut dst = vec![vec![0.0f32; n]; self.held.len()];
+        for (i, frame) in (out..).take(n).enumerate() {
+            let (s, frac) = self.source_index(frame);
+            // held starts at first - 1, so the earlier frame is always held
+            let k = usize::try_from(s - self.held_start).unwrap_or(0);
+            for (d, h) in dst.iter_mut().zip(&self.held) {
+                let a = k.checked_sub(1).and_then(|j| h.get(j)).copied().unwrap_or(0.0);
+                let b = h.get(k).copied().unwrap_or(0.0);
+                if let Some(x) = d.get_mut(i) {
+                    *x = a + (b - a) * frac;
+                }
+            }
+        }
+        // keep what the next block can still read: from its first frame - 1 on
+        let (next_first, _) = self.source_index(out.saturating_add(n as i64));
+        let drop = (next_first - 1 - self.held_start).clamp(0, self.held.first().map_or(0, Vec::len) as i64) as usize;
+        for h in &mut self.held {
+            h.drain(..drop);
+        }
+        self.held_start += drop as i64;
+        self.next_out = Some(out.saturating_add(n as i64));
+        dst
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A sine at `hz`, sampled at `rate`, from absolute frame `start`; logs every pull.
+    fn sine(hz: f64, rate: u32, pulls: &mut Vec<(i64, usize)>) -> impl FnMut(i64, usize) -> Vec<Vec<f32>> + '_ {
+        move |start, frames| {
+            pulls.push((start, frames));
+            let ch: Vec<f32> = (0..frames).map(|i| (2.0 * std::f64::consts::PI * hz * (start + i as i64) as f64 / rate as f64).sin() as f32).collect();
+            vec![ch.clone(), ch]
+        }
+    }
+
+    fn convert(from: u32, to: u32, blocks: &[usize], total: usize) -> (Vec<f32>, Vec<(i64, usize)>) {
+        let mut r = StreamResampler::new(from, to);
+        let mut pulls = Vec::new();
+        let mut out = Vec::new();
+        let mut pos = 0i64;
+        let mut k = 0;
+        while (pos as usize) < total {
+            let n = blocks[k % blocks.len()].min(total - pos as usize);
+            let b = r.process(pos, n, sine(1000.0, from, &mut pulls));
+            assert_eq!(b.len(), 2);
+            assert!(b.iter().all(|c| c.len() == n));
+            out.extend_from_slice(&b[0]);
+            pos += n as i64;
+            k += 1;
+        }
+        (out, pulls)
+    }
+
+    #[test]
+    fn output_is_the_same_however_the_stream_is_cut() {
+        for (from, to) in [(48_000, 44_100), (44_100, 48_000), (48_000, 96_000), (96_000, 48_000)] {
+            let (whole, _) = convert(from, to, &[20_000], 20_000);
+            for blocks in [&[512][..], &[333], &[1, 7, 512, 64]] {
+                let (cut, _) = convert(from, to, blocks, 20_000);
+                assert_eq!(cut, whole, "{from}→{to} in blocks {blocks:?}");
+            }
+        }
+    }
+
+    #[test]
+    fn a_tone_converts_to_the_same_tone() {
+        // 1 kHz at 48 kHz played on a 44.1 kHz device: within linear interpolation's error of the
+        // ideal tone, one source sample late
+        let (from, to) = (48_000u32, 44_100u32);
+        let (out, _) = convert(from, to, &[512], 20_000);
+        for (o, x) in out.iter().enumerate().skip(2) {
+            let t = o as f64 / to as f64 - 1.0 / from as f64;
+            let want = (2.0 * std::f64::consts::PI * 1000.0 * t).sin() as f32;
+            assert!((x - want).abs() < 3e-3, "frame {o}: {x} vs {want}");
+        }
+    }
+
+    #[test]
+    fn the_source_is_pulled_as_one_continuous_stream() {
+        for (from, to) in [(48_000, 44_100), (44_100, 48_000)] {
+            let (_, pulls) = convert(from, to, &[512, 333], 20_000);
+            for w in pulls.windows(2) {
+                assert_eq!(w[0].0 + w[0].1 as i64, w[1].0, "{from}→{to}: {:?} then {:?}", w[0], w[1]);
+            }
+        }
+    }
+
+    #[test]
+    fn a_seek_starts_afresh_and_hostile_input_stays_finite() {
+        let mut r = StreamResampler::new(48_000, 44_100);
+        let mut pulls = Vec::new();
+        let _ = r.process(0, 512, sine(1000.0, 48_000, &mut pulls));
+        let _ = r.process(1_000_000, 512, sine(1000.0, 48_000, &mut pulls));
+        assert_eq!(pulls.last().map(|p| p.0), Some(1_000_000 * 48_000 / 44_100 - 1));
+        // a mixer returning too few channels and samples, and zero rates
+        let mut r = StreamResampler::new(0, 0);
+        let b = r.process(5, 64, |_, _| vec![vec![1.0; 3]]);
+        assert_eq!(b.len(), 1);
+        assert!(b[0].iter().all(|x| x.is_finite()));
+        assert!(r.process(0, 0, |_, _| Vec::new()).iter().all(Vec::is_empty));
+    }
+}

crates/ui-egui/examples/bench_audio.rs

@@ -118,7 +118,7 @@ fn write_wav(path: &std::path::Path, samples: &[f32]) -> std::io::Result<()> {
 }
 
 fn play(s: &Session, seq: ItemId, mode: &str, seconds: f64) -> Run {
-    let mix = filmcraft_ui_egui::playback_mix(s, seq, SR);
+    let mut mix = filmcraft_ui_egui::playback_mix(s, seq, SR);
     match mode {
         "direct" => {
             let mut cursor = 0i64;

crates/ui-egui/src/lib.rs

@@ -615,7 +615,7 @@ impl FilmcraftApp {
         let fill = play_ahead::spawn(mix, cursor, sr, a.channels(), self.playback.audio_stats.clone());
         #[cfg(target_arch = "wasm32")]
         let fill = {
-            let mut cursor = cursor;
+            let (mut mix, mut cursor) = (mix, cursor);
             Box::new(move |buf: &mut [f32], ch: usize| {
                 mix(cursor, buf, ch);
                 cursor += (buf.len() / ch.max(1)) as i64;
@@ -1333,7 +1333,7 @@ impl eframe::App for FilmcraftApp {
 /// The program mix that playback plays: `mix(device_frame, interleaved, channels)` renders the
 /// active sequence at device rate `sr` from `device_frame`, with the live project snapshot (edits
 /// made while playing are heard), Output Mapping, the 5.1 mixdown and voice-over cues.
-pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u32) -> impl Fn(i64, &mut [f32], usize) + Send + 'static {
+pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u32) -> impl FnMut(i64, &mut [f32], usize) + Send + 'static {
     let project = session.project.clone();
     let provider = session.media.provider(project.clone(), session.services.clone());
     let previews = session.previews.clone();
@@ -1343,6 +1343,7 @@ pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u3
     let mixdown = filmcraft_audio_dsp::channels::Mixdown::from_id(&session.prefs.audio.mixdown_type).unwrap_or_default();
     let cues = panels::voiceover::cues(session, sr);
     previews.live.publish_project(project.clone());
+    let mut resampler = None;
     move |cursor: i64, buf: &mut [f32], ch: usize| {
         // the newest project snapshot: mixer moves and other edits are heard while playing
         let project = previews.live.project().filter(|p| p.sequence(seq_id).is_some()).unwrap_or_else(|| project.clone());
@@ -1351,7 +1352,6 @@ pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u3
             return;
         };
         let n = buf.len() / ch.max(1);
-        // Mix at the sequence rate; convert when the device rate differs (nearest sample).
         let seq_sr = seq.settings.sample_rate;
         // a 5.1 Mix plays as six channels (L, R, C, LFE, Ls, Rs) on a device with at least six
         use filmcraft_audio_dsp::channels::Layout;
@@ -1359,17 +1359,18 @@ pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u3
         let mix = if seq_sr == sr {
             previews.mix_layout(&project, seq_id, cursor, n, &provider, layout, mixdown)
         } else {
-            let s0 = (cursor as i128 * seq_sr as i128 / sr as i128) as i64;
-            let m = n * seq_sr as usize / sr as usize + 2;
-            let b = previews.mix_layout(&project, seq_id, s0, m, &provider, layout, mixdown);
-            let mut out = filmcraft_frame::AudioBuffer::silence(sr, b.channels.len(), n);
-            for (o, c) in out.channels.iter_mut().zip(&b.channels) {
-                for (i, x) in o.iter_mut().enumerate() {
-                    let j = (i * seq_sr as usize / sr as usize).min(m - 1);
-                    *x = c[j];
-                }
+            // Mix at the sequence rate as one continuous stream, interpolated to the device rate
+            // (started afresh when the rate or the channel layout changes while playing).
+            let key = (seq_sr, layout);
+            if resampler.as_ref().is_none_or(|(k, _)| *k != key) {
+                resampler = Some((key, filmcraft_audio_dsp::resample::StreamResampler::new(seq_sr, sr)));
             }
-            out
+            let Some((_, r)) = resampler.as_mut() else {
+                buf.fill(0.0);
+                return;
+            };
+            let channels = r.process(cursor, n, |s0, m| previews.mix_layout(&project, seq_id, s0, m, &provider, layout, mixdown).channels);
+            filmcraft_frame::AudioBuffer { sample_rate: sr, channels }
         };
         if layout == Layout::Surround51 {
             buf.fill(0.0);