Copy Special › Audio as MIDI: YIN pitch tracking into MIDI notes on the clipboard

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/dsp/src/lib.rs

@@ -19,6 +19,7 @@ pub mod offline;
 mod osc;
 pub mod pan;
 mod params;
+pub mod pitch_detect;
 pub mod plugins;
 pub mod spectrum;
 mod util;

crates/dsp/src/pitch_detect.rs

@@ -0,0 +1,143 @@
+//! Monophonic pitch tracking (YIN, de Cheveigné & Kawahara 2002) and note segmentation, for
+//! audio-to-MIDI conversion.
+
+/// A detected note: start and end in samples, MIDI pitch, and a velocity estimate.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct DetectedNote {
+    pub start: usize,
+    pub end: usize,
+    pub pitch: u8,
+    pub velocity: u8,
+}
+
+/// Fundamental frequency of one frame, or None when unvoiced. `threshold` ≈ 0.1–0.2.
+pub fn yin(frame: &[f32], sample_rate: f32, min_hz: f32, max_hz: f32, threshold: f32) -> Option<f32> {
+    if !(sample_rate.is_finite() && sample_rate > 0.0) || frame.len() < 64 {
+        return None;
+    }
+    let max_tau = ((sample_rate / min_hz.max(20.0)) as usize).min(frame.len() / 2);
+    let min_tau = ((sample_rate / max_hz.max(min_hz + 1.0)) as usize).max(2);
+    if max_tau <= min_tau + 2 {
+        return None;
+    }
+    let w = frame.len() - max_tau;
+    let mut d = vec![0.0f32; max_tau + 1];
+    for (tau, dv) in d.iter_mut().enumerate().skip(1) {
+        let mut sum = 0.0f32;
+        for j in 0..w {
+            let (Some(a), Some(b)) = (frame.get(j), frame.get(j + tau)) else { break };
+            let diff = a - b;
+            sum += diff * diff;
+        }
+        *dv = sum;
+    }
+    // Cumulative mean normalised difference.
+    let mut cmnd = vec![1.0f32; max_tau + 1];
+    let mut running = 0.0f32;
+    for tau in 1..=max_tau {
+        let v = d.get(tau).copied().unwrap_or(0.0);
+        running += v;
+        if let Some(c) = cmnd.get_mut(tau) {
+            *c = if running > 0.0 { v * tau as f32 / running } else { 1.0 };
+        }
+    }
+    let mut tau = min_tau;
+    while tau < max_tau {
+        let c = cmnd.get(tau).copied().unwrap_or(1.0);
+        if c < threshold {
+            // Walk to the local minimum.
+            while tau + 1 < max_tau && cmnd.get(tau + 1).copied().unwrap_or(1.0) < cmnd.get(tau).copied().unwrap_or(1.0) {
+                tau += 1;
+            }
+            // Parabolic interpolation.
+            let (a, b, c) =
+                (cmnd.get(tau - 1).copied().unwrap_or(1.0), cmnd.get(tau).copied().unwrap_or(1.0), cmnd.get(tau + 1).copied().unwrap_or(1.0));
+            let denom = a - 2.0 * b + c;
+            let shift = if denom.abs() > 1e-9 { 0.5 * (a - c) / denom } else { 0.0 };
+            let t = tau as f32 + shift.clamp(-1.0, 1.0);
+            return (t > 0.0).then_some(sample_rate / t);
+        }
+        tau += 1;
+    }
+    None
+}
+
+/// Convert mono audio to notes: frames of `hop` samples, pitch per frame, then runs of the same
+/// semitone at least `min_frames` long become notes.
+pub fn audio_to_notes(audio: &[f32], sample_rate: f32) -> Vec<DetectedNote> {
+    let sr = if sample_rate.is_finite() && sample_rate > 0.0 { sample_rate } else { 48_000.0 };
+    let win = ((sr * 0.046) as usize).clamp(256, 8192);
+    let hop = win / 4;
+    let min_frames = 3;
+    let mut frames: Vec<(usize, Option<u8>, f32)> = Vec::new();
+    let mut pos = 0usize;
+    while pos + win <= audio.len() && frames.len() < 2_000_000 {
+        let Some(frame) = audio.get(pos..pos + win) else { break };
+        let rms = (frame.iter().map(|x| x * x).sum::<f32>() / win as f32).sqrt();
+        let pitch = if rms > 0.01 {
+            yin(frame, sr, 50.0, 1600.0, 0.15).map(|f| (69.0 + 12.0 * (f / 440.0).log2()).round().clamp(0.0, 127.0) as u8)
+        } else {
+            None
+        };
+        frames.push((pos, pitch, rms));
+        pos += hop;
+    }
+    let mut notes = Vec::new();
+    let mut i = 0;
+    while i < frames.len() {
+        let Some((start, Some(p), _)) = frames.get(i).copied() else {
+            i += 1;
+            continue;
+        };
+        let mut j = i;
+        let mut peak = 0.0f32;
+        while let Some((_, Some(q), rms)) = frames.get(j).copied() {
+            if q != p {
+                break;
+            }
+            peak = peak.max(rms);
+            j += 1;
+        }
+        if j - i >= min_frames {
+            let end = frames.get(j).map_or(audio.len(), |f| f.0);
+            let velocity = (40.0 + 87.0 * (peak / 0.5).min(1.0)) as u8;
+            notes.push(DetectedNote { start, end, pitch: p, velocity: velocity.clamp(1, 127) });
+        }
+        i = j.max(i + 1);
+    }
+    notes
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tone(f: f32, secs: f32, sr: f32) -> Vec<f32> {
+        (0..(secs * sr) as usize).map(|i| (i as f32 * f * std::f32::consts::TAU / sr).sin() * 0.5).collect()
+    }
+
+    #[test]
+    fn yin_finds_a440() {
+        let t = tone(440.0, 0.1, 48_000.0);
+        let f = yin(&t[..2048], 48_000.0, 50.0, 1600.0, 0.15).unwrap();
+        assert!((f - 440.0).abs() < 2.0, "{f}");
+    }
+
+    #[test]
+    fn melody_to_notes() {
+        let sr = 48_000.0;
+        let mut a = tone(261.63, 0.4, sr);
+        a.extend(vec![0.0; 4800]);
+        a.extend(tone(392.0, 0.4, sr));
+        let n = audio_to_notes(&a, sr);
+        let pitches: Vec<u8> = n.iter().map(|x| x.pitch).collect();
+        assert_eq!(pitches, vec![60, 67], "{n:?}");
+    }
+
+    #[test]
+    fn silence_and_garbage() {
+        assert!(audio_to_notes(&vec![0.0; 48_000], 48_000.0).is_empty());
+        assert!(audio_to_notes(&[], f32::NAN).is_empty());
+        assert!(yin(&[0.0; 10], 48_000.0, 50.0, 1600.0, 0.15).is_none());
+    }
+}

crates/engine/src/cmd/audio_midi.rs

@@ -0,0 +1,75 @@
+//! Edit › Copy Special › Audio as MIDI: pitch-track the selected audio into MIDI notes on the
+//! clipboard (paste them onto a MIDI or instrument track).
+
+use super::*;
+use crate::{Clipboard, cmd};
+use serde_json::json;
+use soundcraft_midi::{Note, Sequence};
+use soundcraft_model::{Clip, ClipId};
+
+pub fn specs() -> Vec<CommandSpec> {
+    vec![
+        cmd!(noundo "edit.copy_audio_as_midi", "Audio as MIDI", ["Edit", "Copy Special"], None, "{track?, start?, end?} — monophonic pitch tracking of the selected audio", has_range, audio_as_midi),
+    ]
+}
+
+fn audio_as_midi(e: &mut Engine, p: &Value) -> Result<Value> {
+    let id = "edit.copy_audio_as_midi";
+    let t = tracks_required(e, id, p)?.first().copied().ok_or_else(|| bad(id, "no track"))?;
+    let r = range_param(e, id, p)?;
+    if r.is_empty() {
+        return Err(bad(id, "select some audio first"));
+    }
+    let s = e.session();
+    if s.track(t).is_none_or(|x| x.kind != soundcraft_model::TrackKind::Audio) {
+        return Err(bad(id, "select audio on an audio track"));
+    }
+    let audio = soundcraft_mix::render_clips(s, t, r);
+    let n = audio.first().map_or(0, Vec::len);
+    let k = audio.len().max(1) as f32;
+    let mono: Vec<f32> = (0..n).map(|i| audio.iter().map(|c| c.get(i).copied().unwrap_or(0.0)).sum::<f32>() / k).collect();
+    let detected = soundcraft_dsp::pitch_detect::audio_to_notes(&mono, s.sample_rate.as_f64() as f32);
+    let base = s.tempo.samples_to_ticks(r.start, s.sample_rate);
+    let mut seq = Sequence::default();
+    for d in &detected {
+        let a = s.tempo.samples_to_ticks(r.start + d.start as i64, s.sample_rate) - base;
+        let b = s.tempo.samples_to_ticks(r.start + d.end as i64, s.sample_rate) - base;
+        seq.notes.push(Note { pitch: d.pitch, velocity: d.velocity, release_velocity: 64, channel: 0, start: a.max(0), length: (b - a).max(1) });
+    }
+    seq.sort();
+    let count = seq.notes.len();
+    let clip = Clip::midi(ClipId(0), "Audio as MIDI", 0, r.len(), seq);
+    e.clipboard = Clipboard { length: r.len(), tracks: vec![vec![clip]], automation: vec![Vec::new()], ..Clipboard::default() };
+    Ok(json!({"notes": count, "length": r.len()}))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use soundcraft_audio_io::AudioBuffer;
+    use soundcraft_model::{ChannelFormat, TrackKind};
+
+    #[test]
+    fn copies_a_melody_as_midi_and_pastes_it() {
+        let mut e = Engine::default();
+        let sr = 48_000.0f32;
+        let tone = |f: f32| -> Vec<f32> { (0..(0.4 * sr) as usize).map(|i| (i as f32 * f * std::f32::consts::TAU / sr).sin() * 0.5).collect() };
+        let mut a = tone(220.0);
+        a.extend(tone(330.0));
+        let frames = a.len() as i64;
+        let s = e.session_mut();
+        let src = crate::io::add_source(s, "mel", AudioBuffer { sample_rate: 48_000, channels: vec![a] }, None, soundcraft_audio_io::FileFormat::Wav);
+        let t = s.add_track(TrackKind::Audio, ChannelFormat::Mono, None);
+        let cid = s.new_clip_id();
+        crate::edit::place_clip(s, t, Clip::audio(cid, "mel", src, 0, 0, frames));
+        let r = e.execute("edit.copy_audio_as_midi", &json!({"track": t.0, "start": 0, "end": frames})).unwrap();
+        assert_eq!(r["notes"], 2, "{r}");
+        let m = e.session_mut().add_track(TrackKind::Midi, ChannelFormat::Mono, None);
+        e.execute("edit.paste", &json!({"track": m.0, "at": 0})).unwrap();
+        let clip = e.session().track(m).unwrap().clips()[0].clone();
+        match clip.content {
+            soundcraft_model::ClipContent::Midi { sequence } => assert_eq!(sequence.notes.iter().map(|n| n.pitch).collect::<Vec<_>>(), vec![57, 64]),
+            _ => panic!("expected MIDI"),
+        }
+    }
+}

crates/engine/src/cmd/mod.rs

@@ -6,6 +6,7 @@ use soundcraft_model::{ClipId, TrackId};
 use soundcraft_time::{Range, Samples};
 use std::sync::OnceLock;
 
+mod audio_midi;
 mod audiosuite;
 mod clap;
 mod clip;
@@ -103,6 +104,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
         v.extend(transport::specs());
         v.extend(query::specs());
         v.extend(audiosuite::specs());
+        v.extend(audio_midi::specs());
         v.extend(midi::specs());
         v.extend(view_more::specs());
         v.extend(track_more::specs());