From 0e85d7ee54eb5e169311e2ed9dd537bbd4b8309a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 6 Oct 2026 00:08:04 -0400 Subject: [PATCH] =?UTF-8?q?Copy=20Special=20=E2=80=BA=20Audio=20as=20MIDI:?= =?UTF-8?q?=20YIN=20pitch=20tracking=20into=20MIDI=20notes=20on=20the=20cl?= =?UTF-8?q?ipboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/dsp/src/lib.rs | 1 + crates/dsp/src/pitch_detect.rs | 143 ++++++++++++++++++++++++++++ crates/engine/src/cmd/audio_midi.rs | 75 +++++++++++++++ crates/engine/src/cmd/mod.rs | 2 + 4 files changed, 221 insertions(+) create mode 100644 crates/dsp/src/pitch_detect.rs create mode 100644 crates/engine/src/cmd/audio_midi.rs diff --git a/crates/dsp/src/lib.rs b/crates/dsp/src/lib.rs index 83c7b662..0ad45deb 100644 --- a/crates/dsp/src/lib.rs +++ b/crates/dsp/src/lib.rs @@ -19,6 +19,7 @@ pub mod offline; mod osc; pub mod pan; mod params; +pub mod pitch_detect; pub mod plugins; pub mod spectrum; mod util; diff --git a/crates/dsp/src/pitch_detect.rs b/crates/dsp/src/pitch_detect.rs new file mode 100644 index 00000000..8271ff35 --- /dev/null +++ b/crates/dsp/src/pitch_detect.rs @@ -0,0 +1,143 @@ +//! Monophonic pitch tracking (YIN, de Cheveigné & Kawahara 2002) and note segmentation, for +//! audio-to-MIDI conversion. + +/// A detected note: start and end in samples, MIDI pitch, and a velocity estimate. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DetectedNote { + pub start: usize, + pub end: usize, + pub pitch: u8, + pub velocity: u8, +} + +/// Fundamental frequency of one frame, or None when unvoiced. `threshold` ≈ 0.1–0.2. +pub fn yin(frame: &[f32], sample_rate: f32, min_hz: f32, max_hz: f32, threshold: f32) -> Option { + if !(sample_rate.is_finite() && sample_rate > 0.0) || frame.len() < 64 { + return None; + } + let max_tau = ((sample_rate / min_hz.max(20.0)) as usize).min(frame.len() / 2); + let min_tau = ((sample_rate / max_hz.max(min_hz + 1.0)) as usize).max(2); + if max_tau <= min_tau + 2 { + return None; + } + let w = frame.len() - max_tau; + let mut d = vec![0.0f32; max_tau + 1]; + for (tau, dv) in d.iter_mut().enumerate().skip(1) { + let mut sum = 0.0f32; + for j in 0..w { + let (Some(a), Some(b)) = (frame.get(j), frame.get(j + tau)) else { break }; + let diff = a - b; + sum += diff * diff; + } + *dv = sum; + } + // Cumulative mean normalised difference. + let mut cmnd = vec![1.0f32; max_tau + 1]; + let mut running = 0.0f32; + for tau in 1..=max_tau { + let v = d.get(tau).copied().unwrap_or(0.0); + running += v; + if let Some(c) = cmnd.get_mut(tau) { + *c = if running > 0.0 { v * tau as f32 / running } else { 1.0 }; + } + } + let mut tau = min_tau; + while tau < max_tau { + let c = cmnd.get(tau).copied().unwrap_or(1.0); + if c < threshold { + // Walk to the local minimum. + while tau + 1 < max_tau && cmnd.get(tau + 1).copied().unwrap_or(1.0) < cmnd.get(tau).copied().unwrap_or(1.0) { + tau += 1; + } + // Parabolic interpolation. + let (a, b, c) = + (cmnd.get(tau - 1).copied().unwrap_or(1.0), cmnd.get(tau).copied().unwrap_or(1.0), cmnd.get(tau + 1).copied().unwrap_or(1.0)); + let denom = a - 2.0 * b + c; + let shift = if denom.abs() > 1e-9 { 0.5 * (a - c) / denom } else { 0.0 }; + let t = tau as f32 + shift.clamp(-1.0, 1.0); + return (t > 0.0).then_some(sample_rate / t); + } + tau += 1; + } + None +} + +/// Convert mono audio to notes: frames of `hop` samples, pitch per frame, then runs of the same +/// semitone at least `min_frames` long become notes. +pub fn audio_to_notes(audio: &[f32], sample_rate: f32) -> Vec { + let sr = if sample_rate.is_finite() && sample_rate > 0.0 { sample_rate } else { 48_000.0 }; + let win = ((sr * 0.046) as usize).clamp(256, 8192); + let hop = win / 4; + let min_frames = 3; + let mut frames: Vec<(usize, Option, f32)> = Vec::new(); + let mut pos = 0usize; + while pos + win <= audio.len() && frames.len() < 2_000_000 { + let Some(frame) = audio.get(pos..pos + win) else { break }; + let rms = (frame.iter().map(|x| x * x).sum::() / win as f32).sqrt(); + let pitch = if rms > 0.01 { + yin(frame, sr, 50.0, 1600.0, 0.15).map(|f| (69.0 + 12.0 * (f / 440.0).log2()).round().clamp(0.0, 127.0) as u8) + } else { + None + }; + frames.push((pos, pitch, rms)); + pos += hop; + } + let mut notes = Vec::new(); + let mut i = 0; + while i < frames.len() { + let Some((start, Some(p), _)) = frames.get(i).copied() else { + i += 1; + continue; + }; + let mut j = i; + let mut peak = 0.0f32; + while let Some((_, Some(q), rms)) = frames.get(j).copied() { + if q != p { + break; + } + peak = peak.max(rms); + j += 1; + } + if j - i >= min_frames { + let end = frames.get(j).map_or(audio.len(), |f| f.0); + let velocity = (40.0 + 87.0 * (peak / 0.5).min(1.0)) as u8; + notes.push(DetectedNote { start, end, pitch: p, velocity: velocity.clamp(1, 127) }); + } + i = j.max(i + 1); + } + notes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tone(f: f32, secs: f32, sr: f32) -> Vec { + (0..(secs * sr) as usize).map(|i| (i as f32 * f * std::f32::consts::TAU / sr).sin() * 0.5).collect() + } + + #[test] + fn yin_finds_a440() { + let t = tone(440.0, 0.1, 48_000.0); + let f = yin(&t[..2048], 48_000.0, 50.0, 1600.0, 0.15).unwrap(); + assert!((f - 440.0).abs() < 2.0, "{f}"); + } + + #[test] + fn melody_to_notes() { + let sr = 48_000.0; + let mut a = tone(261.63, 0.4, sr); + a.extend(vec![0.0; 4800]); + a.extend(tone(392.0, 0.4, sr)); + let n = audio_to_notes(&a, sr); + let pitches: Vec = n.iter().map(|x| x.pitch).collect(); + assert_eq!(pitches, vec![60, 67], "{n:?}"); + } + + #[test] + fn silence_and_garbage() { + assert!(audio_to_notes(&vec![0.0; 48_000], 48_000.0).is_empty()); + assert!(audio_to_notes(&[], f32::NAN).is_empty()); + assert!(yin(&[0.0; 10], 48_000.0, 50.0, 1600.0, 0.15).is_none()); + } +} diff --git a/crates/engine/src/cmd/audio_midi.rs b/crates/engine/src/cmd/audio_midi.rs new file mode 100644 index 00000000..9e810902 --- /dev/null +++ b/crates/engine/src/cmd/audio_midi.rs @@ -0,0 +1,75 @@ +//! Edit › Copy Special › Audio as MIDI: pitch-track the selected audio into MIDI notes on the +//! clipboard (paste them onto a MIDI or instrument track). + +use super::*; +use crate::{Clipboard, cmd}; +use serde_json::json; +use soundcraft_midi::{Note, Sequence}; +use soundcraft_model::{Clip, ClipId}; + +pub fn specs() -> Vec { + vec![ + cmd!(noundo "edit.copy_audio_as_midi", "Audio as MIDI", ["Edit", "Copy Special"], None, "{track?, start?, end?} — monophonic pitch tracking of the selected audio", has_range, audio_as_midi), + ] +} + +fn audio_as_midi(e: &mut Engine, p: &Value) -> Result { + let id = "edit.copy_audio_as_midi"; + let t = tracks_required(e, id, p)?.first().copied().ok_or_else(|| bad(id, "no track"))?; + let r = range_param(e, id, p)?; + if r.is_empty() { + return Err(bad(id, "select some audio first")); + } + let s = e.session(); + if s.track(t).is_none_or(|x| x.kind != soundcraft_model::TrackKind::Audio) { + return Err(bad(id, "select audio on an audio track")); + } + let audio = soundcraft_mix::render_clips(s, t, r); + let n = audio.first().map_or(0, Vec::len); + let k = audio.len().max(1) as f32; + let mono: Vec = (0..n).map(|i| audio.iter().map(|c| c.get(i).copied().unwrap_or(0.0)).sum::() / k).collect(); + let detected = soundcraft_dsp::pitch_detect::audio_to_notes(&mono, s.sample_rate.as_f64() as f32); + let base = s.tempo.samples_to_ticks(r.start, s.sample_rate); + let mut seq = Sequence::default(); + for d in &detected { + let a = s.tempo.samples_to_ticks(r.start + d.start as i64, s.sample_rate) - base; + let b = s.tempo.samples_to_ticks(r.start + d.end as i64, s.sample_rate) - base; + seq.notes.push(Note { pitch: d.pitch, velocity: d.velocity, release_velocity: 64, channel: 0, start: a.max(0), length: (b - a).max(1) }); + } + seq.sort(); + let count = seq.notes.len(); + let clip = Clip::midi(ClipId(0), "Audio as MIDI", 0, r.len(), seq); + e.clipboard = Clipboard { length: r.len(), tracks: vec![vec![clip]], automation: vec![Vec::new()], ..Clipboard::default() }; + Ok(json!({"notes": count, "length": r.len()})) +} + +#[cfg(test)] +mod tests { + use super::*; + use soundcraft_audio_io::AudioBuffer; + use soundcraft_model::{ChannelFormat, TrackKind}; + + #[test] + fn copies_a_melody_as_midi_and_pastes_it() { + let mut e = Engine::default(); + let sr = 48_000.0f32; + let tone = |f: f32| -> Vec { (0..(0.4 * sr) as usize).map(|i| (i as f32 * f * std::f32::consts::TAU / sr).sin() * 0.5).collect() }; + let mut a = tone(220.0); + a.extend(tone(330.0)); + let frames = a.len() as i64; + let s = e.session_mut(); + let src = crate::io::add_source(s, "mel", AudioBuffer { sample_rate: 48_000, channels: vec![a] }, None, soundcraft_audio_io::FileFormat::Wav); + let t = s.add_track(TrackKind::Audio, ChannelFormat::Mono, None); + let cid = s.new_clip_id(); + crate::edit::place_clip(s, t, Clip::audio(cid, "mel", src, 0, 0, frames)); + let r = e.execute("edit.copy_audio_as_midi", &json!({"track": t.0, "start": 0, "end": frames})).unwrap(); + assert_eq!(r["notes"], 2, "{r}"); + let m = e.session_mut().add_track(TrackKind::Midi, ChannelFormat::Mono, None); + e.execute("edit.paste", &json!({"track": m.0, "at": 0})).unwrap(); + let clip = e.session().track(m).unwrap().clips()[0].clone(); + match clip.content { + soundcraft_model::ClipContent::Midi { sequence } => assert_eq!(sequence.notes.iter().map(|n| n.pitch).collect::>(), vec![57, 64]), + _ => panic!("expected MIDI"), + } + } +} diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 2903436f..34cc14ee 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -6,6 +6,7 @@ use soundcraft_model::{ClipId, TrackId}; use soundcraft_time::{Range, Samples}; use std::sync::OnceLock; +mod audio_midi; mod audiosuite; mod clap; mod clip; @@ -103,6 +104,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(transport::specs()); v.extend(query::specs()); v.extend(audiosuite::specs()); + v.extend(audio_midi::specs()); v.extend(midi::specs()); v.extend(view_more::specs()); v.extend(track_more::specs());