aleleba/filmcraft
1
0
Fork 0
mirror of https://github.com/storytold/filmcraft.git synced 2026-10-24 10:10:05 -06:00
Code Issues Packages Projects Releases Wiki Activity

security(remix): bound analysis resources (#2926)

Browse Source 
* security(remix): bound analysis resources

* fix(audio-dsp): bound public analysis inputs
This commit is contained in:
Earl Co
2026-10-24 04:49:48 -04:00
committed by GitHub
parent ea8380a24f
commit 8c928442ab
3 changed files with 83 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/audio-dsp/src/remix.rs
+66 -20
View File
@@ -37,6 +37,13 @@ pub const CONTEXT: usize = 4;
pub const MIN_PIECE_BEATS: usize = 4;
/// Crossfade at each joint (seconds).
pub const XFADE_S: f64 = 0.02;
/// Long enough for a complete music cue while bounding the audio decoded and analysed by Remix.
pub const MAX_ANALYSIS_SECONDS: u64 = 10 * 60;
/// A dense similarity matrix is useful for planning, but must remain a bounded allocation.
const MAX_SIMILARITY_CELLS: usize = 4_000_000;
/// Remix analysis supports ordinary production and high-resolution audio rates, not arbitrary
/// integers supplied through a public library API.
pub const MAX_ANALYSIS_SAMPLE_RATE: u32 = 384_000;
/// One piece of the output: `len` samples read from the source at `src`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -139,6 +146,12 @@ impl Analysis {
/// Why music cannot be remixed.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum RemixError {
/// The source sample rate is zero or above the supported analysis limit.
InvalidSampleRate,
/// The source would exceed the bounded Remix analysis duration.
TooLong,
/// The detected beat grid would need too much similarity storage or work.
TooComplex,
/// Not enough rhythmic content to find a beat (silence, noise, speech).
NoBeat,
/// The source has fewer beats than a remix needs.
@@ -150,6 +163,9 @@ pub enum RemixError {
impl std::fmt::Display for RemixError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
RemixError::InvalidSampleRate => "the audio sample rate is not supported for remix analysis",
RemixError::TooLong => "the music is too long to remix",
RemixError::TooComplex => "the music has too many beats to remix",
RemixError::NoBeat => "no beat found in the audio",
RemixError::TooShort => "the music is too short to remix",
RemixError::TargetTooShort => "the target duration is too short for this music",
@@ -188,7 +204,8 @@ fn stft_mag(x: &[f32], n: usize, hop: usize) -> Vec<Vec<f32>> {
/// STFT frame for the onset envelope: the power of two nearest to 46 ms (1024 at 22.05 kHz, 2048
/// at 44.1/48 kHz).
fn onset_frame(sr: u32) -> usize {
let n = (sr as f64 * 0.046).max(256.0);
// `onset_envelope` is public on its own, so it must remain bounded even outside `analyze`.
let n = (sr.min(MAX_ANALYSIS_SAMPLE_RATE) as f64 * 0.046).max(256.0);
1usize << n.log2().round() as u32
}
@@ -344,31 +361,34 @@ fn beat_features(x: &[f32], sr: u32, beats: &[i64]) -> (Vec<[f32; 12]>, Vec<[f32
}
frame_cep.push(cep);
}
let mut chroma = Vec::with_capacity(beats.len());
let mut cep = Vec::with_capacity(beats.len());
// Frames and beats are both ordered. Assign every frame once instead of scanning the entire
// feature timeline for every beat interval.
let mut chroma = vec![[0f32; 12]; beats.len()];
let mut cep = vec![[0f32; 12]; beats.len()];
let mut counts = vec![0f32; beats.len()];
let end = x.len() as i64;
for (i, &b) in beats.iter().enumerate() {
let e = beats.get(i + 1).copied().unwrap_or(end);
let (mut c, mut q, mut cnt) = ([0f32; 12], [0f32; 12], 0f32);
for (t, (fc, fq)) in frame_chroma.iter().zip(&frame_cep).enumerate() {
let centre = (t * hop + n / 2) as i64;
if centre >= b && centre < e {
for k in 0..12 {
c[k] += fc[k];
q[k] += fq[k];
}
cnt += 1.0;
}
let mut beat = 0;
for (t, (fc, fq)) in frame_chroma.iter().zip(&frame_cep).enumerate() {
let centre = (t * hop + n / 2) as i64;
while beat + 1 < beats.len() && centre >= beats[beat + 1] {
beat += 1;
}
if cnt > 0.0 {
q.iter_mut().for_each(|v| *v /= cnt);
if centre >= beats[beat] && centre < end {
for k in 0..12 {
chroma[beat][k] += fc[k];
cep[beat][k] += fq[k];
}
counts[beat] += 1.0;
}
}
for ((c, q), count) in chroma.iter_mut().zip(&mut cep).zip(counts) {
if count > 0.0 {
q.iter_mut().for_each(|v| *v /= count);
}
let norm = c.iter().map(|v| v * v).sum::<f32>().sqrt();
if norm > 0.0 {
c.iter_mut().for_each(|v| *v /= norm);
}
chroma.push(c);
cep.push(q);
}
// z-score the cepstra across beats
let m = cep.len().max(1) as f32;
@@ -380,6 +400,10 @@ fn beat_features(x: &[f32], sr: u32, beats: &[i64]) -> (Vec<[f32; 12]>, Vec<[f32
(chroma, cep)
}
fn similarity_cells(beats: usize) -> Result<usize, RemixError> {
beats.checked_mul(beats).filter(|&n| n <= MAX_SIMILARITY_CELLS).ok_or(RemixError::TooComplex)
}
fn cos(a: &[f32; 12], b: &[f32; 12]) -> f32 {
let d: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
let na = a.iter().map(|v| v * v).sum::<f32>().sqrt();
@@ -398,7 +422,13 @@ fn local_max(env: &[f32], t: usize, r: usize) -> f32 {
/// Analyse planar audio (channels are summed to mono).
pub fn analyze(channels: &[&[f32]], sample_rate: u32) -> Result<Analysis, RemixError> {
if sample_rate == 0 || sample_rate > MAX_ANALYSIS_SAMPLE_RATE {
return Err(RemixError::InvalidSampleRate);
}
let len = channels.iter().map(|c| c.len()).max().unwrap_or(0);
if (len as u64) > u64::from(sample_rate).saturating_mul(MAX_ANALYSIS_SECONDS) {
return Err(RemixError::TooLong);
}
let k = 1.0 / channels.len().max(1) as f32;
let mono: Vec<f32> = (0..len).map(|i| channels.iter().map(|c| c.get(i).copied().unwrap_or(0.0)).sum::<f32>() * k).collect();
let (env, n, hop) = onset_envelope(&mono, sample_rate);
@@ -434,7 +464,7 @@ pub fn analyze(channels: &[&[f32]], sample_rate: u32) -> Result<Analysis, RemixE
}
let (chroma, cep) = beat_features(&mono, sample_rate, &beats);
let m = beats.len();
let mut sim = vec![0f32; m * m];
let mut sim = vec![0f32; similarity_cells(m)?];
for i in 0..m {
for j in 0..m {
sim[i * m + j] = 0.6 * cos(&chroma[i], &chroma[j]) + 0.4 * (1.0 + cos(&cep[i], &cep[j])) * 0.5;
@@ -796,6 +826,22 @@ mod tests {
assert!(analyze(&[&noise], SR).is_err());
}
#[test]
fn analysis_duration_and_similarity_storage_are_bounded() {
let too_long = vec![0.0; MAX_ANALYSIS_SECONDS as usize + 1];
assert_eq!(analyze(&[&too_long], 1), Err(RemixError::TooLong));
assert_eq!(similarity_cells(2_000), Ok(4_000_000));
assert_eq!(similarity_cells(2_001), Err(RemixError::TooComplex));
}
#[test]
fn public_analysis_rate_is_bounded_before_fft_planning() {
assert_eq!(analyze(&[], 0), Err(RemixError::InvalidSampleRate));
assert_eq!(analyze(&[], u32::MAX), Err(RemixError::InvalidSampleRate));
let (_, n, _) = onset_envelope(&[], u32::MAX);
assert!(n <= onset_frame(MAX_ANALYSIS_SAMPLE_RATE));
}
#[test]
fn render_crossfades_without_clicks_and_ignores_block_size() {
let (m, _, a) = music(120.0, 24);
crates/audio-dsp/src/sync.rs
+12 -1
View File
@@ -77,9 +77,13 @@ pub fn mixdown(channels: &[&[f32]]) -> Vec<f32> {
/// Low-pass filter (Blackman-windowed sinc, cutoff at 0.42 × the new sample rate) and keep every
/// `d`-th sample.
pub fn decimate(x: &[f32], d: usize) -> Vec<f32> {
if d <= 1 {
if x.is_empty() || d <= 1 {
return x.to_vec();
}
// Factors above the input length cannot produce another output sample. Cap the useful factor
// as well as the filter width so a public caller cannot turn a tiny input into a huge plan.
const MAX_FACTOR: usize = 4096;
let d = d.min(x.len()).min(MAX_FACTOR);
let half = 8 * d;
let fc = 0.42 / d as f64; // cycles per input sample
let taps: Vec<f32> = {
@@ -396,6 +400,13 @@ mod tests {
assert!(rh < 0.01, "{rh}");
}
#[test]
fn decimate_does_not_plan_a_filter_for_empty_or_extreme_requests() {
assert!(decimate(&[], usize::MAX).is_empty());
let tiny = [1.0, -1.0];
assert_eq!(decimate(&tiny, usize::MAX), decimate(&tiny, tiny.len()));
}
#[test]
fn pure_delay_exact_both_signs() {
let sr = 48_000;
crates/engine/src/remix.rs
+5 -1
View File
@@ -102,13 +102,17 @@ fn analysis(s: &Session, item: ItemId, source_in: Tick, dur: Tick) -> Result<Arc
let sr = s.active_sequence().map(|q| q.settings.sample_rate).unwrap_or(48_000).max(1);
let m0 = source_in.to_units_floor(sr as i64);
let len = (source_in + dur).to_units_floor(sr as i64) - m0;
let max = i64::try_from(u64::from(sr).saturating_mul(dsp::MAX_ANALYSIS_SECONDS)).unwrap_or(i64::MAX);
if !(0..=max).contains(&len) {
return Err(EngineError::Other(format!("cannot remix: audio is longer than {} minutes", dsp::MAX_ANALYSIS_SECONDS / 60)));
}
let key = (item.0, m0, len, sr);
if let Some((_, a)) = cache().lock().unwrap_or_else(|e| e.into_inner()).iter().find(|(k, _)| *k == key) {
return Ok(a.clone());
}
let provider = s.media.provider(s.project.clone(), s.services.clone());
let src = filmcraft_render::SourceProvider::source(&provider, item).ok_or_else(|| EngineError::Other("the clip's media is offline".into()))?;
let buf = src.audio(m0, len.max(0) as usize, sr).map_err(|e| EngineError::Other(format!("reading the clip's audio: {e}")))?;
let buf = src.audio(m0, len as usize, sr).map_err(|e| EngineError::Other(format!("reading the clip's audio: {e}")))?;
let refs: Vec<&[f32]> = buf.channels.iter().map(Vec::as_slice).collect();
let a = Arc::new(dsp::analyze(&refs, sr).map_err(|e| EngineError::Other(format!("cannot remix: {e}")))?);
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());