mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-30 13:08:07 -06:00
Media tests: every AAC layout mixes by speaker in MP4, Matroska and ADTS (#1701)
Tones encoded with FilmCraft's AAC encoder, with the configuration rewritten to 11, 12 or ffmpeg's 6.1 PCE where the coded elements fit it, so nothing depends on ffmpeg or skips. Expected gains come from each channel's speaker.
This commit is contained in:
tylerjenningsw
@@ -0,0 +1,224 @@
|
||||
//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel
|
||||
//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no
|
||||
//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded
|
||||
//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and
|
||||
//! 8-channel output, and a program config element can name any of them.
|
||||
|
||||
use std::f64::consts::PI;
|
||||
use std::io::Cursor;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use effectcraft_media::{MediaPool, probe, probe_bytes};
|
||||
use effectcraft_time::Tick;
|
||||
use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig};
|
||||
|
||||
const RATE: u32 = 48_000;
|
||||
/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry.
|
||||
const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700];
|
||||
const LFE_TONE: u32 = 60;
|
||||
const AMP: f32 = 0.25;
|
||||
const K: f64 = std::f64::consts::FRAC_1_SQRT_2;
|
||||
|
||||
/// A channel's content, by where it plays.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum Ch {
|
||||
L,
|
||||
R,
|
||||
C,
|
||||
Lfe,
|
||||
Ls,
|
||||
Rs,
|
||||
Cs,
|
||||
}
|
||||
|
||||
impl Ch {
|
||||
/// ITU-R BS.775 gains into `[left, right]`, before normalisation.
|
||||
fn gains(self) -> [f64; 2] {
|
||||
match self {
|
||||
Ch::L => [1.0, 0.0],
|
||||
Ch::R => [0.0, 1.0],
|
||||
Ch::C => [K, K],
|
||||
Ch::Lfe => [0.0, 0.0],
|
||||
Ch::Ls => [K, 0.0],
|
||||
Ch::Rs => [0.0, K],
|
||||
Ch::Cs => [0.5, 0.5],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
/// What the encoder's input channels carry (its element order: C, L R, …, LFE last).
|
||||
content: &'static [Ch],
|
||||
/// The AudioSpecificConfig written to the container instead of the encoder's, if any.
|
||||
asc: Option<AudioSpecificConfig>,
|
||||
}
|
||||
|
||||
fn lc(config: u8) -> Option<AudioSpecificConfig> {
|
||||
Some(AudioSpecificConfig::lc(3, config))
|
||||
}
|
||||
|
||||
fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec<u8>, comment: &[u8]) -> Option<AudioSpecificConfig> {
|
||||
let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() };
|
||||
Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) })
|
||||
}
|
||||
|
||||
use Ch::*;
|
||||
|
||||
fn cases() -> Vec<Case> {
|
||||
vec![
|
||||
Case { name: "config3", content: &[C, L, R], asc: None },
|
||||
Case { name: "config4", content: &[C, L, R, Cs], asc: None },
|
||||
Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None },
|
||||
Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None },
|
||||
// configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds
|
||||
Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None },
|
||||
Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) },
|
||||
Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) },
|
||||
// the encoder's own 6.1: a PCE listing the centre first
|
||||
Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None },
|
||||
// ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE
|
||||
// element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the
|
||||
// encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag
|
||||
// 1) and the LFE in the side SCE left over (tag 5).
|
||||
Case {
|
||||
name: "pce61-ffmpeg",
|
||||
content: &[C, L, R, Cs, Ls, Rs, Lfe],
|
||||
asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn freq(content: &[Ch], i: usize) -> u32 {
|
||||
if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] }
|
||||
}
|
||||
|
||||
/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units.
|
||||
fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {
|
||||
let n = content.len();
|
||||
let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32);
|
||||
cfg.adts = adts;
|
||||
let mut enc = Encoder::new(cfg).expect("encoder");
|
||||
let planes: Vec<Vec<f32>> =
|
||||
(0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect();
|
||||
let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect();
|
||||
let mut units = enc.encode(&refs);
|
||||
units.extend(enc.flush());
|
||||
(enc.audio_specific_config(), units)
|
||||
}
|
||||
|
||||
fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
|
||||
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
|
||||
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");
|
||||
let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");
|
||||
for u in units {
|
||||
w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");
|
||||
}
|
||||
w.finish().expect("finish").into_inner()
|
||||
}
|
||||
|
||||
fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
|
||||
use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
|
||||
let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");
|
||||
t.codec_private = asc;
|
||||
t.audio = Some((f64::from(RATE), u64::from(channels), None));
|
||||
let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer");
|
||||
let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);
|
||||
for (i, u) in units.iter().enumerate() {
|
||||
w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");
|
||||
}
|
||||
w.finish().expect("finish").into_inner()
|
||||
}
|
||||
|
||||
/// The fixture written to disk as `name`.
|
||||
fn write(name: &str, bytes: &[u8]) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).expect("mkdir");
|
||||
let p = dir.join(name);
|
||||
std::fs::write(&p, bytes).expect("write");
|
||||
p
|
||||
}
|
||||
|
||||
/// Amplitude of the tone at `freq` in one side of interleaved stereo.
|
||||
fn amp(s: &[f32], side: usize, freq: u32) -> f64 {
|
||||
let (mut re, mut im, mut n) = (0.0, 0.0, 0usize);
|
||||
for (i, f) in s.as_chunks::<2>().0.iter().enumerate() {
|
||||
let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE);
|
||||
re += f64::from(f[side]) * w.cos();
|
||||
im += f64::from(f[side]) * w.sin();
|
||||
n += 1;
|
||||
}
|
||||
2.0 * (re * re + im * im).sqrt() / n.max(1) as f64
|
||||
}
|
||||
|
||||
/// Stereo of one second from 0.5 s in, through the media pool as footage reads it.
|
||||
fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec<f32> {
|
||||
pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE)
|
||||
}
|
||||
|
||||
/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger
|
||||
/// side's sum (an LFE nowhere).
|
||||
fn check(label: &str, content: &[Ch], s: &[f32]) {
|
||||
let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect();
|
||||
let norm = raw.iter().map(|g| g[0]).sum::<f64>().max(raw.iter().map(|g| g[1]).sum::<f64>()).max(1.0);
|
||||
for (i, (ch, g)) in content.iter().zip(&raw).enumerate() {
|
||||
for side in 0..2 {
|
||||
let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm);
|
||||
assert!(
|
||||
(got - want).abs() <= f64::from(AMP) * 0.05,
|
||||
"{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}",
|
||||
freq(content, i),
|
||||
["L", "R"][side]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The LFE tone survives encoding, so its absence from the mix means it was left out.
|
||||
fn lfe_is_coded(case: &Case, bytes: Vec<u8>) {
|
||||
if !case.content.contains(&Lfe) {
|
||||
return;
|
||||
}
|
||||
let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open");
|
||||
let buf = src.audio(24_000, RATE as usize, RATE).expect("decode");
|
||||
let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::<Vec<_>>(), 0, LFE_TONE)).fold(0.0, f64::max);
|
||||
assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_aac_layout_mixes_by_speaker_in_mp4() {
|
||||
for case in cases() {
|
||||
let (asc, units) = encode(case.content, false);
|
||||
let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
|
||||
let bytes = mp4(asc, case.content.len() as u32, &units);
|
||||
lfe_is_coded(&case, bytes.clone());
|
||||
let p = write(&format!("{}.mp4", case.name), &bytes);
|
||||
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
|
||||
// in memory (`add_bytes`) as well as in place
|
||||
let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into());
|
||||
let pool = MediaPool::new();
|
||||
pool.add_bytes(&path, bytes.clone());
|
||||
check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matroska_aac_layouts_mix_by_speaker() {
|
||||
for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) {
|
||||
let (asc, units) = encode(case.content, false);
|
||||
let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
|
||||
let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units));
|
||||
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
|
||||
}
|
||||
}
|
||||
|
||||
/// ADTS signals configurations 1 to 7 only, which the channel count tells apart.
|
||||
#[test]
|
||||
fn adts_aac_layouts_mix_by_speaker() {
|
||||
for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) {
|
||||
let (_, units) = encode(case.content, true);
|
||||
let p = write(&format!("{}.aac", case.name), &units.concat());
|
||||
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! The crate's integration tests, one binary (each module is a former `tests/<name>.rs`).
|
||||
|
||||
mod aac_layout;
|
||||
mod downmix;
|
||||
mod exif_orientation;
|
||||
mod footage;
|
||||
|
||||
Reference in New Issue
Block a user