mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-30 13:08:07 -06:00
Media: downmix AAC by where each decoded channel plays (#1701)
The gains of an AAC source now come from the configuration FilmCraft decodes it with, read from MP4/MOV (esds), Matroska (CodecPrivate) and LATM in MPEG TS/PS (its first StreamMuxConfig). Channel configurations 3 to 6, 11 and 12 arrive reordered to WAVE order, 4 as L R C Cs, 7 in element order. A program config element goes by its element positions, except the six structures in which ffmpeg's encoder fills its own channel order across them (an LFE in a side SCE, a back centre in a side pair), recognised by its "Lavc" comment. ADTS can only signal configurations 1 to 7 that FilmCraft decodes, which the channel count tells apart. A decode of fewer channels than the layout keeps its first ones (7 of configuration 7's 8 in MPEG-TS LATM).
This commit is contained in:
tylerjenningsw
Generated
+2
@@ -1637,7 +1637,9 @@ dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-frame",
|
||||
"filmcraft-isobmff",
|
||||
"filmcraft-matroska",
|
||||
"filmcraft-media",
|
||||
"filmcraft-mpegts",
|
||||
"filmcraft-time",
|
||||
"gif",
|
||||
"image",
|
||||
|
||||
@@ -87,6 +87,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09a
|
||||
filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
# container headers read for AAC channel layouts (crates/media)
|
||||
filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency)
|
||||
filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
|
||||
@@ -128,6 +131,8 @@ sys-locale = "0.3"
|
||||
# filmcraft-prores = { path = "../filmcraft/crates/prores" }
|
||||
# filmcraft-aac = { path = "../filmcraft/crates/aac" }
|
||||
# filmcraft-isobmff = { path = "../filmcraft/crates/isobmff" }
|
||||
# filmcraft-matroska = { path = "../filmcraft/crates/matroska" }
|
||||
# filmcraft-mpegts = { path = "../filmcraft/crates/mpegts" }
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
|
||||
@@ -23,6 +23,8 @@ filmcraft-codecs = { workspace = true }
|
||||
filmcraft-media = { workspace = true }
|
||||
filmcraft-aac = { workspace = true }
|
||||
filmcraft-isobmff = { workspace = true }
|
||||
filmcraft-matroska = { workspace = true }
|
||||
filmcraft-mpegts = { workspace = true }
|
||||
filmcraft-frame = { workspace = true }
|
||||
filmcraft-color = { workspace = true }
|
||||
filmcraft-time = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry
|
||||
//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG
|
||||
//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`),
|
||||
//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart
|
||||
//! ([`crate::downmix::Order::Aac`]).
|
||||
|
||||
use std::io;
|
||||
|
||||
use filmcraft_aac::AudioSpecificConfig;
|
||||
use filmcraft_media::ByteReader;
|
||||
|
||||
/// How far into an MPEG TS / PS file its first LATM frame is looked for: the stream is indexed
|
||||
/// as far as this, not through the whole file.
|
||||
const MPEG_HEAD: u64 = 8 << 20;
|
||||
|
||||
/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its
|
||||
/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS.
|
||||
pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option<AudioSpecificConfig> {
|
||||
let asc = match header.and_then(mp4) {
|
||||
Some(asc) => asc,
|
||||
None => {
|
||||
let file = file?;
|
||||
let mut magic = [0u8; 4];
|
||||
file.read_at(0, &mut magic).ok()?;
|
||||
if magic == [0x1A, 0x45, 0xDF, 0xA3] { matroska(file)? } else { latm(file)? }
|
||||
}
|
||||
};
|
||||
AudioSpecificConfig::parse(&asc).ok()
|
||||
}
|
||||
|
||||
fn mp4(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
let file = filmcraft_isobmff::open(bytes).ok()?;
|
||||
let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?;
|
||||
match &track.entries.first()?.codec {
|
||||
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn matroska(file: &dyn ByteReader) -> Option<Vec<u8>> {
|
||||
let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() };
|
||||
let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?;
|
||||
match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec {
|
||||
filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn latm(file: &dyn ByteReader) -> Option<Vec<u8>> {
|
||||
let head = Head(file, file.len().min(MPEG_HEAD));
|
||||
let mpeg = filmcraft_mpegts::open(&head).ok()?;
|
||||
let stream = mpeg.find(filmcraft_mpegts::Kind::Audio, |c| *c == filmcraft_mpegts::Codec::AacLatm)?;
|
||||
let frame = mpeg.read_unit(&head, stream, 0).ok()?;
|
||||
Some(filmcraft_codecs::audio::latm_config(&frame)?.asc)
|
||||
}
|
||||
|
||||
/// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read.
|
||||
struct Head<'a>(&'a dyn ByteReader, u64);
|
||||
|
||||
impl Head<'_> {
|
||||
fn read(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
|
||||
match offset.checked_add(buf.len() as u64) {
|
||||
Some(end) if end <= self.1 => self.0.read_at(offset, buf),
|
||||
_ => Err(io::Error::new(io::ErrorKind::UnexpectedEof, "read past the end")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl filmcraft_matroska::ByteSource for Head<'_> {
|
||||
fn len(&self) -> u64 {
|
||||
self.1
|
||||
}
|
||||
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
|
||||
self.read(offset, buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl filmcraft_mpegts::ByteSource for Head<'_> {
|
||||
fn len(&self) -> u64 {
|
||||
self.1
|
||||
}
|
||||
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
|
||||
self.read(offset, buf)
|
||||
}
|
||||
}
|
||||
+159
-113
@@ -10,17 +10,19 @@
|
||||
//! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and
|
||||
//! stereo is untouched.
|
||||
//!
|
||||
//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM,
|
||||
//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs,
|
||||
//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC arrives in either
|
||||
//! that order or AAC's own (C L R first, LFE last) depending on its channel configuration
|
||||
//! ([`aac_order`]). More than 8 channels carry no known layout: only the first two (front L/R)
|
||||
//! are used.
|
||||
//! A decoded channel's gains come from where it plays. WAVE / SMPTE order by channel count is what
|
||||
//! FilmCraft's AC-3, PCM, FLAC and Opus readers deliver: 3 = L R C, 4 = quad L R Ls Rs,
|
||||
//! 5 = L R C Ls Rs, 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC
|
||||
//! arrives in an order its configuration decides ([`aac_order`]). More than 8 channels in WAVE
|
||||
//! order carry no known layout: only the first two (front L/R) are used.
|
||||
|
||||
use std::f32::consts::FRAC_1_SQRT_2;
|
||||
use std::sync::Arc;
|
||||
|
||||
use filmcraft_aac::{AudioSpecificConfig, ProgramConfig};
|
||||
|
||||
/// A source channel's contribution to the left and right outputs, before normalisation.
|
||||
type Pair = [f32; 2];
|
||||
pub type Pair = [f32; 2];
|
||||
|
||||
const FL: Pair = [1.0, 0.0];
|
||||
const FR: Pair = [0.0, 1.0];
|
||||
@@ -30,33 +32,18 @@ const SL: Pair = [FRAC_1_SQRT_2, 0.0];
|
||||
const SR: Pair = [0.0, FRAC_1_SQRT_2];
|
||||
const CS: Pair = [0.5, 0.5];
|
||||
|
||||
/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised.
|
||||
/// `aac` selects AAC's native channel order. Empty for zero channels.
|
||||
pub fn stereo_gains(count: usize, aac: bool) -> Vec<Pair> {
|
||||
let raw: &[Pair] = match (count, aac) {
|
||||
(0, _) => &[],
|
||||
(1, _) => &[[1.0, 1.0]],
|
||||
(2, _) => &[FL, FR],
|
||||
(3, false) => &[FL, FR, FC],
|
||||
(3, true) => &[FC, FL, FR],
|
||||
(4, false) => &[FL, FR, SL, SR],
|
||||
(4, true) => &[FC, FL, FR, CS],
|
||||
(5, false) => &[FL, FR, FC, SL, SR],
|
||||
(5, true) => &[FC, FL, FR, SL, SR],
|
||||
(6, false) => &[FL, FR, FC, LFE, SL, SR],
|
||||
(6, true) => &[FC, FL, FR, SL, SR, LFE],
|
||||
(7, false) => &[FL, FR, FC, LFE, CS, SL, SR],
|
||||
(7, true) => &[FC, FL, FR, SL, SR, CS, LFE],
|
||||
(8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR],
|
||||
(8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE],
|
||||
_ => &[FL, FR],
|
||||
};
|
||||
/// AAC channel configuration 7 as FilmCraft decodes it (element order, not reordered).
|
||||
const CONFIG_7: [Pair; 8] = [FC, FL, FR, SL, SR, SL, SR, LFE];
|
||||
/// AAC channel configuration 4 (C L R Cs), which FilmCraft reorders to L R C Cs.
|
||||
const CONFIG_4: [Pair; 4] = [FL, FR, FC, CS];
|
||||
|
||||
/// `raw` scaled by the reciprocal of its larger row sum when that exceeds 1 (more than two
|
||||
/// channels): full-scale input can never exceed full scale.
|
||||
fn normalised(raw: &[Pair]) -> Vec<Pair> {
|
||||
let mut out = raw.to_vec();
|
||||
// Channels beyond a known layout (more than 8) are silent.
|
||||
out.resize(count, [0.0, 0.0]);
|
||||
let peak = out.iter().map(|g| g[0]).sum::<f32>().max(out.iter().map(|g| g[1]).sum::<f32>());
|
||||
// Mono is deliberately left at unity on both sides; so is anything already within range.
|
||||
if count > 2 && peak > 1.0 {
|
||||
if out.len() > 2 && peak > 1.0 {
|
||||
for g in &mut out {
|
||||
g[0] /= peak;
|
||||
g[1] /= peak;
|
||||
@@ -65,63 +52,109 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec<Pair> {
|
||||
out
|
||||
}
|
||||
|
||||
/// The order a source's decoded channels arrive in.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
/// The per-channel `[left, right]` gains for `count` channels in WAVE / SMPTE order, normalised.
|
||||
/// Empty for zero channels.
|
||||
pub fn stereo_gains(count: usize) -> Vec<Pair> {
|
||||
let raw: &[Pair] = match count {
|
||||
0 => &[],
|
||||
1 => &[[1.0, 1.0]],
|
||||
2 => &[FL, FR],
|
||||
3 => &[FL, FR, FC],
|
||||
4 => &[FL, FR, SL, SR],
|
||||
5 => &[FL, FR, FC, SL, SR],
|
||||
6 => &[FL, FR, FC, LFE, SL, SR],
|
||||
7 => &[FL, FR, FC, LFE, CS, SL, SR],
|
||||
8 => &[FL, FR, FC, LFE, SL, SR, SL, SR],
|
||||
_ => &[FL, FR],
|
||||
};
|
||||
let mut raw = raw.to_vec();
|
||||
// Channels beyond a known layout (more than 8) are silent.
|
||||
raw.resize(count, [0.0, 0.0]);
|
||||
normalised(&raw)
|
||||
}
|
||||
|
||||
/// Where a source's decoded channels play.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub enum Order {
|
||||
/// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest.
|
||||
#[default]
|
||||
Wav,
|
||||
/// AAC that FilmCraft reorders to WAVE / SMPTE (configurations 3 to 6, 11 and 12; 4 becomes
|
||||
/// L R C Cs) or that ffmpeg wrote in it (a program config element listing the front pair
|
||||
/// first: ffmpeg fills a PCE's elements in its own channel order and reads them back so).
|
||||
AacWav,
|
||||
/// AAC in its element order (C L R … LFE), which FilmCraft keeps: configuration 7 and a PCE
|
||||
/// listing the centre first.
|
||||
AacElements,
|
||||
/// AAC whose configuration the container doesn't show us (only MP4 / MOV headers are read):
|
||||
/// as [`Order::AacWav`] up to 7 channels (configurations 3 to 6, 11 and ffmpeg's 6.1 PCE),
|
||||
/// as [`Order::AacElements`] above (configuration 7, ffmpeg's 7.1).
|
||||
/// AAC whose configuration wasn't read: by channel count as the configurations 1 to 7 decode
|
||||
/// (exact for ADTS, which can signal no other configuration FilmCraft decodes), 7 channels as
|
||||
/// configuration 11.
|
||||
Aac,
|
||||
/// Each decoded channel's gains, before normalisation.
|
||||
Speakers(Arc<[Pair]>),
|
||||
}
|
||||
|
||||
/// How FilmCraft's decoder hands over the AAC described by the AudioSpecificConfig `asc`.
|
||||
pub fn aac_order(asc: &[u8]) -> Order {
|
||||
let Ok(asc) = filmcraft_aac::AudioSpecificConfig::parse(asc) else { return Order::Aac };
|
||||
match (asc.channel_config, asc.pce) {
|
||||
(3..=6 | 11 | 12, _) => Order::AacWav,
|
||||
(0, Some(pce)) if pce.front.first().is_some_and(|&(cpe, _)| cpe) => Order::AacWav,
|
||||
(0, None) => Order::Aac,
|
||||
_ => Order::AacElements,
|
||||
/// How FilmCraft's decoder hands over AAC with the configuration `asc`: it reorders channel
|
||||
/// configurations 3 to 6, 11 and 12 to WAVE / SMPTE and leaves 7 and PCE layouts in element order.
|
||||
pub fn aac_order(asc: &AudioSpecificConfig) -> Order {
|
||||
match (asc.channel_config, &asc.pce) {
|
||||
(4, _) => Order::Speakers(CONFIG_4.into()),
|
||||
(1..=6 | 11 | 12, _) => Order::Wav,
|
||||
(7, _) => Order::Speakers(CONFIG_7.into()),
|
||||
(0, Some(pce)) => Order::Speakers(pce_speakers(pce).into()),
|
||||
_ => Order::Aac,
|
||||
}
|
||||
}
|
||||
|
||||
/// The gains for `count` channels arriving in `order`.
|
||||
pub fn decoded_gains(count: usize, order: Order) -> Vec<Pair> {
|
||||
let order = match order {
|
||||
Order::Aac if (3..=7).contains(&count) => Order::AacWav,
|
||||
Order::Aac => Order::AacElements,
|
||||
o => o,
|
||||
};
|
||||
match (order, count) {
|
||||
// C L R Cs, reordered to L R C Cs: the same gains, permuted.
|
||||
(Order::AacWav, 4) => {
|
||||
let g = stereo_gains(4, true);
|
||||
[1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect()
|
||||
/// ffmpeg's encoder fills a program config element's elements with its layout's channels in its
|
||||
/// own order (L R C LFE …), and for these structures that order doesn't match the element
|
||||
/// positions: an LFE in a side SCE, a back centre in a side pair. Front, side and back element
|
||||
/// types (`true` for a pair) and the channels of the ffmpeg layout with that structure (the first
|
||||
/// ffmpeg 8.1 lists, where several share it: 6.1 for 6.1(back) and 6.1(front), 7.1(wide) for
|
||||
/// 7.1(wide-side)).
|
||||
const FFMPEG_PCE: [(&[bool], &[bool], &[bool], &[Pair]); 6] = [
|
||||
(&[true, false], &[false], &[false], &[FL, FR, FC, LFE, CS]),
|
||||
(&[true, false], &[false], &[true], &[FL, FR, FC, LFE, SL, SR]),
|
||||
(&[true, false], &[true], &[false], &[FL, FR, FC, CS, SL, SR]),
|
||||
(&[true, false], &[false], &[true, false], &[FL, FR, FC, LFE, CS, SL, SR]),
|
||||
(&[true, false], &[false], &[true, true], &[FL, FR, FC, LFE, SL, SR, FL, FR]),
|
||||
(&[true, false], &[true], &[true, false], &[FL, FR, FC, SL, SR, CS, SL, SR]),
|
||||
];
|
||||
|
||||
/// The channels of a PCE layout in decoded (element) order: ffmpeg's for its structures above
|
||||
/// (it names itself in the PCE's comment), otherwise by position (ISO/IEC 14496-3's program config element):
|
||||
/// front pairs left and right, a front single the centre, side and back pairs surrounds, a side
|
||||
/// or back single a back centre.
|
||||
fn pce_speakers(pce: &ProgramConfig) -> Vec<Pair> {
|
||||
let pairs = |els: &[(bool, u8)]| els.iter().map(|e| e.0).collect::<Vec<_>>();
|
||||
let (front, side, back) = (pairs(&pce.front), pairs(&pce.side), pairs(&pce.back));
|
||||
if pce.comment.starts_with(b"Lavc") && pce.lfe.is_empty() {
|
||||
let known = FFMPEG_PCE.iter().find(|(f, s, b, _)| *f == front.as_slice() && *s == side.as_slice() && *b == back.as_slice());
|
||||
if let Some((.., speakers)) = known {
|
||||
return speakers.to_vec();
|
||||
}
|
||||
(Order::AacElements, _) => stereo_gains(count, true),
|
||||
_ => stereo_gains(count, false),
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for pair in front {
|
||||
out.extend_from_slice(if pair { &[FL, FR] } else { &[FC] });
|
||||
}
|
||||
for pair in side.into_iter().chain(back) {
|
||||
out.extend_from_slice(if pair { &[SL, SR] } else { &[CS] });
|
||||
}
|
||||
out.extend(pce.lfe.iter().map(|_| LFE));
|
||||
out
|
||||
}
|
||||
|
||||
/// The gains for `count` channels arriving in `order`. FilmCraft decodes as many channels as the
|
||||
/// container names, the first ones of the layout where that is fewer (MPEG-TS LATM names 7 of
|
||||
/// configuration 7's 8, ffmpeg 4.2 MP4s 2 of a PCE's); more go by count.
|
||||
pub fn decoded_gains(count: usize, order: &Order) -> Vec<Pair> {
|
||||
match order {
|
||||
Order::Speakers(s) if count <= s.len() => normalised(s.get(..count).unwrap_or_default()),
|
||||
Order::Aac if count == 4 => normalised(&CONFIG_4),
|
||||
Order::Aac if count == 8 => normalised(&CONFIG_7),
|
||||
_ => stereo_gains(count),
|
||||
}
|
||||
}
|
||||
|
||||
/// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2`
|
||||
/// frames); frames a channel is too short for count as silence, extra frames in `out` stay as
|
||||
/// they are.
|
||||
pub fn downmix_into(channels: &[Vec<f32>], order: Order, out: &mut [f32]) {
|
||||
mix(channels, &decoded_gains(channels.len(), order), out);
|
||||
}
|
||||
|
||||
fn mix(channels: &[Vec<f32>], gains: &[Pair], out: &mut [f32]) {
|
||||
for (ch, g) in channels.iter().zip(gains) {
|
||||
pub fn downmix_into(channels: &[Vec<f32>], order: &Order, out: &mut [f32]) {
|
||||
for (ch, g) in channels.iter().zip(&decoded_gains(channels.len(), order)) {
|
||||
if g[0] == 0.0 && g[1] == 0.0 {
|
||||
continue;
|
||||
}
|
||||
@@ -138,13 +171,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn every_channel_count_has_a_bounded_defined_matrix() {
|
||||
for aac in [false, true] {
|
||||
let pce = Order::Speakers([FL, FR, FC, FC, SL, SR, SL, SR, CS, LFE].into());
|
||||
for order in [Order::Wav, Order::Aac, Order::Speakers(CONFIG_7.into()), pce] {
|
||||
for n in 0..40usize {
|
||||
let g = stereo_gains(n, aac);
|
||||
let g = decoded_gains(n, &order);
|
||||
assert_eq!(g.len(), n);
|
||||
let (l, r) = (g.iter().map(|p| p[0]).sum::<f32>(), g.iter().map(|p| p[1]).sum::<f32>());
|
||||
if n > 2 {
|
||||
assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}");
|
||||
assert!(l <= 1.0001 && r <= 1.0001, "{n} channels ({order:?}): {l} {r}");
|
||||
assert!(l > 0.0 && r > 0.0);
|
||||
}
|
||||
}
|
||||
@@ -155,75 +189,87 @@ mod tests {
|
||||
const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500";
|
||||
const PCE_FILMCRAFT_61: &str = "118004c845000108840000";
|
||||
|
||||
fn asc(hex: &str) -> Vec<u8> {
|
||||
(0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect()
|
||||
fn order(hex: &str) -> Order {
|
||||
let asc: Vec<u8> = (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect();
|
||||
aac_order(&AudioSpecificConfig::parse(&asc).expect("an AudioSpecificConfig"))
|
||||
}
|
||||
|
||||
fn speakers(order: &Order) -> Vec<Pair> {
|
||||
match order {
|
||||
Order::Speakers(s) => s.to_vec(),
|
||||
o => panic!("{o:?} names no speakers"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aac_order_follows_the_channel_configuration() {
|
||||
// AAC-LC at 48 kHz with channel configuration n
|
||||
let config = |n: u8| aac_order(&[0x11, 0x80 | n << 3]);
|
||||
for n in [3, 4, 5, 6, 11, 12] {
|
||||
assert_eq!(config(n), Order::AacWav, "configuration {n}");
|
||||
for n in [1, 2, 3, 5, 6, 11, 12] {
|
||||
assert_eq!(aac_order(&AudioSpecificConfig::lc(3, n)), Order::Wav, "configuration {n}");
|
||||
}
|
||||
for n in [1, 2, 7] {
|
||||
assert_eq!(config(n), Order::AacElements, "configuration {n}");
|
||||
}
|
||||
assert_eq!(aac_order(&asc(PCE_FFMPEG_61)), Order::AacWav, "ffmpeg's 6.1 PCE lists L R first");
|
||||
assert_eq!(aac_order(&asc(PCE_FILMCRAFT_61)), Order::AacElements, "a PCE listing the centre first");
|
||||
assert_eq!(aac_order(&[]), Order::Aac);
|
||||
assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 4))), [FL, FR, FC, CS]);
|
||||
assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 7))), [FC, FL, FR, SL, SR, SL, SR, LFE]);
|
||||
assert_eq!(speakers(&order(PCE_FFMPEG_61)), [FL, FR, FC, LFE, CS, SL, SR], "ffmpeg's 6.1 carries the LFE in a side SCE");
|
||||
assert_eq!(speakers(&order(PCE_FILMCRAFT_61)), [FC, FL, FR, SL, SR, CS, LFE], "a centre-first PCE by position");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_aac_gains_follow_the_order() {
|
||||
for n in [3, 5, 6, 7, 8] {
|
||||
assert_eq!(decoded_gains(n, Order::AacWav), stereo_gains(n, false), "{n} channels");
|
||||
assert_eq!(decoded_gains(n, Order::AacElements), stereo_gains(n, true), "{n} channels");
|
||||
}
|
||||
let g = decoded_gains(4, Order::AacWav);
|
||||
assert_eq!(g.len(), 4);
|
||||
assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first");
|
||||
assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides");
|
||||
assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre");
|
||||
assert_eq!(decoded_gains(4, Order::Wav), stereo_gains(4, false), "PCM 4 channels are quad");
|
||||
for n in [3, 4, 5, 6, 7] {
|
||||
assert_eq!(decoded_gains(n, Order::Aac), decoded_gains(n, Order::AacWav), "{n} channels");
|
||||
}
|
||||
for n in [1, 2, 8, 9] {
|
||||
assert_eq!(decoded_gains(n, Order::Aac), stereo_gains(n, true), "{n} channels");
|
||||
fn only_ffmpeg_pce_structures_it_misplaces_take_its_channel_order() {
|
||||
let pce = |comment: &[u8], side: Vec<(bool, u8)>, lfe: Vec<u8>| ProgramConfig {
|
||||
front: vec![(true, 0), (false, 0)],
|
||||
side,
|
||||
back: vec![(true, 1)],
|
||||
lfe,
|
||||
comment: comment.to_vec(),
|
||||
..Default::default()
|
||||
};
|
||||
// 5.1(side): ffmpeg's LFE sits in the side SCE
|
||||
assert_eq!(pce_speakers(&pce(b"Lavc58.54.100", vec![(false, 1)], vec![])), [FL, FR, FC, LFE, SL, SR]);
|
||||
// the same structure from another encoder: a back centre by position
|
||||
assert_eq!(pce_speakers(&pce(b"", vec![(false, 1)], vec![])), [FL, FR, FC, CS, SL, SR]);
|
||||
// 7.0: ffmpeg's order matches the positions
|
||||
assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(true, 2)], vec![])), [FL, FR, FC, SL, SR, SL, SR]);
|
||||
// a real LFE element is never ffmpeg's misplaced one
|
||||
assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(false, 1)], vec![0])), [FL, FR, FC, CS, SL, SR, LFE]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_decode_of_fewer_channels_takes_the_first_of_the_layout_and_more_go_by_count() {
|
||||
assert_eq!(decoded_gains(2, &order(PCE_FILMCRAFT_61)), [FC, FL]);
|
||||
assert_eq!(decoded_gains(7, &Order::Speakers(CONFIG_7.into())), normalised(&CONFIG_7[..7]));
|
||||
assert_eq!(decoded_gains(9, &Order::Speakers(CONFIG_7.into())), stereo_gains(9));
|
||||
assert_eq!(decoded_gains(4, &Order::Aac), normalised(&[FL, FR, FC, CS]));
|
||||
assert_eq!(decoded_gains(8, &Order::Aac), normalised(&CONFIG_7));
|
||||
for n in [3, 5, 6, 7] {
|
||||
assert_eq!(decoded_gains(n, &Order::Aac), stereo_gains(n), "{n} channels");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn five_one_matches_the_bs775_matrix() {
|
||||
let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2);
|
||||
let g = stereo_gains(6, false);
|
||||
let g = stereo_gains(6);
|
||||
assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0);
|
||||
assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6);
|
||||
assert_eq!(g[3], [0.0, 0.0], "LFE is left out");
|
||||
assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0);
|
||||
assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0);
|
||||
// AAC order: the centre is first, LFE last
|
||||
let a = stereo_gains(6, true);
|
||||
assert_eq!((a[0], a[5]), (g[2], g[3]));
|
||||
assert_eq!((a[1], a[2]), (g[0], g[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() {
|
||||
assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]);
|
||||
assert_eq!(stereo_gains(2, true), vec![FL, FR]);
|
||||
let g = stereo_gains(12, false);
|
||||
assert_eq!(stereo_gains(1), vec![[1.0, 1.0]]);
|
||||
assert_eq!(stereo_gains(2), vec![FL, FR]);
|
||||
let g = stereo_gains(12);
|
||||
assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_and_empty_input_never_panics() {
|
||||
let mut out = vec![0.0; 8];
|
||||
downmix_into(&[], Order::Wav, &mut out);
|
||||
downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], Order::Aac, &mut out);
|
||||
downmix_into(&[], &Order::Wav, &mut out);
|
||||
downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], &Order::Aac, &mut out);
|
||||
assert!(out.iter().all(|v| v.is_finite()));
|
||||
let mut none: [f32; 0] = [];
|
||||
downmix_into(&vec![vec![1.0; 3]; 7], Order::Wav, &mut none);
|
||||
downmix_into(&vec![vec![1.0; 3]; 7], &Order::Speakers(CONFIG_7.into()), &mut none);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod aac;
|
||||
mod convert;
|
||||
pub mod downmix;
|
||||
mod dpx;
|
||||
|
||||
+17
-19
@@ -193,21 +193,20 @@ struct Opened {
|
||||
}
|
||||
|
||||
impl Opened {
|
||||
/// `src` with what its MP4/MOV `header` (ftyp and moov) says, if it has one.
|
||||
fn new(src: SharedSource, header: Option<&[u8]>) -> Opened {
|
||||
/// `src` with what its container says: `header` is an MP4/MOV's ftyp and moov (or the whole
|
||||
/// file), `file` the file for other containers' AAC configuration.
|
||||
fn new(src: SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Opened {
|
||||
let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn);
|
||||
let order = channel_order(&src, header.and_then(crate::probe::aac_config).as_deref());
|
||||
Opened { src, turn, order }
|
||||
Opened { order: channel_order(&src, header, file), src, turn }
|
||||
}
|
||||
}
|
||||
|
||||
/// The order `src`'s audio channels decode in, AAC's decided by its AudioSpecificConfig `asc`
|
||||
/// when the container's header was read.
|
||||
fn channel_order(src: &SharedSource, asc: Option<&[u8]>) -> Order {
|
||||
match (src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")), asc) {
|
||||
(false, _) => Order::Wav,
|
||||
(true, Some(asc)) => crate::downmix::aac_order(asc),
|
||||
(true, None) => Order::Aac,
|
||||
/// The order `src`'s audio channels decode in; multichannel AAC's by its configuration, where
|
||||
/// the container's header shows it ([`crate::aac::config`]).
|
||||
fn channel_order(src: &SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Order {
|
||||
match src.info().audio() {
|
||||
Some(a) if a.codec.contains("AAC") && a.channels > 2 => crate::aac::config(header, file).map_or(Order::Aac, |asc| crate::downmix::aac_order(&asc)),
|
||||
_ => Order::Wav,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -551,14 +550,14 @@ impl MediaPool {
|
||||
return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz"));
|
||||
}
|
||||
if native == 0 || rate == 0 || native == rate {
|
||||
return read_stereo(&src, order, s0, frames, rate);
|
||||
return read_stereo(&src, &order, s0, frames, rate);
|
||||
}
|
||||
// Output sample `n` sits at source position n × native / rate.
|
||||
let (native_i, rate_i) = (i128::from(native), i128::from(rate));
|
||||
let first = (i128::from(s0) * native_i).div_euclid(rate_i);
|
||||
let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1;
|
||||
let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) };
|
||||
let buf = read_stereo(&src, order, first64, count, native)?;
|
||||
let buf = read_stereo(&src, &order, first64, count, native)?;
|
||||
let mut out = Vec::with_capacity(frames * 2);
|
||||
for k in 0..frames as i128 {
|
||||
let pos = (i128::from(s0) + k) * native_i;
|
||||
@@ -636,15 +635,14 @@ impl Inner {
|
||||
let file = std::path::Path::new(&**path);
|
||||
let in_place = if in_memory { None } else { crate::stream::open_source(file) };
|
||||
let in_place = in_place.map(|src| {
|
||||
let header = crate::stream::movie_header(file);
|
||||
src.map(|src| Opened::new(src, header.as_deref()))
|
||||
let (header, reader) = (crate::stream::movie_header(file), crate::stream::file_reader(file));
|
||||
src.map(|src| Opened::new(src, header.as_deref(), reader.as_deref()))
|
||||
});
|
||||
let opened = in_place.unwrap_or_else(|| {
|
||||
self.read(path).and_then(|bytes| {
|
||||
let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let (turn, asc) = (crate::probe::video_turn(&bytes), crate::probe::aac_config(&bytes));
|
||||
let src = filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)?;
|
||||
Ok(Opened { order: channel_order(&src, asc.as_deref()), src, turn })
|
||||
let src = filmcraft_codecs::open_bytes(&name, bytes.clone()).map_err(MediaError::from)?;
|
||||
Ok(Opened::new(src, Some(&bytes), Some(&filmcraft_media::reader::MemReader(bytes.clone()))))
|
||||
})
|
||||
});
|
||||
match opened {
|
||||
@@ -978,7 +976,7 @@ impl FootageSource for MediaPool {
|
||||
/// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which
|
||||
/// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both
|
||||
/// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails.
|
||||
fn read_stereo(src: &SharedSource, order: Order, s0: i64, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {
|
||||
fn read_stereo(src: &SharedSource, order: &Order, s0: i64, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {
|
||||
let mut out = vec![0.0; frames * 2];
|
||||
let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0);
|
||||
if skip >= frames {
|
||||
|
||||
@@ -259,17 +259,6 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option<image::ImageFormat>) -> im
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// The AudioSpecificConfig of an MP4/MOV movie's first audio track (the one FilmCraft plays), if
|
||||
/// that is AAC.
|
||||
pub(crate) fn aac_config(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
let file = filmcraft_isobmff::open(bytes).ok()?;
|
||||
let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?;
|
||||
match &track.entries.first()?.codec {
|
||||
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and
|
||||
/// says so in its video track's header): the quarter turn nearest the track's rotation. Only the
|
||||
/// rotation counts; any mirroring, scaling or offset in the header is ignored, and other
|
||||
|
||||
@@ -85,6 +85,17 @@ pub(crate) fn open_source(_path: &Path) -> Option<Result<SharedSource>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// `path` read in place (`None` when it can't be opened, or without a native file system).
|
||||
#[cfg(any(unix, windows))]
|
||||
pub(crate) fn file_reader(path: &Path) -> Option<Box<dyn filmcraft_media::ByteReader>> {
|
||||
Some(Box::new(FileReader::open(path).ok()?))
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub(crate) fn file_reader(_path: &Path) -> Option<Box<dyn filmcraft_media::ByteReader>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// The leading `ftyp` and the `moov` box of an ISO-BMFF movie on disk (all a track's display
|
||||
/// matrix needs), without reading the samples: top-level box headers are walked with positional
|
||||
/// reads. `None` when the file isn't ISO-BMFF or its `moov` is missing or implausibly large.
|
||||
|
||||
@@ -915,7 +915,7 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() {
|
||||
// ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries
|
||||
// channels 0 and 1 at full level and the others at half, so a multichannel file sounds as
|
||||
// its stereo downmix (tests/it/downmix.rs checks that matrix channel by channel).
|
||||
let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false);
|
||||
let gains = effectcraft_media::downmix::stereo_gains(channels as usize);
|
||||
let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::<f64>();
|
||||
let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt();
|
||||
let rms = (sum / n.max(1) as f64).sqrt();
|
||||
|
||||
Reference in New Issue
Block a user