diff --git a/Cargo.lock b/Cargo.lock index a48f2d44b..629fc65e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,18 +1633,20 @@ dependencies = [ "effectcraft-svg", "effectcraft-time", "exr", + "filmcraft-aac", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", "filmcraft-isobmff", + "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpegts", "filmcraft-time", "gif", "image", "log", "rayon", "serde_json", - "symphonia", "thiserror 2.0.21", ] @@ -2246,8 +2248,8 @@ dependencies = [ [[package]] name = "filmcraft-aac" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2255,40 +2257,57 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] -name = "filmcraft-av1" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +name = "filmcraft-apv" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-av1" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + +[[package]] +name = "filmcraft-avi" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" + [[package]] name = "filmcraft-bitstream" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-cfb" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-codecs" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-aac", "filmcraft-ac3", + "filmcraft-apv", "filmcraft-av1", + "filmcraft-avi", "filmcraft-bitstream", "filmcraft-color", "filmcraft-dnx", @@ -2298,6 +2317,7 @@ dependencies = [ "filmcraft-isobmff", "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpa", "filmcraft-mpeg2v", "filmcraft-mpegts", "filmcraft-mxf", @@ -2305,9 +2325,11 @@ dependencies = [ "filmcraft-opus", "filmcraft-prores", "filmcraft-time", + "filmcraft-vp8", "filmcraft-vp9", "image", "log", + "rayon", "symphonia", "thiserror 2.0.21", "web-time", @@ -2315,16 +2337,25 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-dng" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-dnx" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2333,8 +2364,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2344,16 +2375,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2362,8 +2393,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2372,8 +2403,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2382,8 +2413,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2399,8 +2430,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2408,15 +2439,16 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-media" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", + "filmcraft-dng", "filmcraft-frame", "filmcraft-time", "image", @@ -2425,10 +2457,19 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-mpa" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "filmcraft-bitstream", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-mpeg2v" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2436,31 +2477,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-ogg" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-opus" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-project" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2473,8 +2514,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2483,16 +2524,24 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-vp8" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-vp9" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index dee46306d..c101c489c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,29 +77,29 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +# container headers read for AAC channel layouts (crates/media) +filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -# The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned -# rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that. -symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -131,6 +131,8 @@ sys-locale = "0.3" # filmcraft-prores = { path = "../filmcraft/crates/prores" } # filmcraft-aac = { path = "../filmcraft/crates/aac" } # filmcraft-isobmff = { path = "../filmcraft/crates/isobmff" } +# filmcraft-matroska = { path = "../filmcraft/crates/matroska" } +# filmcraft-mpegts = { path = "../filmcraft/crates/mpegts" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index a58507161..2171bfe57 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,7 +94,9 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); - assert_eq!((data.len(), fnv1a(&data)), (104_180, 8_728_430_532_616_484_528), "H.264 MP4"); + // FilmCraft 09aaf9b's encoder lists its settings in the first frame's user-data SEI (211 + // bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin. + assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); } #[test] diff --git a/crates/export/tests/it/vp9_roundtrip.rs b/crates/export/tests/it/vp9_roundtrip.rs index e7c2b0346..1432e3892 100644 --- a/crates/export/tests/it/vp9_roundtrip.rs +++ b/crates/export/tests/it/vp9_roundtrip.rs @@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { } fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let mut pics = d.decode(frame, 0).expect("decode"); assert_eq!(pics.len(), 1); pics.remove(0) @@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { let (w, h) = (cfg.width as usize, cfg.height as usize); let mut e = Vp9Encoder::new(cfg); - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let (mut bytes, mut db) = (0usize, 0.0); for t in 0..n { let (y, u, v) = moving(w, h, t, kind); @@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() { } let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") }; // The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg. - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); for (t, (f, r)) in frames.iter().zip(&recon).enumerate() { let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}")); assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}"); diff --git a/crates/host/tests/it/timeline_interop.rs b/crates/host/tests/it/timeline_interop.rs index 5123dc721..9688a409f 100644 --- a/crates/host/tests/it/timeline_interop.rs +++ b/crates/host/tests/it/timeline_interop.rs @@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId { color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/interchange/src/export.rs b/crates/interchange/src/export.rs index cc44475f3..594620f65 100644 --- a/crates/interchange/src/export.rs +++ b/crates/interchange/src/export.rs @@ -688,8 +688,12 @@ fn media_info(name: &str, kind: MediaKind, duration: Tick, video: Option { MediaKind::AudioOnly => FootageKind::Audio, MediaKind::Still => FootageKind::Still, MediaKind::ImageSequence => FootageKind::Sequence, - _ if v.is_none() && info.audio.is_some() => FootageKind::Audio, + _ if v.is_none() && info.has_audio() => FootageKind::Audio, _ => FootageKind::Video, }; let _ = name; @@ -289,7 +289,7 @@ impl Importer<'_> { frame_rate: rate_in(info.frame_rate()), duration: tick_in(info.duration), has_video: kind != FootageKind::Audio, - has_audio: info.audio.is_some() || kind == FootageKind::Audio, + has_audio: info.has_audio() || kind == FootageKind::Audio, alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, codec: v.map(|v| v.codec.clone()).unwrap_or_default(), missing: true, diff --git a/crates/interchange/src/tests.rs b/crates/interchange/src/tests.rs index c7ccdef33..1257ba3fd 100644 --- a/crates/interchange/src/tests.rs +++ b/crates/interchange/src/tests.rs @@ -36,8 +36,12 @@ fn info(kind: filmcraft_media::MediaKind, video: bool, audio: bool) -> filmcraft color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }), + audio_streams: audio + .then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }) + .into_iter() + .collect(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 52a375fef..a4070297b 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -22,12 +22,13 @@ effectcraft-pdf = { workspace = true } effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } +filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +filmcraft-matroska = { workspace = true } +filmcraft-mpegts = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } -# AIFF import (#1428): enables the aiff reader of filmcraft-codecs' symphonia. -symphonia = { workspace = true } exr = { workspace = true } gif = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr", "tga", "hdr"] } diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs new file mode 100644 index 000000000..625c37b43 --- /dev/null +++ b/crates/media/src/aac.rs @@ -0,0 +1,116 @@ +//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry +//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG +//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`), +//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart +//! ([`crate::downmix::Order::Aac`]). + +use std::io; + +use filmcraft_aac::AudioSpecificConfig; +use filmcraft_media::ByteReader; + +/// How far into an MPEG TS / PS file the audio streams are looked for: the stream is indexed as +/// far as this, not through the whole file. +const MPEG_HEAD: u64 = 8 << 20; + +/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header` +/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or +/// MPEG TS / PS. +pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option { + let asc = match header.and_then(|h| mp4(h, file)) { + Some(asc) => asc, + None => { + let file = file?; + let mut magic = [0u8; 4]; + file.read_at(0, &mut magic).ok()?; + if magic == [0x1A, 0x45, 0xDF, 0xA3] { matroska(file)? } else { latm(file)? } + } + }; + AudioSpecificConfig::parse(&asc).ok() +} + +/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample +/// description and a timescale, read from the whole file as it is (a fragmented movie's samples +/// are in its fragments, not the `moov`). +fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option> { + let mp4 = match file { + Some(f) => filmcraft_isobmff::open(Head(f, f.len())), + None => filmcraft_isobmff::open(header), + }; + let mp4 = mp4.ok()?; + let playable = + |t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0; + let track = mp4.tracks.iter().find(playable)?; + match &track.entries.first()?.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), + _ => None, + } +} + +/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by +/// indexing the clusters as that source does. +fn matroska(file: &dyn ByteReader) -> Option> { + let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?; + match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec { + filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()), + _ => None, + } +} + +/// The configuration of the stream FilmCraft's MPEG source plays: the first audio stream whose +/// first unit it can read. Only AAC is asked about, so the streams before that one were refused +/// whatever their codec, and an ADTS stream played has no configuration outside its frames. +fn latm(file: &dyn ByteReader) -> Option> { + use filmcraft_mpegts::Codec; + let head = Head(file, file.len().min(MPEG_HEAD)); + let mpeg = filmcraft_mpegts::open(&head).ok()?; + let audio = mpeg.streams.iter().enumerate().filter(|(_, s)| s.kind() == filmcraft_mpegts::Kind::Audio && !s.units.is_empty()); + let played = audio.take(filmcraft_media::MAX_AUDIO_STREAMS).find_map(|(i, s)| { + let first = mpeg.read_unit(&head, i, 0).ok()?; + match s.codec { + Codec::AacAdts => filmcraft_mpegts::frame_info(&s.codec, &first).map(|_| None), + Codec::AacLatm => filmcraft_codecs::audio::latm_config(&first).map(|c| Some(c.asc)), + _ => None, + } + }); + played? +} + +/// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read. +struct Head<'a>(&'a dyn ByteReader, u64); + +impl Head<'_> { + fn read(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + match offset.checked_add(buf.len() as u64) { + Some(end) if end <= self.1 => self.0.read_at(offset, buf), + _ => Err(io::Error::new(io::ErrorKind::UnexpectedEof, "read past the end")), + } + } +} + +impl filmcraft_isobmff::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + +impl filmcraft_matroska::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + +impl filmcraft_mpegts::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 2ef417dd7..09bb1e1f6 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -10,16 +10,19 @@ //! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and //! stereo is untouched. //! -//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM, -//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs, -//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. FilmCraft's AAC-LC -//! decoder instead delivers AAC's own order (C L R first, LFE last), so AAC gets its own tables. -//! More than 8 channels carry no known layout: only the first two (front L/R) are used. +//! A decoded channel's gains come from where it plays. WAVE / SMPTE order by channel count is what +//! FilmCraft's AC-3, PCM, FLAC and Opus readers deliver: 3 = L R C, 4 = quad L R Ls Rs, +//! 5 = L R C Ls Rs, 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC +//! arrives in an order its configuration decides ([`aac_order`]). More than 8 channels in WAVE +//! order carry no known layout: only the first two (front L/R) are used. use std::f32::consts::FRAC_1_SQRT_2; +use std::sync::Arc; + +use filmcraft_aac::{AudioSpecificConfig, ProgramConfig}; /// A source channel's contribution to the left and right outputs, before normalisation. -type Pair = [f32; 2]; +pub type Pair = [f32; 2]; const FL: Pair = [1.0, 0.0]; const FR: Pair = [0.0, 1.0]; @@ -29,33 +32,18 @@ const SL: Pair = [FRAC_1_SQRT_2, 0.0]; const SR: Pair = [0.0, FRAC_1_SQRT_2]; const CS: Pair = [0.5, 0.5]; -/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised. -/// `aac` selects AAC's native channel order. Empty for zero channels. -pub fn stereo_gains(count: usize, aac: bool) -> Vec { - let raw: &[Pair] = match (count, aac) { - (0, _) => &[], - (1, _) => &[[1.0, 1.0]], - (2, _) => &[FL, FR], - (3, false) => &[FL, FR, FC], - (3, true) => &[FC, FL, FR], - (4, false) => &[FL, FR, SL, SR], - (4, true) => &[FC, FL, FR, CS], - (5, false) => &[FL, FR, FC, SL, SR], - (5, true) => &[FC, FL, FR, SL, SR], - (6, false) => &[FL, FR, FC, LFE, SL, SR], - (6, true) => &[FC, FL, FR, SL, SR, LFE], - (7, false) => &[FL, FR, FC, LFE, CS, SL, SR], - (7, true) => &[FC, FL, FR, SL, SR, CS, LFE], - (8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR], - (8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE], - _ => &[FL, FR], - }; +/// AAC channel configuration 7 as FilmCraft decodes it (element order, not reordered). +const CONFIG_7: [Pair; 8] = [FC, FL, FR, SL, SR, SL, SR, LFE]; +/// AAC channel configuration 4 (C L R Cs), which FilmCraft reorders to L R C Cs. +const CONFIG_4: [Pair; 4] = [FL, FR, FC, CS]; + +/// `raw` scaled by the reciprocal of its larger row sum when that exceeds 1 (more than two +/// channels): full-scale input can never exceed full scale. +fn normalised(raw: &[Pair]) -> Vec { let mut out = raw.to_vec(); - // Channels beyond a known layout (more than 8) are silent. - out.resize(count, [0.0, 0.0]); let peak = out.iter().map(|g| g[0]).sum::().max(out.iter().map(|g| g[1]).sum::()); // Mono is deliberately left at unity on both sides; so is anything already within range. - if count > 2 && peak > 1.0 { + if out.len() > 2 && peak > 1.0 { for g in &mut out { g[0] /= peak; g[1] /= peak; @@ -64,11 +52,109 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } -/// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a -/// channel is too short for count as silence, extra frames in `out` stay as they are. -pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = stereo_gains(channels.len(), aac); - for (ch, g) in channels.iter().zip(&gains) { +/// The per-channel `[left, right]` gains for `count` channels in WAVE / SMPTE order, normalised. +/// Empty for zero channels. +pub fn stereo_gains(count: usize) -> Vec { + let raw: &[Pair] = match count { + 0 => &[], + 1 => &[[1.0, 1.0]], + 2 => &[FL, FR], + 3 => &[FL, FR, FC], + 4 => &[FL, FR, SL, SR], + 5 => &[FL, FR, FC, SL, SR], + 6 => &[FL, FR, FC, LFE, SL, SR], + 7 => &[FL, FR, FC, LFE, CS, SL, SR], + 8 => &[FL, FR, FC, LFE, SL, SR, SL, SR], + _ => &[FL, FR], + }; + let mut raw = raw.to_vec(); + // Channels beyond a known layout (more than 8) are silent. + raw.resize(count, [0.0, 0.0]); + normalised(&raw) +} + +/// Where a source's decoded channels play. +#[derive(Clone, Debug, Default, PartialEq)] +pub enum Order { + /// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest. + #[default] + Wav, + /// AAC whose configuration wasn't read: by channel count as the configurations 1 to 7 decode + /// (exact for ADTS, which can signal no other configuration FilmCraft decodes), 7 channels as + /// configuration 11. + Aac, + /// Each decoded channel's gains, before normalisation. + Speakers(Arc<[Pair]>), +} + +/// How FilmCraft's decoder hands over AAC with the configuration `asc`: it reorders channel +/// configurations 3 to 6, 11 and 12 to WAVE / SMPTE and leaves 7 and PCE layouts in element order. +pub fn aac_order(asc: &AudioSpecificConfig) -> Order { + match (asc.channel_config, &asc.pce) { + (4, _) => Order::Speakers(CONFIG_4.into()), + (1..=6 | 11 | 12, _) => Order::Wav, + (7, _) => Order::Speakers(CONFIG_7.into()), + (0, Some(pce)) => Order::Speakers(pce_speakers(pce).into()), + _ => Order::Aac, + } +} + +/// ffmpeg's encoder fills a program config element's elements with its layout's channels in its +/// own order (L R C LFE …), and for these structures that order doesn't match the element +/// positions: an LFE in a side SCE, a back centre in a side pair. Front, side and back element +/// types (`true` for a pair) and the channels of the ffmpeg layout with that structure (the first +/// ffmpeg 8.1 lists, where several share it: 6.1 for 6.1(back) and 6.1(front), 7.1(wide) for +/// 7.1(wide-side)). +const FFMPEG_PCE: [(&[bool], &[bool], &[bool], &[Pair]); 6] = [ + (&[true, false], &[false], &[false], &[FL, FR, FC, LFE, CS]), + (&[true, false], &[false], &[true], &[FL, FR, FC, LFE, SL, SR]), + (&[true, false], &[true], &[false], &[FL, FR, FC, CS, SL, SR]), + (&[true, false], &[false], &[true, false], &[FL, FR, FC, LFE, CS, SL, SR]), + (&[true, false], &[false], &[true, true], &[FL, FR, FC, LFE, SL, SR, FL, FR]), + (&[true, false], &[true], &[true, false], &[FL, FR, FC, SL, SR, CS, SL, SR]), +]; + +/// The channels of a PCE layout in decoded (element) order: ffmpeg's for its structures above +/// (it names itself in the PCE's comment), otherwise by position (ISO/IEC 14496-3's program config element): +/// front pairs left and right, a front single the centre, side and back pairs surrounds, a side +/// or back single a back centre. +fn pce_speakers(pce: &ProgramConfig) -> Vec { + let pairs = |els: &[(bool, u8)]| els.iter().map(|e| e.0).collect::>(); + let (front, side, back) = (pairs(&pce.front), pairs(&pce.side), pairs(&pce.back)); + if pce.comment.starts_with(b"Lavc") && pce.lfe.is_empty() { + let known = FFMPEG_PCE.iter().find(|(f, s, b, _)| *f == front.as_slice() && *s == side.as_slice() && *b == back.as_slice()); + if let Some((.., speakers)) = known { + return speakers.to_vec(); + } + } + let mut out = Vec::new(); + for pair in front { + out.extend_from_slice(if pair { &[FL, FR] } else { &[FC] }); + } + for pair in side.into_iter().chain(back) { + out.extend_from_slice(if pair { &[SL, SR] } else { &[CS] }); + } + out.extend(pce.lfe.iter().map(|_| LFE)); + out +} + +/// The gains for `count` channels arriving in `order`. FilmCraft decodes as many channels as the +/// container names, the first ones of the layout where that is fewer (MPEG-TS LATM names 7 of +/// configuration 7's 8, ffmpeg 4.2 MP4s 2 of a PCE's); more go by count. +pub fn decoded_gains(count: usize, order: &Order) -> Vec { + match order { + Order::Speakers(s) if count <= s.len() => normalised(s.get(..count).unwrap_or_default()), + Order::Aac if count == 4 => normalised(&CONFIG_4), + Order::Aac if count == 8 => normalised(&CONFIG_7), + _ => stereo_gains(count), + } +} + +/// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2` +/// frames); frames a channel is too short for count as silence, extra frames in `out` stay as +/// they are. +pub fn downmix_into(channels: &[Vec], order: &Order, out: &mut [f32]) { + for (ch, g) in channels.iter().zip(&decoded_gains(channels.len(), order)) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -85,49 +171,105 @@ mod tests { #[test] fn every_channel_count_has_a_bounded_defined_matrix() { - for aac in [false, true] { + let pce = Order::Speakers([FL, FR, FC, FC, SL, SR, SL, SR, CS, LFE].into()); + for order in [Order::Wav, Order::Aac, Order::Speakers(CONFIG_7.into()), pce] { for n in 0..40usize { - let g = stereo_gains(n, aac); + let g = decoded_gains(n, &order); assert_eq!(g.len(), n); let (l, r) = (g.iter().map(|p| p[0]).sum::(), g.iter().map(|p| p[1]).sum::()); if n > 2 { - assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}"); + assert!(l <= 1.0001 && r <= 1.0001, "{n} channels ({order:?}): {l} {r}"); assert!(l > 0.0 && r > 0.0); } } } } + /// AudioSpecificConfigs as ffmpeg 8.1 and FilmCraft's encoder write them (48 kHz AAC-LC). + const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500"; + const PCE_FILMCRAFT_61: &str = "118004c845000108840000"; + + fn order(hex: &str) -> Order { + let asc: Vec = (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect(); + aac_order(&AudioSpecificConfig::parse(&asc).expect("an AudioSpecificConfig")) + } + + fn speakers(order: &Order) -> Vec { + match order { + Order::Speakers(s) => s.to_vec(), + o => panic!("{o:?} names no speakers"), + } + } + + #[test] + fn aac_order_follows_the_channel_configuration() { + for n in [1, 2, 3, 5, 6, 11, 12] { + assert_eq!(aac_order(&AudioSpecificConfig::lc(3, n)), Order::Wav, "configuration {n}"); + } + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 4))), [FL, FR, FC, CS]); + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 7))), [FC, FL, FR, SL, SR, SL, SR, LFE]); + assert_eq!(speakers(&order(PCE_FFMPEG_61)), [FL, FR, FC, LFE, CS, SL, SR], "ffmpeg's 6.1 carries the LFE in a side SCE"); + assert_eq!(speakers(&order(PCE_FILMCRAFT_61)), [FC, FL, FR, SL, SR, CS, LFE], "a centre-first PCE by position"); + } + + #[test] + fn only_ffmpeg_pce_structures_it_misplaces_take_its_channel_order() { + let pce = |comment: &[u8], side: Vec<(bool, u8)>, lfe: Vec| ProgramConfig { + front: vec![(true, 0), (false, 0)], + side, + back: vec![(true, 1)], + lfe, + comment: comment.to_vec(), + ..Default::default() + }; + // 5.1(side): ffmpeg's LFE sits in the side SCE + assert_eq!(pce_speakers(&pce(b"Lavc58.54.100", vec![(false, 1)], vec![])), [FL, FR, FC, LFE, SL, SR]); + // the same structure from another encoder: a back centre by position + assert_eq!(pce_speakers(&pce(b"", vec![(false, 1)], vec![])), [FL, FR, FC, CS, SL, SR]); + // 7.0: ffmpeg's order matches the positions + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(true, 2)], vec![])), [FL, FR, FC, SL, SR, SL, SR]); + // a real LFE element is never ffmpeg's misplaced one + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(false, 1)], vec![0])), [FL, FR, FC, CS, SL, SR, LFE]); + } + + #[test] + fn a_decode_of_fewer_channels_takes_the_first_of_the_layout_and_more_go_by_count() { + assert_eq!(decoded_gains(2, &order(PCE_FILMCRAFT_61)), [FC, FL]); + assert_eq!(decoded_gains(7, &Order::Speakers(CONFIG_7.into())), normalised(&CONFIG_7[..7])); + assert_eq!(decoded_gains(9, &Order::Speakers(CONFIG_7.into())), stereo_gains(9)); + assert_eq!(decoded_gains(4, &Order::Aac), normalised(&[FL, FR, FC, CS])); + assert_eq!(decoded_gains(8, &Order::Aac), normalised(&CONFIG_7)); + for n in [3, 5, 6, 7] { + assert_eq!(decoded_gains(n, &Order::Aac), stereo_gains(n), "{n} channels"); + } + } + #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); - let g = stereo_gains(6, false); + let g = stereo_gains(6); assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0); assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6); assert_eq!(g[3], [0.0, 0.0], "LFE is left out"); assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0); assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0); - // AAC order: the centre is first, LFE last - let a = stereo_gains(6, true); - assert_eq!((a[0], a[5]), (g[2], g[3])); - assert_eq!((a[1], a[2]), (g[0], g[1])); } #[test] fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() { - assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]); - assert_eq!(stereo_gains(2, true), vec![FL, FR]); - let g = stereo_gains(12, false); + assert_eq!(stereo_gains(1), vec![[1.0, 1.0]]); + assert_eq!(stereo_gains(2), vec![FL, FR]); + let g = stereo_gains(12); assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0])); } #[test] fn ragged_and_empty_input_never_panics() { let mut out = vec![0.0; 8]; - downmix_into(&[], false, &mut out); - downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], true, &mut out); + downmix_into(&[], &Order::Wav, &mut out); + downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], &Order::Aac, &mut out); assert!(out.iter().all(|v| v.is_finite())); let mut none: [f32; 0] = []; - downmix_into(&vec![vec![1.0; 3]; 7], false, &mut none); + downmix_into(&vec![vec![1.0; 3]; 7], &Order::Speakers(CONFIG_7.into()), &mut none); } } diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 946e0d12a..c80b9377d 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -15,6 +15,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod aac; mod convert; pub mod downmix; mod dpx; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 1f1587059..655ca4ed8 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -12,6 +12,7 @@ use filmcraft_media::{FrameRequest, SharedSource}; use image::metadata::Orientation; use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned}; +use crate::downmix::Order; use crate::gif_anim::{self, Anim}; use crate::{MediaError, Result}; @@ -178,10 +179,36 @@ struct CacheState { last: HashMap, i64>, } -/// A movie/audio source by path and the turn its picture is shown with -/// ([`crate::probe::video_turn`]), opened by the first caller while the others wait for it -/// (`None`: failed to open; not retried until `forget`). -type SourceSlot = Arc>>; +/// A movie/audio source by path ([`Opened`]), opened by the first caller while the others wait +/// for it (`None`: failed to open; not retried until `forget`). +type SourceSlot = Arc>>; + +/// An opened movie/audio source, the turn its picture is shown with and the order its decoded +/// audio channels arrive in. +#[derive(Clone)] +struct Opened { + src: SharedSource, + turn: Orientation, + order: Order, +} + +impl Opened { + /// `src` with what its container says: `header` is an MP4/MOV's ftyp and moov (or the whole + /// file), `file` the file for other containers' AAC configuration. + fn new(src: SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Opened { + let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn); + Opened { order: channel_order(&src, header, file), src, turn } + } +} + +/// The order `src`'s audio channels decode in; multichannel AAC's by its configuration, where +/// the container's header shows it ([`crate::aac::config`]). +fn channel_order(src: &SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Order { + match src.info().audio() { + Some(a) if a.codec.contains("AAC") && a.channels > 2 => crate::aac::config(header, file).map_or(Order::Aac, |asc| crate::downmix::aac_order(&asc)), + _ => Order::Wav, + } +} /// An animated GIF by path, as [`SourceSlot`] (the error message when it failed to load). type GifSlot = Arc, String>>>; @@ -514,23 +541,23 @@ impl MediaPool { /// [`MediaPool::decode_audio`], failing when the source can't be opened or read. fn try_decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> std::result::Result, String> { let path: Arc = footage.path.as_str().into(); - let src = self.inner.source(&path).ok_or("the file can't be opened")?; + let Opened { src, order, .. } = self.inner.opened(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); - let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate); + let native = src.info().audio().map_or(rate, |a| a.sample_rate); // The header's rate scales the samples read: 0xFFFFFFFF Hz made one conform chunk about // 47 GB (#2141). Real audio is 1–768 kHz. if native != 0 && !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&native) { return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz")); } if native == 0 || rate == 0 || native == rate { - return read_stereo(&src, s0, frames, rate); + return read_stereo(&src, &order, s0, frames, rate); } // Output sample `n` sits at source position n × native / rate. let (native_i, rate_i) = (i128::from(native), i128::from(rate)); let first = (i128::from(s0) * native_i).div_euclid(rate_i); let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) }; - let buf = read_stereo(&src, first64, count, native)?; + let buf = read_stereo(&src, &order, first64, count, native)?; let mut out = Vec::with_capacity(frames * 2); for k in 0..frames as i128 { let pos = (i128::from(s0) + k) * native_i; @@ -593,17 +620,12 @@ impl Inner { crate::read_regular_file(std::path::Path::new(path)).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}"))) } - /// The opened movie/audio source for `path`. The first caller opens it, outside the map's - /// lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF files are - /// decoded whole); callers asking for the same file meanwhile wait for that instead of opening - /// it again: a preview's frame workers, audio feeder, conform and waveform threads each - /// decoded a long FLAC in full at once (#408). - fn source(&self, path: &Arc) -> Option { - self.opened(path).map(|(src, _)| src) - } - - /// [`Self::source`] and the turn the movie's picture is shown with. - fn opened(&self, path: &Arc) -> Option<(SharedSource, Orientation)> { + /// The opened movie/audio source for `path` ([`Opened`]). The first caller opens it, outside + /// the map's lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF + /// files are decoded whole); callers asking for the same file meanwhile wait for that instead + /// of opening it again: a preview's frame workers, audio feeder, conform and waveform threads + /// each decoded a long FLAC in full at once (#408). + fn opened(&self, path: &Arc) -> Option { let slot = lock(&self.sources).entry(path.clone()).or_default().clone(); slot.get_or_init(|| { self.opened.fetch_add(1, Ordering::Relaxed); @@ -613,14 +635,14 @@ impl Inner { let file = std::path::Path::new(&**path); let in_place = if in_memory { None } else { crate::stream::open_source(file) }; let in_place = in_place.map(|src| { - let turn = crate::stream::movie_header(file).map_or(Orientation::NoTransforms, |h| crate::probe::video_turn(&h)); - src.map(|src| (src, turn)) + let (header, reader) = (crate::stream::movie_header(file), crate::stream::file_reader(file)); + src.map(|src| Opened::new(src, header.as_deref(), reader.as_deref())) }); let opened = in_place.unwrap_or_else(|| { self.read(path).and_then(|bytes| { let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let turn = crate::probe::video_turn(&bytes); - filmcraft_codecs::open_bytes(&name, bytes).map(|src| (src, turn)).map_err(MediaError::from) + let src = filmcraft_codecs::open_bytes(&name, bytes.clone()).map_err(MediaError::from)?; + Ok(Opened::new(src, Some(&bytes), Some(&filmcraft_media::reader::MemReader(bytes.clone())))) }) }); match opened { @@ -712,9 +734,12 @@ impl Inner { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); let i = Self::frame_index(rate, t, n, footage.loop_count); - // Middle of the frame in the file's own timing (robust to timestamp rounding). + // A quarter into the frame in the file's own timing: FilmCraft rounds a request to + // the track's nearest timestamp unit (halves up), so the middle of a frame would + // land on the next one when the timescale is the frame rate; a quarter still + // tolerates timestamps rounded either way. let native = footage.native_rate.unwrap_or(rate); - let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2); + let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 4); Loc { key: key(&footage.path, i), media_t: Some(mt), float } } } @@ -851,7 +876,7 @@ impl Inner { Ok(dynamic_to_image(frame, op)) } Some(mt) => { - let (src, turn) = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; + let Opened { src, turn, .. } = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src .video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))) .map_err(MediaError::from) @@ -950,8 +975,8 @@ impl FootageSource for MediaPool { /// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which /// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both -/// channels, multichannel sources are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. -fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { +/// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. +fn read_stereo(src: &SharedSource, order: &Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { let mut out = vec![0.0; frames * 2]; let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); if skip >= frames { @@ -959,8 +984,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC")); - crate::downmix::downmix_into(&buf.channels, aac, dst); + crate::downmix::downmix_into(&buf.channels, order, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 41b50b0f4..6bae86f5b 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -175,7 +175,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - (_, None) => FootageKind::Audio, }; let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate)); - let codec = match (v, info.audio.as_ref()) { + let codec = match (v, info.audio()) { (Some(v), _) => v.codec.clone(), (None, Some(a)) => a.codec.clone(), _ => String::new(), @@ -190,7 +190,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - native_rate: None, duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) }, has_video: v.is_some(), - has_audio: info.audio.is_some(), + has_audio: info.has_audio(), alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, premul_color: [0.0; 3], loop_count: 1, @@ -295,12 +295,19 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other -/// containers have none. +/// containers have none. FilmCraft turns the frames of a pure 0/90/180/270° header itself, so this +/// is only the quarter turn nearest to a header it does not turn. pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms }; - let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else { + let Some(track) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video) else { return Orientation::NoTransforms; }; + // A pure rotation is already applied by FilmCraft's decoder (frames and size come out + // upright); only the headers it leaves as stored (mirrored, scaled, sheared) are turned here. + if track.display_rotation().is_some() { + return Orientation::NoTransforms; + } + let m = track.matrix; // [a b u; c d v; x y w]: (a, b) is where the picture's x axis points. let (a, b) = (m[0] as i64, m[1] as i64); match (a.abs() >= b.abs(), a >= 0, b >= 0) { diff --git a/crates/media/src/stream.rs b/crates/media/src/stream.rs index 447e239d1..80db21fc4 100644 --- a/crates/media/src/stream.rs +++ b/crates/media/src/stream.rs @@ -85,6 +85,17 @@ pub(crate) fn open_source(_path: &Path) -> Option> { None } +/// `path` read in place (`None` when it can't be opened, or without a native file system). +#[cfg(any(unix, windows))] +pub(crate) fn file_reader(path: &Path) -> Option> { + Some(Box::new(FileReader::open(path).ok()?)) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn file_reader(_path: &Path) -> Option> { + None +} + /// The leading `ftyp` and the `moov` box of an ISO-BMFF movie on disk (all a track's display /// matrix needs), without reading the samples: top-level box headers are walked with positional /// reads. `None` when the file isn't ISO-BMFF or its `moov` is missing or implausibly large. diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs new file mode 100644 index 000000000..fb3e078a6 --- /dev/null +++ b/crates/media/tests/it/aac_layout.rs @@ -0,0 +1,422 @@ +//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel +//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no +//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded +//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and +//! 8-channel output, and a program config element can name any of them. + +use std::f64::consts::PI; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe, probe_bytes}; +use effectcraft_time::Tick; +use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig}; + +const RATE: u32 = 48_000; +/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry. +const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700]; +const LFE_TONE: u32 = 60; +const AMP: f32 = 0.25; +const K: f64 = std::f64::consts::FRAC_1_SQRT_2; + +/// A channel's content, by where it plays. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Ch { + L, + R, + C, + Lfe, + Ls, + Rs, + Cs, +} + +impl Ch { + /// ITU-R BS.775 gains into `[left, right]`, before normalisation. + fn gains(self) -> [f64; 2] { + match self { + Ch::L => [1.0, 0.0], + Ch::R => [0.0, 1.0], + Ch::C => [K, K], + Ch::Lfe => [0.0, 0.0], + Ch::Ls => [K, 0.0], + Ch::Rs => [0.0, K], + Ch::Cs => [0.5, 0.5], + } + } +} + +struct Case { + name: &'static str, + /// What the encoder's input channels carry (its element order: C, L R, …, LFE last). + content: &'static [Ch], + /// The AudioSpecificConfig written to the container instead of the encoder's, if any. + asc: Option, +} + +fn lc(config: u8) -> Option { + Some(AudioSpecificConfig::lc(3, config)) +} + +fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec, comment: &[u8]) -> Option { + let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() }; + Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }) +} + +use Ch::*; + +fn cases() -> Vec { + vec![ + Case { name: "config3", content: &[C, L, R], asc: None }, + Case { name: "config4", content: &[C, L, R, Cs], asc: None }, + Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None }, + Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None }, + // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds + Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None }, + Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) }, + Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) }, + // the encoder's own 6.1: a PCE listing the centre first + Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None }, + // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE + // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the + // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag + // 1) and the LFE in the side SCE left over (tag 5). + Case { + name: "pce61-ffmpeg", + content: &[C, L, R, Cs, Ls, Rs, Lfe], + asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"), + }, + ] +} + +fn freq(content: &[Ch], i: usize) -> u32 { + if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] } +} + +/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units. +fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { + let n = content.len(); + let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32); + cfg.adts = adts; + let mut enc = Encoder::new(cfg).expect("encoder"); + let planes: Vec> = + (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect(); + let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect(); + let mut units = enc.encode(&refs); + units.extend(enc.flush()); + (enc.audio_specific_config(), units) +} + +fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mp4_after(None, asc, channels, units) +} + +/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no +/// samples, if any. +fn mp4_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + if let Some((asc, channels)) = empty { + w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track"); + } + let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); + for u in units { + w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); + } + w.finish().expect("finish").into_inner() +} + +fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mkv_after(None, asc, channels, units) +} + +/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no +/// frames, if any. +fn mkv_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; + let track = |asc: Vec, channels: u32| { + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + t + }; + let tracks: Vec = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect(); + let played = tracks.len() - 1; + let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer"); + let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); + for (i, u) in units.iter().enumerate() { + w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + } + w.finish().expect("finish").into_inner() +} + +/// The fixture written to disk as `name`. +fn write(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join(name); + std::fs::write(&p, bytes).expect("write"); + p +} + +/// Amplitude of the tone at `freq` in one side of interleaved stereo. +fn amp(s: &[f32], side: usize, freq: u32) -> f64 { + let (mut re, mut im, mut n) = (0.0, 0.0, 0usize); + for (i, f) in s.as_chunks::<2>().0.iter().enumerate() { + let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE); + re += f64::from(f[side]) * w.cos(); + im += f64::from(f[side]) * w.sin(); + n += 1; + } + 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64 +} + +/// Stereo of one second from 0.5 s in, through the media pool as footage reads it. +fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec { + pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE) +} + +/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger +/// side's sum (an LFE nowhere). +fn check(label: &str, content: &[Ch], s: &[f32]) { + let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect(); + let norm = raw.iter().map(|g| g[0]).sum::().max(raw.iter().map(|g| g[1]).sum::()).max(1.0); + for (i, (ch, g)) in content.iter().zip(&raw).enumerate() { + for side in 0..2 { + let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm); + assert!( + (got - want).abs() <= f64::from(AMP) * 0.05, + "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}", + freq(content, i), + ["L", "R"][side] + ); + } + } +} + +/// The LFE tone survives encoding, so its absence from the mix means it was left out. +fn lfe_is_coded(case: &Case, bytes: Vec) { + if !case.content.contains(&Lfe) { + return; + } + let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open"); + let buf = src.audio(24_000, RATE as usize, RATE).expect("decode"); + let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::>(), 0, LFE_TONE)).fold(0.0, f64::max); + assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name); +} + +#[test] +fn every_aac_layout_mixes_by_speaker_in_mp4() { + for case in cases() { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let bytes = mp4(asc, case.content.len() as u32, &units); + lfe_is_coded(&case, bytes.clone()); + let p = write(&format!("{}.mp4", case.name), &bytes); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + // in memory (`add_bytes`) as well as in place + let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into()); + let pool = MediaPool::new(); + pool.add_bytes(&path, bytes.clone()); + check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe"))); + } +} + +#[test] +fn matroska_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units)); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// ADTS signals configurations 1 to 7 only, which the channel count tells apart. +#[test] +fn adts_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) { + let (_, units) = encode(case.content, true); + let p = write(&format!("{}.aac", case.name), &units.concat()); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before +/// it lends it no configuration. Each case decodes in another order than the empty track's +/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE). +#[test] +fn an_empty_audio_track_lends_its_configuration_to_no_other() { + for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] { + let case = cases().into_iter().find(|c| c.name == name).expect("case"); + let n = case.content.len() as u32; + let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n)); + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units)); + check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units)); + check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// A transport stream plays its first audio stream and mixes by that stream's configuration, +/// whatever follows it. ADTS of configuration 7 decodes in element order, LATM of configuration +/// 12 reordered to WAVE / SMPTE; LATM of configuration 7 names 7 channels, which mix as +/// configuration 7's rather than as 7 channels by count. (FilmCraft decodes LATM of configuration +/// 12 as 12 channels, so it isn't played here.) +#[test] +fn mpeg_aac_mixes_by_the_stream_played() { + let case = cases().into_iter().find(|c| c.name == "config7").expect("case"); + let (_, adts) = encode(case.content, true); + let (_, raw) = encode(case.content, false); + for (label, latm_first, latm_config) in [("ADTS before LATM", false, 12), ("LATM before ADTS", true, 7)] { + let (adts_pid, latm_pid) = if latm_first { (0x102, 0x101) } else { (0x101, 0x102) }; + let mut ts = Ts::default(); + let mut streams = [(0x0F, adts_pid), (0x11, latm_pid)]; + streams.sort_by_key(|s| s.1); + ts.psi(&streams); + for (i, (a, r)) in adts.iter().zip(&raw).enumerate() { + let pts = 1920 * i as i64; + let mut units = [(adts_pid, pes(pts, a)), (latm_pid, pes(pts, &loas(latm_config, r)))]; + units.sort_by_key(|u| u.0); + for (pid, unit) in units { + ts.pes(pid, &unit); + } + } + let p = write(&format!("latm-first-{latm_first}.ts"), &ts.out); + check(label, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// A LOAS frame of `payload` with its StreamMuxConfig: AAC LC, 48 kHz, channel configuration +/// `config`. +fn loas(config: u32, payload: &[u8]) -> Vec { + let mut bits = Bits::default(); + // useSameStreamMux, audioMuxVersion, allStreamsSameTimeFraming, numSubFrames, numProgram, numLayer + bits.put(0b001, 3); + bits.put(0, 13); + // AudioSpecificConfig and GASpecificConfig + bits.put(2, 5); + bits.put(3, 4); + bits.put(config, 4); + bits.put(0, 3); + // frameLengthType, latmBufferFullness, otherDataPresent, crcCheckPresent + bits.put(0, 3); + bits.put(0xFF, 8); + bits.put(0, 2); + let mut len = payload.len(); + while len >= 255 { + bits.put(255, 8); + len -= 255; + } + bits.put(len as u32, 8); + for &b in payload { + bits.put(u32::from(b), 8); + } + let body = bits.finish(); + let mut v = vec![0x56, 0xE0 | (body.len() >> 8) as u8, body.len() as u8]; + v.extend(body); + v +} + +/// A big-endian bit writer. +#[derive(Default)] +struct Bits { + out: Vec, + acc: u64, + n: u32, +} + +impl Bits { + fn put(&mut self, v: u32, n: u32) { + self.acc = self.acc << n | u64::from(v); + self.n += n; + while self.n >= 8 { + self.n -= 8; + self.out.push((self.acc >> self.n) as u8); + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.put(0, 8 - self.n); + } + self.out + } +} + +/// An audio PES packet (stream id 0xC0) presented at `pts` (90 kHz). +fn pes(pts: i64, payload: &[u8]) -> Vec { + let t = [0x21 | ((pts >> 29) & 0x0E) as u8, (pts >> 22) as u8, ((pts >> 14) & 0xFE) as u8 | 1, (pts >> 7) as u8, ((pts << 1) & 0xFE) as u8 | 1]; + let len = 8 + payload.len(); + let mut v = vec![0, 0, 1, 0xC0, (len >> 8) as u8, len as u8, 0x80, 0x80, 5]; + v.extend(t); + v.extend_from_slice(payload); + v +} + +/// A transport stream of one program (its PMT on PID 0x100), PES cut into 188-byte packets. +#[derive(Default)] +struct Ts { + out: Vec, + cc: std::collections::HashMap, +} + +impl Ts { + fn packet(&mut self, pid: u16, start: bool, payload: &[u8]) -> usize { + let cc = self.cc.entry(pid).or_insert(0); + let mut p = vec![0x47, u8::from(start) << 6 | (pid >> 8) as u8, pid as u8]; + let n = payload.len().min(184); + if n < 184 { + let stuffing = 183 - n; + p.extend([0x30 | *cc, stuffing as u8]); + if stuffing > 0 { + p.push(0); + p.extend(std::iter::repeat_n(0xFF, stuffing - 1)); + } + } else { + p.push(0x10 | *cc); + } + *cc = (*cc + 1) & 15; + p.extend_from_slice(&payload[..n]); + self.out.extend(p); + n + } + fn section(&mut self, pid: u16, body: &[u8]) { + let mut s = vec![0]; + s.extend_from_slice(body); + s.extend(crc32_mpeg(body).to_be_bytes()); + self.packet(pid, true, &s); + } + /// The PAT and a PMT of `streams` (stream type, PID). + fn psi(&mut self, streams: &[(u8, u16)]) { + self.section(0, &[0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00]); + let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x01, 0xF0, 0]; + for &(kind, pid) in streams { + pmt.extend([kind, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]); + } + pmt[2] = (pmt.len() + 1) as u8; + self.section(0x100, &pmt); + } + fn pes(&mut self, pid: u16, data: &[u8]) { + let (mut rest, mut start) = (data, true); + while !rest.is_empty() { + let n = self.packet(pid, start, rest); + rest = &rest[n..]; + start = false; + } + } +} + +/// CRC-32 of PSI sections (MSB first, polynomial 0x04C11DB7, initial value all ones). +fn crc32_mpeg(data: &[u8]) -> u32 { + let mut crc = u32::MAX; + for &b in data { + crc ^= u32::from(b) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { crc << 1 ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} diff --git a/crates/media/tests/it/downmix.rs b/crates/media/tests/it/downmix.rs index e0e7d2252..05205ab6a 100644 --- a/crates/media/tests/it/downmix.rs +++ b/crates/media/tests/it/downmix.rs @@ -94,8 +94,18 @@ fn cases() -> Vec { let (n71, g71) = seven_one(); let k5 = 1.0 / (1.0 + 2.0 * K); let kq = 1.0 / (1.0 + K); + let k61 = 1.0 / (1.5 + 2.0 * K); vec![ Case { file: "dm51.m4a", layout: "5.1", names: n51, codec: &["-c:a", "aac", "-b:a", "384k"], gains: g51.clone(), tol: 0.15 }, + Case { + file: "dm61.m4a", + layout: "6.1", + names: &["FL", "FR", "FC", "LFE", "BC", "SL", "SR"], + codec: &["-c:a", "aac", "-b:a", "448k"], + gains: vec![[k61, 0.0], [0.0, k61], [k61 * K, k61 * K], [0.0, 0.0], [k61 / 2.0, k61 / 2.0], [k61 * K, 0.0], [0.0, k61 * K]], + tol: 0.15, + }, + Case { file: "dm71.m4a", layout: "7.1", names: n71, codec: &["-c:a", "aac", "-b:a", "512k"], gains: g71.clone(), tol: 0.15 }, Case { file: "dm51.ac3.mp4", layout: "5.1", names: n51, codec: &["-c:a", "ac3", "-b:a", "640k"], gains: g51.clone(), tol: 0.15 }, Case { file: "dm51.wav", layout: "5.1", names: n51, codec: &["-c:a", "pcm_s16le"], gains: g51.clone(), tol: 0.02 }, Case { file: "dm51.flac", layout: "5.1", names: n51, codec: &["-c:a", "flac"], gains: g51, tol: 0.02 }, @@ -154,24 +164,58 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() { for c in cases() { let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { continue }; - let s = ours(&p); - let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) - for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { - for (side, gain) in g.iter().enumerate() { - let got = amp(&s, side, freqs[i]); - let want = sine * gain; - assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); - } - } - let theirs = ffmpeg_ac2(&p); - let (a, b) = (rms(&s), rms(&theirs)); - let db = 20.0 * (a / b).log10(); - assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); - let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); - assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); + check(&c, &p); } } +/// `p`, or a copy whose AAC sample entry names all `channels` of its PCE layout: ffmpeg 4.2 writes +/// 2 there for a PCE layout (8.1 the real count), and FilmCraft decodes as many channels as the +/// sample entry names, so the other five of a 6.1 would never reach the mix. +fn whole_pce_entry(p: &Path, channels: usize) -> PathBuf { + let mut bytes = std::fs::read(p).expect("read"); + let Some((asc, 2)) = aac_entry(&bytes) else { return p.to_path_buf() }; + if channels <= 2 || filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config).ok() != Some(0) { + return p.to_path_buf(); + } + // the `mp4a` entry's channelcount follows its 6 reserved bytes, data reference index and + // 8 reserved bytes + let at = bytes.windows(4).position(|w| w == b"mp4a").expect("an mp4a sample entry") + 4 + 6 + 2 + 8; + bytes[at..at + 2].copy_from_slice(&(channels as u16).to_be_bytes()); + eprintln!("{}: its sample entry names 2 of {channels} channels (ffmpeg 4.2): reading a copy naming all", p.display()); + let out = p.with_extension("entry.m4a"); + std::fs::write(&out, bytes).expect("write"); + out +} + +/// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track. +fn aac_entry(bytes: &[u8]) -> Option<(Vec, u32)> { + let file = filmcraft_isobmff::open(bytes).ok()?; + file.tracks.iter().flat_map(|t| &t.entries).find_map(|e| match &e.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some((a.asc.clone(), e.audio.as_ref()?.channels)), + _ => None, + }) +} + +fn check(c: &Case, p: &Path) { + let p = &whole_pce_entry(p, c.names.len()); + let s = ours(p); + let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); + let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) + for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { + for (side, gain) in g.iter().enumerate() { + let got = amp(&s, side, freqs[i]); + let want = sine * gain; + assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); + } + } + let theirs = ffmpeg_ac2(p); + let (a, b) = (rms(&s), rms(&theirs)); + let db = 20.0 * (a / b).log10(); + assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); + let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); + assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); +} + /// Full-scale in every channel must not clip. #[test] fn loud_surround_does_not_clip() { diff --git a/crates/media/tests/it/footage.rs b/crates/media/tests/it/footage.rs index 4d539c4e5..c57cec7eb 100644 --- a/crates/media/tests/it/footage.rs +++ b/crates/media/tests/it/footage.rs @@ -915,7 +915,7 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() { // ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries // channels 0 and 1 at full level and the others at half, so a multichannel file sounds as // its stereo downmix (tests/it/downmix.rs checks that matrix channel by channel). - let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false); + let gains = effectcraft_media::downmix::stereo_gains(channels as usize); let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::(); let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt(); let rms = (sum / n.max(1) as f64).sqrt(); diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 5f7fb8037..69bed4a9f 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -1,5 +1,6 @@ //! The crate's integration tests, one binary (each module is a former `tests/.rs`). +mod aac_layout; mod downmix; mod exif_orientation; mod footage;