aleleba/effectcraft
1
0
Fork 0
mirror of https://github.com/storytold/effectcraft.git synced 2026-10-30 04:57:54 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1736 from tylerjenningsw/enhance/filmcraft-avi-backend

Browse Source 
Media: update the FilmCraft backend for AVI import
This commit is contained in:
BFlat
2026-10-30 09:01:23 +07:00
committed by GitHub
parent c0a8e5871a a3f9e0d024
commit 601b985ff6
19 changed files with 1014 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+103 -54
View File
@@ -1633,18 +1633,20 @@ dependencies = [
"effectcraft-svg",
"effectcraft-time",
"exr",
"filmcraft-aac",
"filmcraft-codecs",
"filmcraft-color",
"filmcraft-frame",
"filmcraft-isobmff",
"filmcraft-matroska",
"filmcraft-media",
"filmcraft-mpegts",
"filmcraft-time",
"gif",
"image",
"log",
"rayon",
"serde_json",
"symphonia",
"thiserror 2.0.21",
]
@@ -2246,8 +2248,8 @@ dependencies = [
[[package]]
name = "filmcraft-aac"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"thiserror 2.0.21",
@@ -2255,40 +2257,57 @@ dependencies = [
[[package]]
name = "filmcraft-ac3"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-av1"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
name = "filmcraft-apv"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-av1"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-avi"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-bitstream"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-cfb"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-codecs"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-aac",
"filmcraft-ac3",
"filmcraft-apv",
"filmcraft-av1",
"filmcraft-avi",
"filmcraft-bitstream",
"filmcraft-color",
"filmcraft-dnx",
@@ -2298,6 +2317,7 @@ dependencies = [
"filmcraft-isobmff",
"filmcraft-matroska",
"filmcraft-media",
"filmcraft-mpa",
"filmcraft-mpeg2v",
"filmcraft-mpegts",
"filmcraft-mxf",
@@ -2305,9 +2325,11 @@ dependencies = [
"filmcraft-opus",
"filmcraft-prores",
"filmcraft-time",
"filmcraft-vp8",
"filmcraft-vp9",
"image",
"log",
"rayon",
"symphonia",
"thiserror 2.0.21",
"web-time",
@@ -2315,16 +2337,25 @@ dependencies = [
[[package]]
name = "filmcraft-color"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"serde",
]
[[package]]
name = "filmcraft-dng"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"rayon",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-dnx"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
@@ -2333,8 +2364,8 @@ dependencies = [
[[package]]
name = "filmcraft-frame"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-color",
"filmcraft-time",
@@ -2344,16 +2375,16 @@ dependencies = [
[[package]]
name = "filmcraft-geom"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"serde",
]
[[package]]
name = "filmcraft-h264"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
@@ -2362,8 +2393,8 @@ dependencies = [
[[package]]
name = "filmcraft-h264enc"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
@@ -2372,8 +2403,8 @@ dependencies = [
[[package]]
name = "filmcraft-hevc"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
@@ -2382,8 +2413,8 @@ dependencies = [
[[package]]
name = "filmcraft-interchange"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-cfb",
"filmcraft-color",
@@ -2399,8 +2430,8 @@ dependencies = [
[[package]]
name = "filmcraft-isobmff"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"thiserror 2.0.21",
@@ -2408,15 +2439,16 @@ dependencies = [
[[package]]
name = "filmcraft-matroska"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-media"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-color",
"filmcraft-dng",
"filmcraft-frame",
"filmcraft-time",
"image",
@@ -2425,10 +2457,19 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpa"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mpeg2v"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"rayon",
"thiserror 2.0.21",
@@ -2436,31 +2477,31 @@ dependencies = [
[[package]]
name = "filmcraft-mpegts"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-mxf"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-ogg"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-opus"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
[[package]]
name = "filmcraft-project"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-color",
"filmcraft-geom",
@@ -2473,8 +2514,8 @@ dependencies = [
[[package]]
name = "filmcraft-prores"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
@@ -2483,16 +2524,24 @@ dependencies = [
[[package]]
name = "filmcraft-time"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"serde",
]
[[package]]
name = "filmcraft-vp8"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "filmcraft-vp9"
version = "0.1.1"
source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234"
version = "0.8.0"
source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5"
dependencies = [
"filmcraft-bitstream",
"rayon",
Cargo.toml
+21 -19
View File
@@ -77,29 +77,29 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"]
# media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001).
# Only `crates/media`, `crates/export` and `crates/interchange` may depend on these.
filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# encoders + muxer (crates/export)
filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# container headers read for AAC channel layouts (crates/media)
filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency)
filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only)
filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only)
filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" }
# The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned
# rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that.
symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] }
filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" }
# gpu (the same wgpu eframe / egui-wgpu use)
wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] }
@@ -131,6 +131,8 @@ sys-locale = "0.3"
# filmcraft-prores = { path = "../filmcraft/crates/prores" }
# filmcraft-aac = { path = "../filmcraft/crates/aac" }
# filmcraft-isobmff = { path = "../filmcraft/crates/isobmff" }
# filmcraft-matroska = { path = "../filmcraft/crates/matroska" }
# filmcraft-mpegts = { path = "../filmcraft/crates/mpegts" }
[workspace.lints.rust]
unsafe_code = "forbid"
crates/export/tests/it/movie_bytes.rs
+3 -1
View File
@@ -94,7 +94,9 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec<u8> {
#[test]
fn h264_movie_bytes_are_pinned() {
let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4");
assert_eq!((data.len(), fnv1a(&data)), (104_180, 8_728_430_532_616_484_528), "H.264 MP4");
// FilmCraft 09aaf9b's encoder lists its settings in the first frame's user-data SEI (211
// bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin.
assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4");
}
#[test]
crates/export/tests/it/vp9_roundtrip.rs
+3 -3
View File
@@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
}
fn decode(frame: &[u8]) -> filmcraft_vp9::Picture {
let mut d = Decoder::new();
let mut d = Decoder::with_threads(1);
let mut pics = d.decode(frame, 0).expect("decode");
assert_eq!(pics.len(), 1);
pics.remove(0)
@@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec<u8>; 3] {
fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) {
let (w, h) = (cfg.width as usize, cfg.height as usize);
let mut e = Vp9Encoder::new(cfg);
let mut d = Decoder::new();
let mut d = Decoder::with_threads(1);
let (mut bytes, mut db) = (0usize, 0.0);
for t in 0..n {
let (y, u, v) = moving(w, h, t, kind);
@@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() {
}
let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") };
// The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg.
let mut d = Decoder::new();
let mut d = Decoder::with_threads(1);
for (t, (f, r)) in frames.iter().zip(&recon).enumerate() {
let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}"));
assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}");
crates/host/tests/it/timeline_interop.rs
+2 -1
View File
@@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId {
color: Default::default(),
has_alpha: false,
bitrate: None,
hdr: None,
}),
audio: None,
audio_streams: Vec::new(),
container: String::new(),
start_timecode: None,
file_size: None,
crates/interchange/src/export.rs
+5 -1
View File
@@ -688,8 +688,12 @@ fn media_info(name: &str, kind: MediaKind, duration: Tick, video: Option<VideoSp
color: Default::default(),
has_alpha,
bitrate: None,
hdr: None,
}),
audio: audio.then(|| AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }),
audio_streams: audio
.then(|| AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) })
.into_iter()
.collect(),
container: String::new(),
start_timecode: None,
file_size: None,
crates/interchange/src/import.rs
+2 -2
View File
@@ -276,7 +276,7 @@ impl Importer<'_> {
MediaKind::AudioOnly => FootageKind::Audio,
MediaKind::Still => FootageKind::Still,
MediaKind::ImageSequence => FootageKind::Sequence,
_ if v.is_none() && info.audio.is_some() => FootageKind::Audio,
_ if v.is_none() && info.has_audio() => FootageKind::Audio,
_ => FootageKind::Video,
};
let _ = name;
@@ -289,7 +289,7 @@ impl Importer<'_> {
frame_rate: rate_in(info.frame_rate()),
duration: tick_in(info.duration),
has_video: kind != FootageKind::Audio,
has_audio: info.audio.is_some() || kind == FootageKind::Audio,
has_audio: info.has_audio() || kind == FootageKind::Audio,
alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore },
codec: v.map(|v| v.codec.clone()).unwrap_or_default(),
missing: true,
crates/interchange/src/tests.rs
+5 -1
View File
@@ -36,8 +36,12 @@ fn info(kind: filmcraft_media::MediaKind, video: bool, audio: bool) -> filmcraft
color: Default::default(),
has_alpha: false,
bitrate: None,
hdr: None,
}),
audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }),
audio_streams: audio
.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) })
.into_iter()
.collect(),
container: String::new(),
start_timecode: None,
file_size: None,
crates/media/Cargo.toml
+3 -2
View File
@@ -22,12 +22,13 @@ effectcraft-pdf = { workspace = true }
effectcraft-model = { workspace = true }
filmcraft-codecs = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-aac = { workspace = true }
filmcraft-isobmff = { workspace = true }
filmcraft-matroska = { workspace = true }
filmcraft-mpegts = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-color = { workspace = true }
filmcraft-time = { workspace = true }
# AIFF import (#1428): enables the aiff reader of filmcraft-codecs' symphonia.
symphonia = { workspace = true }
exr = { workspace = true }
gif = { workspace = true }
image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr", "tga", "hdr"] }
crates/media/src/aac.rs
+116
View File
@@ -0,0 +1,116 @@
//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry
//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG
//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`),
//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart
//! ([`crate::downmix::Order::Aac`]).
use std::io;
use filmcraft_aac::AudioSpecificConfig;
use filmcraft_media::ByteReader;
/// How far into an MPEG TS / PS file the audio streams are looked for: the stream is indexed as
/// far as this, not through the whole file.
const MPEG_HEAD: u64 = 8 << 20;
/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header`
/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or
/// MPEG TS / PS.
pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option<AudioSpecificConfig> {
let asc = match header.and_then(|h| mp4(h, file)) {
Some(asc) => asc,
None => {
let file = file?;
let mut magic = [0u8; 4];
file.read_at(0, &mut magic).ok()?;
if magic == [0x1A, 0x45, 0xDF, 0xA3] { matroska(file)? } else { latm(file)? }
}
};
AudioSpecificConfig::parse(&asc).ok()
}
/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample
/// description and a timescale, read from the whole file as it is (a fragmented movie's samples
/// are in its fragments, not the `moov`).
fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option<Vec<u8>> {
let mp4 = match file {
Some(f) => filmcraft_isobmff::open(Head(f, f.len())),
None => filmcraft_isobmff::open(header),
};
let mp4 = mp4.ok()?;
let playable =
|t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0;
let track = mp4.tracks.iter().find(playable)?;
match &track.entries.first()?.codec {
filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()),
_ => None,
}
}
/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by
/// indexing the clusters as that source does.
fn matroska(file: &dyn ByteReader) -> Option<Vec<u8>> {
let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?;
match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec {
filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()),
_ => None,
}
}
/// The configuration of the stream FilmCraft's MPEG source plays: the first audio stream whose
/// first unit it can read. Only AAC is asked about, so the streams before that one were refused
/// whatever their codec, and an ADTS stream played has no configuration outside its frames.
fn latm(file: &dyn ByteReader) -> Option<Vec<u8>> {
use filmcraft_mpegts::Codec;
let head = Head(file, file.len().min(MPEG_HEAD));
let mpeg = filmcraft_mpegts::open(&head).ok()?;
let audio = mpeg.streams.iter().enumerate().filter(|(_, s)| s.kind() == filmcraft_mpegts::Kind::Audio && !s.units.is_empty());
let played = audio.take(filmcraft_media::MAX_AUDIO_STREAMS).find_map(|(i, s)| {
let first = mpeg.read_unit(&head, i, 0).ok()?;
match s.codec {
Codec::AacAdts => filmcraft_mpegts::frame_info(&s.codec, &first).map(|_| None),
Codec::AacLatm => filmcraft_codecs::audio::latm_config(&first).map(|c| Some(c.asc)),
_ => None,
}
});
played?
}
/// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read.
struct Head<'a>(&'a dyn ByteReader, u64);
impl Head<'_> {
fn read(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
match offset.checked_add(buf.len() as u64) {
Some(end) if end <= self.1 => self.0.read_at(offset, buf),
_ => Err(io::Error::new(io::ErrorKind::UnexpectedEof, "read past the end")),
}
}
}
impl filmcraft_isobmff::ByteSource for Head<'_> {
fn len(&self) -> u64 {
self.1
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.read(offset, buf)
}
}
impl filmcraft_matroska::ByteSource for Head<'_> {
fn len(&self) -> u64 {
self.1
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.read(offset, buf)
}
}
impl filmcraft_mpegts::ByteSource for Head<'_> {
fn len(&self) -> u64 {
self.1
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
self.read(offset, buf)
}
}
crates/media/src/downmix.rs
+191 -49
View File
@@ -10,16 +10,19 @@
//! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and
//! stereo is untouched.
//!
//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM,
//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs,
//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. FilmCraft's AAC-LC
//! decoder instead delivers AAC's own order (C L R first, LFE last), so AAC gets its own tables.
//! More than 8 channels carry no known layout: only the first two (front L/R) are used.
//! A decoded channel's gains come from where it plays. WAVE / SMPTE order by channel count is what
//! FilmCraft's AC-3, PCM, FLAC and Opus readers deliver: 3 = L R C, 4 = quad L R Ls Rs,
//! 5 = L R C Ls Rs, 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC
//! arrives in an order its configuration decides ([`aac_order`]). More than 8 channels in WAVE
//! order carry no known layout: only the first two (front L/R) are used.
use std::f32::consts::FRAC_1_SQRT_2;
use std::sync::Arc;
use filmcraft_aac::{AudioSpecificConfig, ProgramConfig};
/// A source channel's contribution to the left and right outputs, before normalisation.
type Pair = [f32; 2];
pub type Pair = [f32; 2];
const FL: Pair = [1.0, 0.0];
const FR: Pair = [0.0, 1.0];
@@ -29,33 +32,18 @@ const SL: Pair = [FRAC_1_SQRT_2, 0.0];
const SR: Pair = [0.0, FRAC_1_SQRT_2];
const CS: Pair = [0.5, 0.5];
/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised.
/// `aac` selects AAC's native channel order. Empty for zero channels.
pub fn stereo_gains(count: usize, aac: bool) -> Vec<Pair> {
let raw: &[Pair] = match (count, aac) {
(0, _) => &[],
(1, _) => &[[1.0, 1.0]],
(2, _) => &[FL, FR],
(3, false) => &[FL, FR, FC],
(3, true) => &[FC, FL, FR],
(4, false) => &[FL, FR, SL, SR],
(4, true) => &[FC, FL, FR, CS],
(5, false) => &[FL, FR, FC, SL, SR],
(5, true) => &[FC, FL, FR, SL, SR],
(6, false) => &[FL, FR, FC, LFE, SL, SR],
(6, true) => &[FC, FL, FR, SL, SR, LFE],
(7, false) => &[FL, FR, FC, LFE, CS, SL, SR],
(7, true) => &[FC, FL, FR, SL, SR, CS, LFE],
(8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR],
(8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE],
_ => &[FL, FR],
};
/// AAC channel configuration 7 as FilmCraft decodes it (element order, not reordered).
const CONFIG_7: [Pair; 8] = [FC, FL, FR, SL, SR, SL, SR, LFE];
/// AAC channel configuration 4 (C L R Cs), which FilmCraft reorders to L R C Cs.
const CONFIG_4: [Pair; 4] = [FL, FR, FC, CS];
/// `raw` scaled by the reciprocal of its larger row sum when that exceeds 1 (more than two
/// channels): full-scale input can never exceed full scale.
fn normalised(raw: &[Pair]) -> Vec<Pair> {
let mut out = raw.to_vec();
// Channels beyond a known layout (more than 8) are silent.
out.resize(count, [0.0, 0.0]);
let peak = out.iter().map(|g| g[0]).sum::<f32>().max(out.iter().map(|g| g[1]).sum::<f32>());
// Mono is deliberately left at unity on both sides; so is anything already within range.
if count > 2 && peak > 1.0 {
if out.len() > 2 && peak > 1.0 {
for g in &mut out {
g[0] /= peak;
g[1] /= peak;
@@ -64,11 +52,109 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec<Pair> {
out
}
/// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a
/// channel is too short for count as silence, extra frames in `out` stay as they are.
pub fn downmix_into(channels: &[Vec<f32>], aac: bool, out: &mut [f32]) {
let gains = stereo_gains(channels.len(), aac);
for (ch, g) in channels.iter().zip(&gains) {
/// The per-channel `[left, right]` gains for `count` channels in WAVE / SMPTE order, normalised.
/// Empty for zero channels.
pub fn stereo_gains(count: usize) -> Vec<Pair> {
let raw: &[Pair] = match count {
0 => &[],
1 => &[[1.0, 1.0]],
2 => &[FL, FR],
3 => &[FL, FR, FC],
4 => &[FL, FR, SL, SR],
5 => &[FL, FR, FC, SL, SR],
6 => &[FL, FR, FC, LFE, SL, SR],
7 => &[FL, FR, FC, LFE, CS, SL, SR],
8 => &[FL, FR, FC, LFE, SL, SR, SL, SR],
_ => &[FL, FR],
};
let mut raw = raw.to_vec();
// Channels beyond a known layout (more than 8) are silent.
raw.resize(count, [0.0, 0.0]);
normalised(&raw)
}
/// Where a source's decoded channels play.
#[derive(Clone, Debug, Default, PartialEq)]
pub enum Order {
/// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest.
#[default]
Wav,
/// AAC whose configuration wasn't read: by channel count as the configurations 1 to 7 decode
/// (exact for ADTS, which can signal no other configuration FilmCraft decodes), 7 channels as
/// configuration 11.
Aac,
/// Each decoded channel's gains, before normalisation.
Speakers(Arc<[Pair]>),
}
/// How FilmCraft's decoder hands over AAC with the configuration `asc`: it reorders channel
/// configurations 3 to 6, 11 and 12 to WAVE / SMPTE and leaves 7 and PCE layouts in element order.
pub fn aac_order(asc: &AudioSpecificConfig) -> Order {
match (asc.channel_config, &asc.pce) {
(4, _) => Order::Speakers(CONFIG_4.into()),
(1..=6 | 11 | 12, _) => Order::Wav,
(7, _) => Order::Speakers(CONFIG_7.into()),
(0, Some(pce)) => Order::Speakers(pce_speakers(pce).into()),
_ => Order::Aac,
}
}
/// ffmpeg's encoder fills a program config element's elements with its layout's channels in its
/// own order (L R C LFE …), and for these structures that order doesn't match the element
/// positions: an LFE in a side SCE, a back centre in a side pair. Front, side and back element
/// types (`true` for a pair) and the channels of the ffmpeg layout with that structure (the first
/// ffmpeg 8.1 lists, where several share it: 6.1 for 6.1(back) and 6.1(front), 7.1(wide) for
/// 7.1(wide-side)).
const FFMPEG_PCE: [(&[bool], &[bool], &[bool], &[Pair]); 6] = [
(&[true, false], &[false], &[false], &[FL, FR, FC, LFE, CS]),
(&[true, false], &[false], &[true], &[FL, FR, FC, LFE, SL, SR]),
(&[true, false], &[true], &[false], &[FL, FR, FC, CS, SL, SR]),
(&[true, false], &[false], &[true, false], &[FL, FR, FC, LFE, CS, SL, SR]),
(&[true, false], &[false], &[true, true], &[FL, FR, FC, LFE, SL, SR, FL, FR]),
(&[true, false], &[true], &[true, false], &[FL, FR, FC, SL, SR, CS, SL, SR]),
];
/// The channels of a PCE layout in decoded (element) order: ffmpeg's for its structures above
/// (it names itself in the PCE's comment), otherwise by position (ISO/IEC 14496-3's program config element):
/// front pairs left and right, a front single the centre, side and back pairs surrounds, a side
/// or back single a back centre.
fn pce_speakers(pce: &ProgramConfig) -> Vec<Pair> {
let pairs = |els: &[(bool, u8)]| els.iter().map(|e| e.0).collect::<Vec<_>>();
let (front, side, back) = (pairs(&pce.front), pairs(&pce.side), pairs(&pce.back));
if pce.comment.starts_with(b"Lavc") && pce.lfe.is_empty() {
let known = FFMPEG_PCE.iter().find(|(f, s, b, _)| *f == front.as_slice() && *s == side.as_slice() && *b == back.as_slice());
if let Some((.., speakers)) = known {
return speakers.to_vec();
}
}
let mut out = Vec::new();
for pair in front {
out.extend_from_slice(if pair { &[FL, FR] } else { &[FC] });
}
for pair in side.into_iter().chain(back) {
out.extend_from_slice(if pair { &[SL, SR] } else { &[CS] });
}
out.extend(pce.lfe.iter().map(|_| LFE));
out
}
/// The gains for `count` channels arriving in `order`. FilmCraft decodes as many channels as the
/// container names, the first ones of the layout where that is fewer (MPEG-TS LATM names 7 of
/// configuration 7's 8, ffmpeg 4.2 MP4s 2 of a PCE's); more go by count.
pub fn decoded_gains(count: usize, order: &Order) -> Vec<Pair> {
match order {
Order::Speakers(s) if count <= s.len() => normalised(s.get(..count).unwrap_or_default()),
Order::Aac if count == 4 => normalised(&CONFIG_4),
Order::Aac if count == 8 => normalised(&CONFIG_7),
_ => stereo_gains(count),
}
}
/// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2`
/// frames); frames a channel is too short for count as silence, extra frames in `out` stay as
/// they are.
pub fn downmix_into(channels: &[Vec<f32>], order: &Order, out: &mut [f32]) {
for (ch, g) in channels.iter().zip(&decoded_gains(channels.len(), order)) {
if g[0] == 0.0 && g[1] == 0.0 {
continue;
}
@@ -85,49 +171,105 @@ mod tests {
#[test]
fn every_channel_count_has_a_bounded_defined_matrix() {
for aac in [false, true] {
let pce = Order::Speakers([FL, FR, FC, FC, SL, SR, SL, SR, CS, LFE].into());
for order in [Order::Wav, Order::Aac, Order::Speakers(CONFIG_7.into()), pce] {
for n in 0..40usize {
let g = stereo_gains(n, aac);
let g = decoded_gains(n, &order);
assert_eq!(g.len(), n);
let (l, r) = (g.iter().map(|p| p[0]).sum::<f32>(), g.iter().map(|p| p[1]).sum::<f32>());
if n > 2 {
assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}");
assert!(l <= 1.0001 && r <= 1.0001, "{n} channels ({order:?}): {l} {r}");
assert!(l > 0.0 && r > 0.0);
}
}
}
}
/// AudioSpecificConfigs as ffmpeg 8.1 and FilmCraft's encoder write them (48 kHz AAC-LC).
const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500";
const PCE_FILMCRAFT_61: &str = "118004c845000108840000";
fn order(hex: &str) -> Order {
let asc: Vec<u8> = (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect();
aac_order(&AudioSpecificConfig::parse(&asc).expect("an AudioSpecificConfig"))
}
fn speakers(order: &Order) -> Vec<Pair> {
match order {
Order::Speakers(s) => s.to_vec(),
o => panic!("{o:?} names no speakers"),
}
}
#[test]
fn aac_order_follows_the_channel_configuration() {
for n in [1, 2, 3, 5, 6, 11, 12] {
assert_eq!(aac_order(&AudioSpecificConfig::lc(3, n)), Order::Wav, "configuration {n}");
}
assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 4))), [FL, FR, FC, CS]);
assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 7))), [FC, FL, FR, SL, SR, SL, SR, LFE]);
assert_eq!(speakers(&order(PCE_FFMPEG_61)), [FL, FR, FC, LFE, CS, SL, SR], "ffmpeg's 6.1 carries the LFE in a side SCE");
assert_eq!(speakers(&order(PCE_FILMCRAFT_61)), [FC, FL, FR, SL, SR, CS, LFE], "a centre-first PCE by position");
}
#[test]
fn only_ffmpeg_pce_structures_it_misplaces_take_its_channel_order() {
let pce = |comment: &[u8], side: Vec<(bool, u8)>, lfe: Vec<u8>| ProgramConfig {
front: vec![(true, 0), (false, 0)],
side,
back: vec![(true, 1)],
lfe,
comment: comment.to_vec(),
..Default::default()
};
// 5.1(side): ffmpeg's LFE sits in the side SCE
assert_eq!(pce_speakers(&pce(b"Lavc58.54.100", vec![(false, 1)], vec![])), [FL, FR, FC, LFE, SL, SR]);
// the same structure from another encoder: a back centre by position
assert_eq!(pce_speakers(&pce(b"", vec![(false, 1)], vec![])), [FL, FR, FC, CS, SL, SR]);
// 7.0: ffmpeg's order matches the positions
assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(true, 2)], vec![])), [FL, FR, FC, SL, SR, SL, SR]);
// a real LFE element is never ffmpeg's misplaced one
assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(false, 1)], vec![0])), [FL, FR, FC, CS, SL, SR, LFE]);
}
#[test]
fn a_decode_of_fewer_channels_takes_the_first_of_the_layout_and_more_go_by_count() {
assert_eq!(decoded_gains(2, &order(PCE_FILMCRAFT_61)), [FC, FL]);
assert_eq!(decoded_gains(7, &Order::Speakers(CONFIG_7.into())), normalised(&CONFIG_7[..7]));
assert_eq!(decoded_gains(9, &Order::Speakers(CONFIG_7.into())), stereo_gains(9));
assert_eq!(decoded_gains(4, &Order::Aac), normalised(&[FL, FR, FC, CS]));
assert_eq!(decoded_gains(8, &Order::Aac), normalised(&CONFIG_7));
for n in [3, 5, 6, 7] {
assert_eq!(decoded_gains(n, &Order::Aac), stereo_gains(n), "{n} channels");
}
}
#[test]
fn five_one_matches_the_bs775_matrix() {
let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2);
let g = stereo_gains(6, false);
let g = stereo_gains(6);
assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0);
assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6);
assert_eq!(g[3], [0.0, 0.0], "LFE is left out");
assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0);
assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0);
// AAC order: the centre is first, LFE last
let a = stereo_gains(6, true);
assert_eq!((a[0], a[5]), (g[2], g[3]));
assert_eq!((a[1], a[2]), (g[0], g[1]));
}
#[test]
fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() {
assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]);
assert_eq!(stereo_gains(2, true), vec![FL, FR]);
let g = stereo_gains(12, false);
assert_eq!(stereo_gains(1), vec![[1.0, 1.0]]);
assert_eq!(stereo_gains(2), vec![FL, FR]);
let g = stereo_gains(12);
assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0]));
}
#[test]
fn ragged_and_empty_input_never_panics() {
let mut out = vec![0.0; 8];
downmix_into(&[], false, &mut out);
downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], true, &mut out);
downmix_into(&[], &Order::Wav, &mut out);
downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], &Order::Aac, &mut out);
assert!(out.iter().all(|v| v.is_finite()));
let mut none: [f32; 0] = [];
downmix_into(&vec![vec![1.0; 3]; 7], false, &mut none);
downmix_into(&vec![vec![1.0; 3]; 7], &Order::Speakers(CONFIG_7.into()), &mut none);
}
}
crates/media/src/lib.rs
+1
View File
@@ -15,6 +15,7 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod aac;
mod convert;
pub mod downmix;
mod dpx;
crates/media/src/pool.rs
+54 -30
View File
@@ -12,6 +12,7 @@ use filmcraft_media::{FrameRequest, SharedSource};
use image::metadata::Orientation;
use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned};
use crate::downmix::Order;
use crate::gif_anim::{self, Anim};
use crate::{MediaError, Result};
@@ -178,10 +179,36 @@ struct CacheState {
last: HashMap<Arc<str>, i64>,
}
/// A movie/audio source by path and the turn its picture is shown with
/// ([`crate::probe::video_turn`]), opened by the first caller while the others wait for it
/// (`None`: failed to open; not retried until `forget`).
type SourceSlot = Arc<OnceLock<Option<(SharedSource, Orientation)>>>;
/// A movie/audio source by path ([`Opened`]), opened by the first caller while the others wait
/// for it (`None`: failed to open; not retried until `forget`).
type SourceSlot = Arc<OnceLock<Option<Opened>>>;
/// An opened movie/audio source, the turn its picture is shown with and the order its decoded
/// audio channels arrive in.
#[derive(Clone)]
struct Opened {
src: SharedSource,
turn: Orientation,
order: Order,
}
impl Opened {
/// `src` with what its container says: `header` is an MP4/MOV's ftyp and moov (or the whole
/// file), `file` the file for other containers' AAC configuration.
fn new(src: SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Opened {
let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn);
Opened { order: channel_order(&src, header, file), src, turn }
}
}
/// The order `src`'s audio channels decode in; multichannel AAC's by its configuration, where
/// the container's header shows it ([`crate::aac::config`]).
fn channel_order(src: &SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Order {
match src.info().audio() {
Some(a) if a.codec.contains("AAC") && a.channels > 2 => crate::aac::config(header, file).map_or(Order::Aac, |asc| crate::downmix::aac_order(&asc)),
_ => Order::Wav,
}
}
/// An animated GIF by path, as [`SourceSlot`] (the error message when it failed to load).
type GifSlot = Arc<OnceLock<std::result::Result<Arc<Anim>, String>>>;
@@ -514,23 +541,23 @@ impl MediaPool {
/// [`MediaPool::decode_audio`], failing when the source can't be opened or read.
fn try_decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {
let path: Arc<str> = footage.path.as_str().into();
let src = self.inner.source(&path).ok_or("the file can't be opened")?;
let Opened { src, order, .. } = self.inner.opened(&path).ok_or("the file can't be opened")?;
let s0 = start.to_units_floor(rate as i64);
let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate);
let native = src.info().audio().map_or(rate, |a| a.sample_rate);
// The header's rate scales the samples read: 0xFFFFFFFF Hz made one conform chunk about
// 47 GB (#2141). Real audio is 1–768 kHz.
if native != 0 && !(MIN_SAMPLE_RATE..=MAX_SAMPLE_RATE).contains(&native) {
return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz"));
}
if native == 0 || rate == 0 || native == rate {
return read_stereo(&src, s0, frames, rate);
return read_stereo(&src, &order, s0, frames, rate);
}
// Output sample `n` sits at source position n × native / rate.
let (native_i, rate_i) = (i128::from(native), i128::from(rate));
let first = (i128::from(s0) * native_i).div_euclid(rate_i);
let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1;
let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) };
let buf = read_stereo(&src, first64, count, native)?;
let buf = read_stereo(&src, &order, first64, count, native)?;
let mut out = Vec::with_capacity(frames * 2);
for k in 0..frames as i128 {
let pos = (i128::from(s0) + k) * native_i;
@@ -593,17 +620,12 @@ impl Inner {
crate::read_regular_file(std::path::Path::new(path)).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}")))
}
/// The opened movie/audio source for `path`. The first caller opens it, outside the map's
/// lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF files are
/// decoded whole); callers asking for the same file meanwhile wait for that instead of opening
/// it again: a preview's frame workers, audio feeder, conform and waveform threads each
/// decoded a long FLAC in full at once (#408).
fn source(&self, path: &Arc<str>) -> Option<SharedSource> {
self.opened(path).map(|(src, _)| src)
}
/// [`Self::source`] and the turn the movie's picture is shown with.
fn opened(&self, path: &Arc<str>) -> Option<(SharedSource, Orientation)> {
/// The opened movie/audio source for `path` ([`Opened`]). The first caller opens it, outside
/// the map's lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF
/// files are decoded whole); callers asking for the same file meanwhile wait for that instead
/// of opening it again: a preview's frame workers, audio feeder, conform and waveform threads
/// each decoded a long FLAC in full at once (#408).
fn opened(&self, path: &Arc<str>) -> Option<Opened> {
let slot = lock(&self.sources).entry(path.clone()).or_default().clone();
slot.get_or_init(|| {
self.opened.fetch_add(1, Ordering::Relaxed);
@@ -613,14 +635,14 @@ impl Inner {
let file = std::path::Path::new(&**path);
let in_place = if in_memory { None } else { crate::stream::open_source(file) };
let in_place = in_place.map(|src| {
let turn = crate::stream::movie_header(file).map_or(Orientation::NoTransforms, |h| crate::probe::video_turn(&h));
src.map(|src| (src, turn))
let (header, reader) = (crate::stream::movie_header(file), crate::stream::file_reader(file));
src.map(|src| Opened::new(src, header.as_deref(), reader.as_deref()))
});
let opened = in_place.unwrap_or_else(|| {
self.read(path).and_then(|bytes| {
let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let turn = crate::probe::video_turn(&bytes);
filmcraft_codecs::open_bytes(&name, bytes).map(|src| (src, turn)).map_err(MediaError::from)
let src = filmcraft_codecs::open_bytes(&name, bytes.clone()).map_err(MediaError::from)?;
Ok(Opened::new(src, Some(&bytes), Some(&filmcraft_media::reader::MemReader(bytes.clone()))))
})
});
match opened {
@@ -712,9 +734,12 @@ impl Inner {
let rate = footage.frame_rate;
let n = Self::frame_count(rate, footage.duration);
let i = Self::frame_index(rate, t, n, footage.loop_count);
// Middle of the frame in the file's own timing (robust to timestamp rounding).
// A quarter into the frame in the file's own timing: FilmCraft rounds a request to
// the track's nearest timestamp unit (halves up), so the middle of a frame would
// land on the next one when the timescale is the frame rate; a quarter still
// tolerates timestamps rounded either way.
let native = footage.native_rate.unwrap_or(rate);
let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2);
let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 4);
Loc { key: key(&footage.path, i), media_t: Some(mt), float }
}
}
@@ -851,7 +876,7 @@ impl Inner {
Ok(dynamic_to_image(frame, op))
}
Some(mt) => {
let (src, turn) = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
let Opened { src, turn, .. } = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?;
let vf = src
.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0)))
.map_err(MediaError::from)
@@ -950,8 +975,8 @@ impl FootageSource for MediaPool {
/// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which
/// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both
/// channels, multichannel sources are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails.
fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {
/// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails.
fn read_stereo(src: &SharedSource, order: &Order, s0: i64, frames: usize, rate: u32) -> std::result::Result<Vec<f32>, String> {
let mut out = vec![0.0; frames * 2];
let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0);
if skip >= frames {
@@ -959,8 +984,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re
}
let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?;
let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) };
let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC"));
crate::downmix::downmix_into(&buf.channels, aac, dst);
crate::downmix::downmix_into(&buf.channels, order, dst);
Ok(out)
}
crates/media/src/probe.rs
+11 -4
View File
@@ -175,7 +175,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) -
(_, None) => FootageKind::Audio,
};
let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate));
let codec = match (v, info.audio.as_ref()) {
let codec = match (v, info.audio()) {
(Some(v), _) => v.codec.clone(),
(None, Some(a)) => a.codec.clone(),
_ => String::new(),
@@ -190,7 +190,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) -
native_rate: None,
duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) },
has_video: v.is_some(),
has_audio: info.audio.is_some(),
has_audio: info.has_audio(),
alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore },
premul_color: [0.0; 3],
loop_count: 1,
@@ -295,12 +295,19 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option<image::ImageFormat>) -> im
/// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and
/// says so in its video track's header): the quarter turn nearest the track's rotation. Only the
/// rotation counts; any mirroring, scaling or offset in the header is ignored, and other
/// containers have none.
/// containers have none. FilmCraft turns the frames of a pure 0/90/180/270° header itself, so this
/// is only the quarter turn nearest to a header it does not turn.
pub(crate) fn video_turn(bytes: &[u8]) -> Orientation {
let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms };
let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else {
let Some(track) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video) else {
return Orientation::NoTransforms;
};
// A pure rotation is already applied by FilmCraft's decoder (frames and size come out
// upright); only the headers it leaves as stored (mirrored, scaled, sheared) are turned here.
if track.display_rotation().is_some() {
return Orientation::NoTransforms;
}
let m = track.matrix;
// [a b u; c d v; x y w]: (a, b) is where the picture's x axis points.
let (a, b) = (m[0] as i64, m[1] as i64);
match (a.abs() >= b.abs(), a >= 0, b >= 0) {
crates/media/src/stream.rs
+11
View File
@@ -85,6 +85,17 @@ pub(crate) fn open_source(_path: &Path) -> Option<Result<SharedSource>> {
None
}
/// `path` read in place (`None` when it can't be opened, or without a native file system).
#[cfg(any(unix, windows))]
pub(crate) fn file_reader(path: &Path) -> Option<Box<dyn filmcraft_media::ByteReader>> {
Some(Box::new(FileReader::open(path).ok()?))
}
#[cfg(not(any(unix, windows)))]
pub(crate) fn file_reader(_path: &Path) -> Option<Box<dyn filmcraft_media::ByteReader>> {
None
}
/// The leading `ftyp` and the `moov` box of an ISO-BMFF movie on disk (all a track's display
/// matrix needs), without reading the samples: top-level box headers are walked with positional
/// reads. `None` when the file isn't ISO-BMFF or its `moov` is missing or implausibly large.
crates/media/tests/it/aac_layout.rs
+422
View File
@@ -0,0 +1,422 @@
//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel
//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no
//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded
//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and
//! 8-channel output, and a program config element can name any of them.
use std::f64::consts::PI;
use std::io::Cursor;
use std::path::PathBuf;
use std::sync::Arc;
use effectcraft_media::{MediaPool, probe, probe_bytes};
use effectcraft_time::Tick;
use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig};
const RATE: u32 = 48_000;
/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry.
const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700];
const LFE_TONE: u32 = 60;
const AMP: f32 = 0.25;
const K: f64 = std::f64::consts::FRAC_1_SQRT_2;
/// A channel's content, by where it plays.
#[derive(Clone, Copy, Debug, PartialEq)]
enum Ch {
L,
R,
C,
Lfe,
Ls,
Rs,
Cs,
}
impl Ch {
/// ITU-R BS.775 gains into `[left, right]`, before normalisation.
fn gains(self) -> [f64; 2] {
match self {
Ch::L => [1.0, 0.0],
Ch::R => [0.0, 1.0],
Ch::C => [K, K],
Ch::Lfe => [0.0, 0.0],
Ch::Ls => [K, 0.0],
Ch::Rs => [0.0, K],
Ch::Cs => [0.5, 0.5],
}
}
}
struct Case {
name: &'static str,
/// What the encoder's input channels carry (its element order: C, L R, …, LFE last).
content: &'static [Ch],
/// The AudioSpecificConfig written to the container instead of the encoder's, if any.
asc: Option<AudioSpecificConfig>,
}
fn lc(config: u8) -> Option<AudioSpecificConfig> {
Some(AudioSpecificConfig::lc(3, config))
}
fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec<u8>, comment: &[u8]) -> Option<AudioSpecificConfig> {
let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() };
Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) })
}
use Ch::*;
fn cases() -> Vec<Case> {
vec![
Case { name: "config3", content: &[C, L, R], asc: None },
Case { name: "config4", content: &[C, L, R, Cs], asc: None },
Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None },
Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None },
// configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds
Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None },
Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) },
Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) },
// the encoder's own 6.1: a PCE listing the centre first
Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None },
// ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE
// element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the
// encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag
// 1) and the LFE in the side SCE left over (tag 5).
Case {
name: "pce61-ffmpeg",
content: &[C, L, R, Cs, Ls, Rs, Lfe],
asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"),
},
]
}
fn freq(content: &[Ch], i: usize) -> u32 {
if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] }
}
/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units.
fn encode(content: &[Ch], adts: bool) -> (Vec<u8>, Vec<Vec<u8>>) {
let n = content.len();
let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32);
cfg.adts = adts;
let mut enc = Encoder::new(cfg).expect("encoder");
let planes: Vec<Vec<f32>> =
(0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect();
let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect();
let mut units = enc.encode(&refs);
units.extend(enc.flush());
(enc.audio_specific_config(), units)
}
fn mp4(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
mp4_after(None, asc, channels, units)
}
/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no
/// samples, if any.
fn mp4_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer");
if let Some((asc, channels)) = empty {
w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track");
}
let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track");
for u in units {
w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample");
}
w.finish().expect("finish").into_inner()
}
fn mkv(asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
mkv_after(None, asc, channels, units)
}
/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no
/// frames, if any.
fn mkv_after(empty: Option<(Vec<u8>, u32)>, asc: Vec<u8>, channels: u32, units: &[Vec<u8>]) -> Vec<u8> {
use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
let track = |asc: Vec<u8>, channels: u32| {
let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC");
t.codec_private = asc;
t.audio = Some((f64::from(RATE), u64::from(channels), None));
t
};
let tracks: Vec<TrackSpec> = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect();
let played = tracks.len() - 1;
let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer");
let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE);
for (i, u) in units.iter().enumerate() {
w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame");
}
w.finish().expect("finish").into_inner()
}
/// The fixture written to disk as `name`.
fn write(name: &str, bytes: &[u8]) -> PathBuf {
let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id()));
std::fs::create_dir_all(&dir).expect("mkdir");
let p = dir.join(name);
std::fs::write(&p, bytes).expect("write");
p
}
/// Amplitude of the tone at `freq` in one side of interleaved stereo.
fn amp(s: &[f32], side: usize, freq: u32) -> f64 {
let (mut re, mut im, mut n) = (0.0, 0.0, 0usize);
for (i, f) in s.as_chunks::<2>().0.iter().enumerate() {
let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE);
re += f64::from(f[side]) * w.cos();
im += f64::from(f[side]) * w.sin();
n += 1;
}
2.0 * (re * re + im * im).sqrt() / n.max(1) as f64
}
/// Stereo of one second from 0.5 s in, through the media pool as footage reads it.
fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec<f32> {
pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE)
}
/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger
/// side's sum (an LFE nowhere).
fn check(label: &str, content: &[Ch], s: &[f32]) {
let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect();
let norm = raw.iter().map(|g| g[0]).sum::<f64>().max(raw.iter().map(|g| g[1]).sum::<f64>()).max(1.0);
for (i, (ch, g)) in content.iter().zip(&raw).enumerate() {
for side in 0..2 {
let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm);
assert!(
(got - want).abs() <= f64::from(AMP) * 0.05,
"{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}",
freq(content, i),
["L", "R"][side]
);
}
}
}
/// The LFE tone survives encoding, so its absence from the mix means it was left out.
fn lfe_is_coded(case: &Case, bytes: Vec<u8>) {
if !case.content.contains(&Lfe) {
return;
}
let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open");
let buf = src.audio(24_000, RATE as usize, RATE).expect("decode");
let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::<Vec<_>>(), 0, LFE_TONE)).fold(0.0, f64::max);
assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name);
}
#[test]
fn every_aac_layout_mixes_by_speaker_in_mp4() {
for case in cases() {
let (asc, units) = encode(case.content, false);
let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
let bytes = mp4(asc, case.content.len() as u32, &units);
lfe_is_coded(&case, bytes.clone());
let p = write(&format!("{}.mp4", case.name), &bytes);
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
// in memory (`add_bytes`) as well as in place
let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into());
let pool = MediaPool::new();
pool.add_bytes(&path, bytes.clone());
check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe")));
}
}
#[test]
fn matroska_aac_layouts_mix_by_speaker() {
for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) {
let (asc, units) = encode(case.content, false);
let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units));
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
}
}
/// ADTS signals configurations 1 to 7 only, which the channel count tells apart.
#[test]
fn adts_aac_layouts_mix_by_speaker() {
for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) {
let (_, units) = encode(case.content, true);
let p = write(&format!("{}.aac", case.name), &units.concat());
check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
}
}
/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before
/// it lends it no configuration. Each case decodes in another order than the empty track's
/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE).
#[test]
fn an_empty_audio_track_lends_its_configuration_to_no_other() {
for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] {
let case = cases().into_iter().find(|c| c.name == name).expect("case");
let n = case.content.len() as u32;
let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n));
let (asc, units) = encode(case.content, false);
let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes);
let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units));
check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units));
check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
}
}
/// A transport stream plays its first audio stream and mixes by that stream's configuration,
/// whatever follows it. ADTS of configuration 7 decodes in element order, LATM of configuration
/// 12 reordered to WAVE / SMPTE; LATM of configuration 7 names 7 channels, which mix as
/// configuration 7's rather than as 7 channels by count. (FilmCraft decodes LATM of configuration
/// 12 as 12 channels, so it isn't played here.)
#[test]
fn mpeg_aac_mixes_by_the_stream_played() {
let case = cases().into_iter().find(|c| c.name == "config7").expect("case");
let (_, adts) = encode(case.content, true);
let (_, raw) = encode(case.content, false);
for (label, latm_first, latm_config) in [("ADTS before LATM", false, 12), ("LATM before ADTS", true, 7)] {
let (adts_pid, latm_pid) = if latm_first { (0x102, 0x101) } else { (0x101, 0x102) };
let mut ts = Ts::default();
let mut streams = [(0x0F, adts_pid), (0x11, latm_pid)];
streams.sort_by_key(|s| s.1);
ts.psi(&streams);
for (i, (a, r)) in adts.iter().zip(&raw).enumerate() {
let pts = 1920 * i as i64;
let mut units = [(adts_pid, pes(pts, a)), (latm_pid, pes(pts, &loas(latm_config, r)))];
units.sort_by_key(|u| u.0);
for (pid, unit) in units {
ts.pes(pid, &unit);
}
}
let p = write(&format!("latm-first-{latm_first}.ts"), &ts.out);
check(label, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe")));
}
}
/// A LOAS frame of `payload` with its StreamMuxConfig: AAC LC, 48 kHz, channel configuration
/// `config`.
fn loas(config: u32, payload: &[u8]) -> Vec<u8> {
let mut bits = Bits::default();
// useSameStreamMux, audioMuxVersion, allStreamsSameTimeFraming, numSubFrames, numProgram, numLayer
bits.put(0b001, 3);
bits.put(0, 13);
// AudioSpecificConfig and GASpecificConfig
bits.put(2, 5);
bits.put(3, 4);
bits.put(config, 4);
bits.put(0, 3);
// frameLengthType, latmBufferFullness, otherDataPresent, crcCheckPresent
bits.put(0, 3);
bits.put(0xFF, 8);
bits.put(0, 2);
let mut len = payload.len();
while len >= 255 {
bits.put(255, 8);
len -= 255;
}
bits.put(len as u32, 8);
for &b in payload {
bits.put(u32::from(b), 8);
}
let body = bits.finish();
let mut v = vec![0x56, 0xE0 | (body.len() >> 8) as u8, body.len() as u8];
v.extend(body);
v
}
/// A big-endian bit writer.
#[derive(Default)]
struct Bits {
out: Vec<u8>,
acc: u64,
n: u32,
}
impl Bits {
fn put(&mut self, v: u32, n: u32) {
self.acc = self.acc << n | u64::from(v);
self.n += n;
while self.n >= 8 {
self.n -= 8;
self.out.push((self.acc >> self.n) as u8);
}
}
fn finish(mut self) -> Vec<u8> {
if self.n > 0 {
self.put(0, 8 - self.n);
}
self.out
}
}
/// An audio PES packet (stream id 0xC0) presented at `pts` (90 kHz).
fn pes(pts: i64, payload: &[u8]) -> Vec<u8> {
let t = [0x21 | ((pts >> 29) & 0x0E) as u8, (pts >> 22) as u8, ((pts >> 14) & 0xFE) as u8 | 1, (pts >> 7) as u8, ((pts << 1) & 0xFE) as u8 | 1];
let len = 8 + payload.len();
let mut v = vec![0, 0, 1, 0xC0, (len >> 8) as u8, len as u8, 0x80, 0x80, 5];
v.extend(t);
v.extend_from_slice(payload);
v
}
/// A transport stream of one program (its PMT on PID 0x100), PES cut into 188-byte packets.
#[derive(Default)]
struct Ts {
out: Vec<u8>,
cc: std::collections::HashMap<u16, u8>,
}
impl Ts {
fn packet(&mut self, pid: u16, start: bool, payload: &[u8]) -> usize {
let cc = self.cc.entry(pid).or_insert(0);
let mut p = vec![0x47, u8::from(start) << 6 | (pid >> 8) as u8, pid as u8];
let n = payload.len().min(184);
if n < 184 {
let stuffing = 183 - n;
p.extend([0x30 | *cc, stuffing as u8]);
if stuffing > 0 {
p.push(0);
p.extend(std::iter::repeat_n(0xFF, stuffing - 1));
}
} else {
p.push(0x10 | *cc);
}
*cc = (*cc + 1) & 15;
p.extend_from_slice(&payload[..n]);
self.out.extend(p);
n
}
fn section(&mut self, pid: u16, body: &[u8]) {
let mut s = vec![0];
s.extend_from_slice(body);
s.extend(crc32_mpeg(body).to_be_bytes());
self.packet(pid, true, &s);
}
/// The PAT and a PMT of `streams` (stream type, PID).
fn psi(&mut self, streams: &[(u8, u16)]) {
self.section(0, &[0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00]);
let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x01, 0xF0, 0];
for &(kind, pid) in streams {
pmt.extend([kind, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]);
}
pmt[2] = (pmt.len() + 1) as u8;
self.section(0x100, &pmt);
}
fn pes(&mut self, pid: u16, data: &[u8]) {
let (mut rest, mut start) = (data, true);
while !rest.is_empty() {
let n = self.packet(pid, start, rest);
rest = &rest[n..];
start = false;
}
}
}
/// CRC-32 of PSI sections (MSB first, polynomial 0x04C11DB7, initial value all ones).
fn crc32_mpeg(data: &[u8]) -> u32 {
let mut crc = u32::MAX;
for &b in data {
crc ^= u32::from(b) << 24;
for _ in 0..8 {
crc = if crc & 0x8000_0000 != 0 { crc << 1 ^ 0x04C1_1DB7 } else { crc << 1 };
}
}
crc
}
crates/media/tests/it/downmix.rs
+59 -15
View File
@@ -94,8 +94,18 @@ fn cases() -> Vec<Case> {
let (n71, g71) = seven_one();
let k5 = 1.0 / (1.0 + 2.0 * K);
let kq = 1.0 / (1.0 + K);
let k61 = 1.0 / (1.5 + 2.0 * K);
vec![
Case { file: "dm51.m4a", layout: "5.1", names: n51, codec: &["-c:a", "aac", "-b:a", "384k"], gains: g51.clone(), tol: 0.15 },
Case {
file: "dm61.m4a",
layout: "6.1",
names: &["FL", "FR", "FC", "LFE", "BC", "SL", "SR"],
codec: &["-c:a", "aac", "-b:a", "448k"],
gains: vec![[k61, 0.0], [0.0, k61], [k61 * K, k61 * K], [0.0, 0.0], [k61 / 2.0, k61 / 2.0], [k61 * K, 0.0], [0.0, k61 * K]],
tol: 0.15,
},
Case { file: "dm71.m4a", layout: "7.1", names: n71, codec: &["-c:a", "aac", "-b:a", "512k"], gains: g71.clone(), tol: 0.15 },
Case { file: "dm51.ac3.mp4", layout: "5.1", names: n51, codec: &["-c:a", "ac3", "-b:a", "640k"], gains: g51.clone(), tol: 0.15 },
Case { file: "dm51.wav", layout: "5.1", names: n51, codec: &["-c:a", "pcm_s16le"], gains: g51.clone(), tol: 0.02 },
Case { file: "dm51.flac", layout: "5.1", names: n51, codec: &["-c:a", "flac"], gains: g51, tol: 0.02 },
@@ -154,24 +164,58 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() {
for c in cases() {
let freqs: Vec<u32> = FREQS.iter().copied().take(c.names.len()).collect();
let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { continue };
let s = ours(&p);
let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3)
for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() {
for (side, gain) in g.iter().enumerate() {
let got = amp(&s, side, freqs[i]);
let want = sine * gain;
assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]);
}
}
let theirs = ffmpeg_ac2(&p);
let (a, b) = (rms(&s), rms(&theirs));
let db = 20.0 * (a / b).log10();
assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file);
let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10();
assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file);
check(&c, &p);
}
}
/// `p`, or a copy whose AAC sample entry names all `channels` of its PCE layout: ffmpeg 4.2 writes
/// 2 there for a PCE layout (8.1 the real count), and FilmCraft decodes as many channels as the
/// sample entry names, so the other five of a 6.1 would never reach the mix.
fn whole_pce_entry(p: &Path, channels: usize) -> PathBuf {
let mut bytes = std::fs::read(p).expect("read");
let Some((asc, 2)) = aac_entry(&bytes) else { return p.to_path_buf() };
if channels <= 2 || filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config).ok() != Some(0) {
return p.to_path_buf();
}
// the `mp4a` entry's channelcount follows its 6 reserved bytes, data reference index and
// 8 reserved bytes
let at = bytes.windows(4).position(|w| w == b"mp4a").expect("an mp4a sample entry") + 4 + 6 + 2 + 8;
bytes[at..at + 2].copy_from_slice(&(channels as u16).to_be_bytes());
eprintln!("{}: its sample entry names 2 of {channels} channels (ffmpeg 4.2): reading a copy naming all", p.display());
let out = p.with_extension("entry.m4a");
std::fs::write(&out, bytes).expect("write");
out
}
/// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track.
fn aac_entry(bytes: &[u8]) -> Option<(Vec<u8>, u32)> {
let file = filmcraft_isobmff::open(bytes).ok()?;
file.tracks.iter().flat_map(|t| &t.entries).find_map(|e| match &e.codec {
filmcraft_isobmff::CodecConfig::Aac(a) => Some((a.asc.clone(), e.audio.as_ref()?.channels)),
_ => None,
})
}
fn check(c: &Case, p: &Path) {
let p = &whole_pce_entry(p, c.names.len());
let s = ours(p);
let freqs: Vec<u32> = FREQS.iter().copied().take(c.names.len()).collect();
let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3)
for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() {
for (side, gain) in g.iter().enumerate() {
let got = amp(&s, side, freqs[i]);
let want = sine * gain;
assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]);
}
}
let theirs = ffmpeg_ac2(p);
let (a, b) = (rms(&s), rms(&theirs));
let db = 20.0 * (a / b).log10();
assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file);
let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10();
assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file);
}
/// Full-scale in every channel must not clip.
#[test]
fn loud_surround_does_not_clip() {
crates/media/tests/it/footage.rs
+1 -1
View File
@@ -915,7 +915,7 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() {
// ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries
// channels 0 and 1 at full level and the others at half, so a multichannel file sounds as
// its stereo downmix (tests/it/downmix.rs checks that matrix channel by channel).
let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false);
let gains = effectcraft_media::downmix::stereo_gains(channels as usize);
let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::<f64>();
let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt();
let rms = (sum / n.max(1) as f64).sqrt();
crates/media/tests/it/main.rs
+1
View File
@@ -1,5 +1,6 @@
//! The crate's integration tests, one binary (each module is a former `tests/<name>.rs`).
mod aac_layout;
mod downmix;
mod exif_orientation;
mod footage;