From 93b06a6eab9c1636524be8173d95b074f02b20b8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 15 Oct 2026 22:56:38 +0200 Subject: [PATCH 01/33] Bump FilmCraft to 4e2d0bc2 (part of #804) Problem: the hardware-decoding work (#819) needs a newer FilmCraft than the pinned 2c5ba7a, and the pin bump should be reviewable apart from it. Fix: move all filmcraft-* deps to 4e2d0bc2. Adapt to MediaInfo.audio -> audio_streams and VideoStreamInfo.hdr; ask the pool for a quarter into the frame because FilmCraft now rounds requests half up; adapt rotation and the AAC downmix; use Decoder::with_threads(1) in the VP9 tests. No filmcraft-platform, no hardware setting. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 115 ++++++++++++++------------ Cargo.toml | 32 +++---- crates/export/tests/vp9_roundtrip.rs | 6 +- crates/host/tests/timeline_interop.rs | 3 +- crates/interchange/src/export.rs | 6 +- crates/interchange/src/import.rs | 4 +- crates/interchange/src/tests.rs | 6 +- crates/media/src/downmix.rs | 39 ++++++++- crates/media/src/pool.rs | 11 ++- crates/media/src/probe.rs | 15 +++- 10 files changed, 151 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce8fe1231..98e00791d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,8 +2025,8 @@ checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "filmcraft-aac" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2034,16 +2034,26 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-apv" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-av1" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2052,21 +2062,22 @@ dependencies = [ [[package]] name = "filmcraft-bitstream" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-cfb" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-codecs" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-aac", "filmcraft-ac3", + "filmcraft-apv", "filmcraft-av1", "filmcraft-bitstream", "filmcraft-color", @@ -2094,16 +2105,16 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-dnx" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2112,8 +2123,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2123,16 +2134,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2141,8 +2152,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2151,8 +2162,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2161,8 +2172,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2178,8 +2189,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2187,13 +2198,13 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-media" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-frame", @@ -2206,8 +2217,8 @@ dependencies = [ [[package]] name = "filmcraft-mpeg2v" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2215,31 +2226,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-ogg" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-opus" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-project" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2252,8 +2263,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2262,16 +2273,16 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-vp9" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 9f4e9d17f..cb8b7ea38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,26 +71,26 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index e7c2b0346..1432e3892 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { } fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let mut pics = d.decode(frame, 0).expect("decode"); assert_eq!(pics.len(), 1); pics.remove(0) @@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { let (w, h) = (cfg.width as usize, cfg.height as usize); let mut e = Vp9Encoder::new(cfg); - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let (mut bytes, mut db) = (0usize, 0.0); for t in 0..n { let (y, u, v) = moving(w, h, t, kind); @@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() { } let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") }; // The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg. - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); for (t, (f, r)) in frames.iter().zip(&recon).enumerate() { let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}")); assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}"); diff --git a/crates/host/tests/timeline_interop.rs b/crates/host/tests/timeline_interop.rs index 5123dc721..9688a409f 100644 --- a/crates/host/tests/timeline_interop.rs +++ b/crates/host/tests/timeline_interop.rs @@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId { color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/interchange/src/export.rs b/crates/interchange/src/export.rs index 7da18913a..da6ef1382 100644 --- a/crates/interchange/src/export.rs +++ b/crates/interchange/src/export.rs @@ -684,8 +684,12 @@ fn media_info(name: &str, kind: MediaKind, duration: Tick, video: Option { MediaKind::AudioOnly => FootageKind::Audio, MediaKind::Still => FootageKind::Still, MediaKind::ImageSequence => FootageKind::Sequence, - _ if v.is_none() && info.audio.is_some() => FootageKind::Audio, + _ if v.is_none() && info.has_audio() => FootageKind::Audio, _ => FootageKind::Video, }; let _ = name; @@ -284,7 +284,7 @@ impl Importer<'_> { frame_rate: rate_in(info.frame_rate()), duration: tick_in(info.duration), has_video: kind != FootageKind::Audio, - has_audio: info.audio.is_some() || kind == FootageKind::Audio, + has_audio: info.has_audio() || kind == FootageKind::Audio, alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, codec: v.map(|v| v.codec.clone()).unwrap_or_default(), missing: true, diff --git a/crates/interchange/src/tests.rs b/crates/interchange/src/tests.rs index 3ad7fcaae..6e29c79d2 100644 --- a/crates/interchange/src/tests.rs +++ b/crates/interchange/src/tests.rs @@ -36,8 +36,12 @@ fn info(kind: filmcraft_media::MediaKind, video: bool, audio: bool) -> filmcraft color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }), + audio_streams: audio + .then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }) + .into_iter() + .collect(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 2ef417dd7..e8a9237e0 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -64,11 +64,30 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } +/// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 +/// channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs), everything else (mono, +/// stereo, 7 and 8 channels, program-config layouts) in AAC's element order. +pub fn decoded_aac_gains(count: usize) -> Vec { + match count { + // C L R Cs, reordered to L R C Cs: the same gains, permuted. + 4 => { + let g = stereo_gains(4, true); + [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() + } + 3 | 5 | 6 => stereo_gains(count, false), + _ => stereo_gains(count, true), + } +} + /// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a /// channel is too short for count as silence, extra frames in `out` stay as they are. pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = stereo_gains(channels.len(), aac); - for (ch, g) in channels.iter().zip(&gains) { + let gains = if aac { decoded_aac_gains(channels.len()) } else { stereo_gains(channels.len(), false) }; + mix(channels, &gains, out); +} + +fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { + for (ch, g) in channels.iter().zip(gains) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -98,6 +117,22 @@ mod tests { } } + #[test] + fn decoded_aac_is_in_wav_order_for_three_to_six_channels() { + // FilmCraft hands AAC 3.0 to 5.1 over as L R C [LFE] Ls Rs, the order of the plain layouts. + for n in [3, 5, 6] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); + } + let g = decoded_aac_gains(4); + assert_eq!(g.len(), 4); + assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); + assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); + assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); + for n in [1, 2, 7, 8, 9] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); + } + } + #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 98c707108..33fb0b26f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -509,7 +509,7 @@ impl MediaPool { let path: Arc = footage.path.as_str().into(); let src = self.inner.source(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); - let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate); + let native = src.info().audio().map_or(rate, |a| a.sample_rate); if native == 0 || rate == 0 || native == rate { return read_stereo(&src, s0, frames, rate); } @@ -700,9 +700,12 @@ impl Inner { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); let i = Self::frame_index(rate, t, n, footage.loop_count); - // Middle of the frame in the file's own timing (robust to timestamp rounding). + // A quarter into the frame in the file's own timing: FilmCraft rounds a request to + // the track's nearest timestamp unit (halves up), so the middle of a frame would + // land on the next one when the timescale is the frame rate; a quarter still + // tolerates timestamps rounded either way. let native = footage.native_rate.unwrap_or(rate); - let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2); + let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 4); Loc { key: key(&footage.path, i), media_t: Some(mt), float } } } @@ -915,7 +918,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC")); + let aac = src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")); crate::downmix::downmix_into(&buf.channels, aac, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index bae714c81..7c5d1c491 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -119,7 +119,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - (_, None) => FootageKind::Audio, }; let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate)); - let codec = match (v, info.audio.as_ref()) { + let codec = match (v, info.audio()) { (Some(v), _) => v.codec.clone(), (None, Some(a)) => a.codec.clone(), _ => String::new(), @@ -134,7 +134,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - native_rate: None, duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) }, has_video: v.is_some(), - has_audio: info.audio.is_some(), + has_audio: info.has_audio(), alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, premul_color: [0.0; 3], loop_count: 1, @@ -239,12 +239,19 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other -/// containers have none. +/// containers have none. FilmCraft turns the frames of a pure 0/90/180/270° header itself, so this +/// is only the quarter turn nearest to a header it does not turn. pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms }; - let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else { + let Some(track) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video) else { return Orientation::NoTransforms; }; + // A pure rotation is already applied by FilmCraft's decoder (frames and size come out + // upright); only the headers it leaves as stored (mirrored, scaled, sheared) are turned here. + if track.display_rotation().is_some() { + return Orientation::NoTransforms; + } + let m = track.matrix; // [a b u; c d v; x y w]: (a, b) is where the picture's x axis points. let (a, b) = (m[0] as i64, m[1] as i64); match (a.abs() >= b.abs(), a >= 0, b >= 0) { From be9c9077cd62232f3a6ecac2a6c2d309cfb68545 Mon Sep 17 00:00:00 2001 From: p01ntov <278898977+p01ntov@users.noreply.github.com> Date: Sat, 17 Oct 2026 13:56:35 +0300 Subject: [PATCH 02/33] G2: tessellate viewer path overlays in screen space --- crates/ui-egui/src/panels/viewer_overlays.rs | 152 ++++++++++++++++++- 1 file changed, 146 insertions(+), 6 deletions(-) diff --git a/crates/ui-egui/src/panels/viewer_overlays.rs b/crates/ui-egui/src/panels/viewer_overlays.rs index 9117103d5..d0875941e 100644 --- a/crates/ui-egui/src/panels/viewer_overlays.rs +++ b/crates/ui-egui/src/panels/viewer_overlays.rs @@ -132,17 +132,52 @@ pub(crate) fn path_of(ectx: &EvalCtx, layer: LayerId, uid: u64) -> Option Vec { - let k = effectcraft_engine::render::kurbo_path(sp); + let mut k = effectcraft_engine::render::kurbo_path(sp); + // A path-space tolerance becomes visible corners when the viewer or a layer/group is + // magnified. Flatten affine paths in screen space so the error stays below half a point. + // Projective transforms must still map the flattened points: projecting Bezier control + // points would change the curve rather than just its tessellation. + let screen_space = m.is_affine(); + if screen_space { + let z = f64::from(map.zoom); + k.apply_affine(kurbo::Affine::new([ + m.0[0][0] * z, + m.0[1][0] * z, + m.0[0][1] * z, + m.0[1][1] * z, + m.0[0][2] * z + f64::from(map.origin.x), + m.0[1][2] * z + f64::from(map.origin.y), + ])); + } + if !k.is_finite() { + return Vec::new(); + } + let bounds = k.control_box(); + if [bounds.x0, bounds.y0, bounds.x1, bounds.y1].iter().any(|v| v.abs() > f64::from(f32::MAX)) { + return Vec::new(); + } + // Bound subdivision work even for extreme finite scales from a malformed project. For + // ordinary on-screen paths the tolerance remains half a point. + let tolerance = (bounds.width().max(bounds.height()) / 1_000_000.0).max(0.5); let mut pts: Vec = vec![]; - kurbo::flatten(k.iter(), 0.5, |el| match el { + kurbo::flatten(k.iter(), tolerance, |el| match el { kurbo::PathEl::MoveTo(q) | kurbo::PathEl::LineTo(q) => { - let c = m.apply(gv2(q.x, q.y)); - pts.push(map.to_screen([c.x, c.y])); + let p = if screen_space { + egui::pos2(q.x as f32, q.y as f32) + } else { + let c = m.apply(gv2(q.x, q.y)); + map.to_screen([c.x, c.y]) + }; + if p.is_finite() { + pts.push(p); + } } _ => {} }); - if sp.closed && !pts.is_empty() { - pts.push(pts[0]); + if sp.closed + && let Some(first) = pts.first().copied() + { + pts.push(first); } pts } @@ -679,6 +714,111 @@ mod tests { PathInfo { layer: LayerId(3), uid: 7, m: Mat3::IDENTITY, sp, is_mask: true, feather: Vec::new() } } + fn circle() -> ShapePath { + let r = 38.0; + let h = r * 0.552_284_749_830_793_6; + ShapePath { + vertices: vec![[0.0, -r], [r, 0.0], [0.0, r], [-r, 0.0]], + in_tangents: vec![[-h, 0.0], [0.0, -h], [h, 0.0], [0.0, h]], + out_tangents: vec![[h, 0.0], [0.0, h], [-h, 0.0], [0.0, -h]], + closed: true, + feather: Vec::new(), + } + } + + /// Measure the actual screen-space error against dense samples of the original cubics, + /// independently of the tessellator's chosen subdivisions. + fn screen_error(sp: &ShapePath, m: &Mat3, map: &ViewerMap, points: &[Pos2]) -> f32 { + let mut worst = 0.0_f32; + for i in 0..sp.vertices.len() { + let Some(c) = segment(sp, i) else { continue }; + for step in 0..=1024 { + let q = cubic(c[0], c[1], c[2], c[3], f64::from(step) / 1024.0); + let q = m.apply(gv2(q[0], q[1])); + let p = map.to_screen([q.x, q.y]); + let distance = points + .windows(2) + .map(|pair| { + let a = pair[0]; + let v = pair[1] - a; + let t = if v.length_sq() > 0.0 { ((p - a).dot(v) / v.length_sq()).clamp(0.0, 1.0) } else { 0.0 }; + p.distance(a + v * t) + }) + .fold(f32::INFINITY, f32::min); + worst = worst.max(distance); + } + } + worst + } + + #[test] + fn curve_overlay_stays_smooth_at_viewer_magnifications() { + let sp = circle(); + let mut counts = Vec::new(); + for zoom in [0.5, 1.0, 5.0] { + let map = ViewerMap { origin: egui::pos2(121.25, 210.5), zoom, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + let points = flatten(&sp, &Mat3::IDENTITY, &map); + let error = screen_error(&sp, &Mat3::IDENTITY, &map, &points); + assert!(error <= 0.51, "{zoom}x: {error} screen points from the curve"); + assert_eq!(points.first(), points.last(), "closed paths keep their closing edge"); + for v in &sp.vertices { + let anchor = map.to_screen(*v); + assert!(points.iter().any(|p| p.distance(anchor) < 0.001), "original vertex missing at {zoom}x"); + } + counts.push(points.len()); + } + assert!(counts[0] < counts[1] && counts[1] < counts[2], "tessellation follows visible curve size: {counts:?}"); + } + + #[test] + fn curve_overlay_accounts_for_nonuniform_layer_and_group_transforms() { + let sp = circle(); + let layer = Mat3::translate(gv2(70.0, 50.0)) * Mat3::rotate_deg(27.0) * Mat3::scale(gv2(3.2, 0.8)); + let group = Mat3::rotate_deg(-19.0) * Mat3::skew_deg(25.0, 11.0) * Mat3::scale(gv2(0.7, 2.5)); + let m = layer * group; + for zoom in [0.5, 1.0, 5.0] { + let map = ViewerMap { origin: egui::pos2(121.25, 210.5), zoom, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + let points = flatten(&sp, &m, &map); + let error = screen_error(&sp, &m, &map, &points); + assert!(error <= 0.51, "{zoom}x transformed path: {error} screen points from the curve"); + } + } + + #[test] + fn open_overlay_preserves_its_transformed_endpoints() { + let sp = info(vec![[10.0, 20.0], [50.0, 60.0]], false).sp; + let m = Mat3::translate(gv2(70.0, 50.0)) * Mat3::rotate_deg(27.0) * Mat3::scale(gv2(3.2, 0.8)); + let map = ViewerMap { origin: egui::pos2(121.25, 210.5), zoom: 5.0, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + let points = flatten(&sp, &m, &map); + assert_ne!(points.first(), points.last(), "an open path must not gain a closing edge"); + for (point, vertex) in [points.first(), points.last()].into_iter().zip([sp.vertices.first(), sp.vertices.last()]) { + let q = m.apply(gv2(vertex.unwrap()[0], vertex.unwrap()[1])); + assert!(point.unwrap().distance(map.to_screen([q.x, q.y])) < 0.001); + } + } + + #[test] + fn invalid_overlay_coordinates_are_not_tessellated() { + let sp = circle(); + for zoom in [f32::NAN, f32::INFINITY] { + let map = ViewerMap { origin: Pos2::ZERO, zoom, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + assert!(flatten(&sp, &Mat3::IDENTITY, &map).is_empty()); + } + let map = ViewerMap { origin: Pos2::ZERO, zoom: 5.0, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + assert!(flatten(&sp, &Mat3::scale(gv2(f64::INFINITY, 1.0)), &map).is_empty()); + } + + #[test] + fn extreme_finite_overlay_scales_keep_subdivision_bounded() { + let map = ViewerMap { origin: Pos2::ZERO, zoom: 5.0, comp: [640.0, 400.0], area: Rect::EVERYTHING }; + let sp = circle(); + let points = flatten(&sp, &Mat3::scale(gv2(1e12, 1e12)), &map); + assert!(!points.is_empty()); + assert!(points.len() < 10_000, "extreme scales must not allocate millions of vertices"); + assert!(points.iter().all(|p| p.is_finite())); + assert!(flatten(&sp, &Mat3::scale(gv2(1e300, 1e300)), &map).is_empty()); + } + /// #912: clicking the last vertex of an open path with the Pen resumes that path. #[test] fn pen_resumes_an_open_path_at_its_last_vertex() { From ee17172fb688ffe1abb07de2db9c7d806f4e5198 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:50:04 -0400 Subject: [PATCH 03/33] Media: update FilmCraft to 09aaf9b, which reads AVI (#1701) Moves the pin past FilmCraft's AVI / OpenDML reader (Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3). No EffectCraft API changes. --- Cargo.lock | 144 +++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 32 ++++++------ 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06da2cebd..1dea8187a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2043,8 +2043,8 @@ checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "filmcraft-aac" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2052,16 +2052,16 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-apv" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2070,33 +2070,39 @@ dependencies = [ [[package]] name = "filmcraft-av1" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-avi" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" + [[package]] name = "filmcraft-bitstream" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-cfb" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-codecs" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-aac", "filmcraft-ac3", "filmcraft-apv", "filmcraft-av1", + "filmcraft-avi", "filmcraft-bitstream", "filmcraft-color", "filmcraft-dnx", @@ -2106,6 +2112,7 @@ dependencies = [ "filmcraft-isobmff", "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpa", "filmcraft-mpeg2v", "filmcraft-mpegts", "filmcraft-mxf", @@ -2113,9 +2120,11 @@ dependencies = [ "filmcraft-opus", "filmcraft-prores", "filmcraft-time", + "filmcraft-vp8", "filmcraft-vp9", "image", "log", + "rayon", "symphonia", "thiserror 2.0.21", "web-time", @@ -2123,16 +2132,25 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-dng" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-dnx" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2141,8 +2159,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2152,16 +2170,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2170,8 +2188,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2180,8 +2198,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2190,8 +2208,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2207,8 +2225,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2216,15 +2234,16 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-media" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", + "filmcraft-dng", "filmcraft-frame", "filmcraft-time", "image", @@ -2233,10 +2252,19 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-mpa" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "filmcraft-bitstream", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-mpeg2v" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2244,31 +2272,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-ogg" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-opus" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-project" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2281,8 +2309,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2291,16 +2319,24 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-vp8" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-vp9" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 7b9983788..8810e3c16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,26 +72,26 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned # rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that. symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] } From 01ad187b97888efedd588130f12d20887e7a610b Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:53:04 -0400 Subject: [PATCH 04/33] Media: drop the AIFF workaround dependency (#1701) FilmCraft's codecs now turn on symphonia's AIFF reader themselves. --- Cargo.lock | 1 - Cargo.toml | 3 --- crates/media/Cargo.toml | 2 -- 3 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1dea8187a..8fc3183ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1455,7 +1455,6 @@ dependencies = [ "log", "rayon", "serde_json", - "symphonia", "thiserror 2.0.21", ] diff --git a/Cargo.toml b/Cargo.toml index 8810e3c16..b1c6caaf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,9 +92,6 @@ filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9b filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -# The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned -# rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that. -symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index bb47dccb0..a3ff624b4 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -24,8 +24,6 @@ filmcraft-isobmff = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } -# AIFF import (#1428): enables the aiff reader of filmcraft-codecs' symphonia. -symphonia = { workspace = true } exr = { workspace = true } gif = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr", "tga", "hdr"] } From ca724a3d84c8fcd1724b3de724ed4c5c4507ee53 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:53:05 -0400 Subject: [PATCH 05/33] Media: downmix 7.1 AAC in the order FilmCraft now delivers it (#1701) FilmCraft reorders AAC channel configuration 12 (7.1) to WAVE/SMPTE order, as it does 3 to 6, so 7.1 AAC's left channel was mixed as the centre. The downmix oracle test gains 6.1 (a program config element, left in AAC's order) and 7.1 AAC fixtures. --- crates/media/src/downmix.rs | 16 +++++++++------- crates/media/tests/downmix.rs | 10 ++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index e8a9237e0..1ad9721ba 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -65,8 +65,10 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { } /// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 -/// channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs), everything else (mono, -/// stereo, 7 and 8 channels, program-config layouts) in AAC's element order. +/// channels) and 12 (7.1, 8 channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs, +/// L R C LFE Lb Rb Ls Rs), everything else (mono, stereo, program-config layouts such as the 6.1 +/// ffmpeg writes) in AAC's element order. Configuration 11 (6.1) and 7 (7.1 with front wides) +/// can't be told from those by the channel count and are read in the order of the commoner one. pub fn decoded_aac_gains(count: usize) -> Vec { match count { // C L R Cs, reordered to L R C Cs: the same gains, permuted. @@ -74,7 +76,7 @@ pub fn decoded_aac_gains(count: usize) -> Vec { let g = stereo_gains(4, true); [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() } - 3 | 5 | 6 => stereo_gains(count, false), + 3 | 5 | 6 | 8 => stereo_gains(count, false), _ => stereo_gains(count, true), } } @@ -118,9 +120,9 @@ mod tests { } #[test] - fn decoded_aac_is_in_wav_order_for_three_to_six_channels() { - // FilmCraft hands AAC 3.0 to 5.1 over as L R C [LFE] Ls Rs, the order of the plain layouts. - for n in [3, 5, 6] { + fn decoded_aac_is_in_wav_order_for_three_to_six_and_eight_channels() { + // FilmCraft hands AAC 3.0 to 5.1 and 7.1 over in the order of the plain layouts. + for n in [3, 5, 6, 8] { assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); } let g = decoded_aac_gains(4); @@ -128,7 +130,7 @@ mod tests { assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - for n in [1, 2, 7, 8, 9] { + for n in [1, 2, 7, 9] { assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); } } diff --git a/crates/media/tests/downmix.rs b/crates/media/tests/downmix.rs index e0e7d2252..e2c95c251 100644 --- a/crates/media/tests/downmix.rs +++ b/crates/media/tests/downmix.rs @@ -94,8 +94,18 @@ fn cases() -> Vec { let (n71, g71) = seven_one(); let k5 = 1.0 / (1.0 + 2.0 * K); let kq = 1.0 / (1.0 + K); + let k61 = 1.0 / (1.5 + 2.0 * K); vec![ Case { file: "dm51.m4a", layout: "5.1", names: n51, codec: &["-c:a", "aac", "-b:a", "384k"], gains: g51.clone(), tol: 0.15 }, + Case { + file: "dm61.m4a", + layout: "6.1", + names: &["FL", "FR", "FC", "LFE", "BC", "SL", "SR"], + codec: &["-c:a", "aac", "-b:a", "448k"], + gains: vec![[k61, 0.0], [0.0, k61], [k61 * K, k61 * K], [0.0, 0.0], [k61 / 2.0, k61 / 2.0], [k61 * K, 0.0], [0.0, k61 * K]], + tol: 0.15, + }, + Case { file: "dm71.m4a", layout: "7.1", names: n71, codec: &["-c:a", "aac", "-b:a", "512k"], gains: g71.clone(), tol: 0.15 }, Case { file: "dm51.ac3.mp4", layout: "5.1", names: n51, codec: &["-c:a", "ac3", "-b:a", "640k"], gains: g51.clone(), tol: 0.15 }, Case { file: "dm51.wav", layout: "5.1", names: n51, codec: &["-c:a", "pcm_s16le"], gains: g51.clone(), tol: 0.02 }, Case { file: "dm51.flac", layout: "5.1", names: n51, codec: &["-c:a", "flac"], gains: g51, tol: 0.02 }, From bed49c4959787c41e4fd5986db9683f2debe8af5 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 11:35:05 -0400 Subject: [PATCH 06/33] Media: import AVI (#1701) One list of movie extensions, in effectcraft-project, with AVI added: FilmCraft reads AVI 1.0 / OpenDML. Tests against ffmpeg fixtures: Motion JPEG with PCM audio, an unsupported codec (imports and says why it has no picture), a file that isn't an AVI (an import error), and a damaged one (a truncated recording imports as far as it goes; a garbled packet loses its own frame only). --- crates/media/src/lib.rs | 3 +- crates/media/tests/footage.rs | 135 ++++++++++++++++++++++++++++++++-- crates/project/src/lib.rs | 5 ++ 3 files changed, 135 insertions(+), 8 deletions(-) diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 04a9d95e3..1b40fcfac 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -63,8 +63,7 @@ pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "tga", "hdr", "psd", "psb", "svg", "pdf", "ai", "eps"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; -/// File extensions recognised as movies. -pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "m4v", "mov", "mkv", "webm", "mxf"]; +pub use effectcraft_project::VIDEO_EXTENSIONS; /// File extensions recognised as 3D models (Advanced 3D model layers). pub const MODEL_EXTENSIONS: &[&str] = &["gltf", "glb", "obj", "stl", "fbx"]; diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index 13ca605e2..8dd4db672 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -168,9 +168,14 @@ fn tone_wav() -> Option { /// Frame `n` of `src` as RGBA PNG extracted by ffmpeg (Y'CbCr → RGB with the BT.709 matrix). fn reference_frame(src: &Path, n: usize) -> Image { + reference_frame_in(src, n, "bt709") +} + +/// [`reference_frame`] of a movie whose Y'CbCr is in `matrix` (`bt601` for Motion JPEG). +fn reference_frame_in(src: &Path, n: usize, matrix: &str) -> Image { let stem = src.file_stem().expect("stem").to_string_lossy().to_string(); let name = format!("ref_{stem}_{n}.png"); - let vf = format!("select='eq(n\\,{n})',scale=in_color_matrix=bt709:in_range=auto:out_range=pc:flags=bilinear+accurate_rnd+full_chroma_int,format=rgba"); + let vf = format!("select='eq(n\\,{n})',scale=in_color_matrix={matrix}:in_range=auto:out_range=pc:flags=bilinear+accurate_rnd+full_chroma_int,format=rgba"); let p = ffmpeg_fixture(&name, &["-i", src.to_str().expect("utf8"), "-vf", &vf, "-fps_mode", "passthrough", "-frames:v", "1"]).expect("ffmpeg"); load_png(&p) } @@ -472,12 +477,12 @@ fn dnxhd_mxf() -> Option { ) } -/// MXF is importable footage (After Effects imports it), whatever the extension's case. +/// MXF and AVI are importable footage (After Effects imports them), whatever the extension's case. #[test] -fn mxf_is_importable() { - assert!(effectcraft_media::is_importable(Path::new("/media/A001C003.mxf"))); - assert!(effectcraft_media::is_importable(Path::new("/media/A001C003.MXF"))); - assert!(!effectcraft_media::is_importable(Path::new("/media/A001C003.avi")), "no AVI demuxer yet"); +fn mxf_and_avi_are_importable() { + for name in ["A001C003.mxf", "A001C003.MXF", "capture.avi", "CAPTURE.AVI"] { + assert!(effectcraft_media::is_importable(&Path::new("/media").join(name)), "{name}"); + } } #[test] @@ -523,6 +528,124 @@ fn mxf_dnxhd_frames() { } } +fn mjpeg_avi() -> Option { + ffmpeg_fixture( + "testsrc2_mjpeg_pcm.avi", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=size=640x360:rate=25:duration=2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=2", + "-c:v", + "mjpeg", + "-q:v", + "2", + "-pix_fmt", + "yuvj420p", + "-c:a", + "pcm_s16le", + "-ac", + "2", + ], + ) +} + +#[test] +fn avi_mjpeg_with_pcm_audio() { + let Some(p) = mjpeg_avi() else { return }; + let f = probe(&p).expect("probe"); + assert_eq!(f.kind, FootageKind::Video); + assert_eq!((f.width, f.height), (640, 360)); + assert_eq!(f.frame_rate, FrameRate::FPS_25); + assert!((f.duration.seconds() - 2.0).abs() < 0.1, "duration {:?}", f.duration.seconds()); + assert!(f.has_video && f.has_audio); + assert_eq!(f.codec, "Motion JPEG"); + let pool = MediaPool::new(); + for n in [0usize, 1, 30, 7, 49] { + let img = pool.frame_at(&f, t_of(f.frame_rate, n as i64)).expect("frame"); + let db = psnr(&img, &reference_frame_in(&p, n, "bt601")); + assert!(db > 35.0, "frame {n}: {db:.2} dB"); + } + let s = pool.audio_samples(&f, Tick(TICKS_PER_SECOND / 2), 4800, 48_000); + assert_eq!(s.len(), 9600); + let rms = (s.iter().map(|v| v * v).sum::() / s.len() as f32).sqrt(); + assert!((rms - 0.0625).abs() < 0.01, "rms {rms}"); +} + +/// An AVI whose video FilmCraft can't decode imports, and says why it shows no picture (as an +/// unsupported stream in any other container does), rather than failing as a broken file. +#[test] +fn avi_with_an_unsupported_codec_says_so() { + let Some(p) = ffmpeg_fixture("testsrc2_xvid.avi", &["-f", "lavfi", "-i", "testsrc2=size=320x180:rate=25:duration=1", "-c:v", "mpeg4", "-vtag", "XVID"]) + else { + return; + }; + let f = probe(&p).expect("a valid AVI imports"); + assert_eq!((f.kind, f.width, f.height), (FootageKind::Video, 320, 180)); + let pool = MediaPool::new(); + if pool.frame(ItemId(1), &f, Tick::ZERO).is_some() { + // FilmCraft decodes MPEG-4 Part 2 now: nothing to report + assert_eq!(pool.problem(&f), None); + return; + } + let why = pool.problem(&f).expect("a failed frame leaves a reason"); + assert!(why.contains("MPEG-4 Part 2") && why.contains("isn't supported"), "{why}"); +} + +/// A file that only looks like an AVI is an import error that says so. +#[test] +fn an_invalid_avi_is_an_import_error() { + let e = probe_bytes("broken.avi", Arc::from(&b"RIFF\x10\0\0\0AVI LIST\0\0\0\0garbage"[..])).expect_err("not an AVI"); + assert!(e.to_string().contains("invalid AVI"), "{e}"); +} + +/// A damaged AVI imports: a truncated recording as far as it goes, and a packet that doesn't +/// decode loses its own frame only, with a reason that says the file is damaged rather than +/// unsupported. +#[test] +fn a_damaged_avi_loses_only_what_is_damaged() { + let Some(p) = mjpeg_avi() else { return }; + let bytes = std::fs::read(&p).expect("read"); + let full = probe(&p).expect("probe"); + let pool = MediaPool::new(); + + let cut: Arc<[u8]> = bytes[..bytes.len() / 3].into(); + let f = probe_bytes("cut.avi", cut.clone()).expect("a truncated AVI imports"); + assert!(f.duration > Tick::ZERO && f.duration < full.duration, "{:?} of {:?}", f.duration, full.duration); + pool.add_bytes("cut.avi", cut); + let last = f.frame_rate.frame_at(f.duration - Tick(1)); + assert!(pool.frame(ItemId(1), &f, t_of(f.frame_rate, last)).is_some(), "its last frame"); + + // Garble the JPEG data of the video chunk of frame 10. + let mut bad = bytes.clone(); + let mut at = bad.windows(4).position(|w| w == b"movi").expect("movi") + 4; + let mut video = 0; + while let Some(head) = bad.get(at..at + 8) { + let size = u32::from_le_bytes([head[4], head[5], head[6], head[7]]) as usize; + if &head[2..4] == b"dc" { + if video == 10 { + bad[at + 8 + 200..at + 8 + size.min(3000)].iter_mut().enumerate().for_each(|(i, b)| *b = if i % 7 == 0 { 0xFF } else { 0 }); + break; + } + video += 1; + } + at += 8 + size + size % 2; + } + assert_eq!(video, 10, "found frame 10's chunk"); + let f = probe_bytes("bad.avi", bad.clone().into()).expect("a damaged AVI imports"); + pool.add_bytes("bad.avi", bad.into()); + assert!(pool.frame(ItemId(2), &f, t_of(f.frame_rate, 10)).is_none(), "the damaged frame"); + for n in [9, 11, 30] { + assert!(pool.frame(ItemId(2), &f, t_of(f.frame_rate, n)).is_some(), "frame {n}"); + } + let why = pool.problem(&f).expect("a failed frame leaves a reason"); + assert!(why.contains("can't be decoded") && !why.contains("isn't supported"), "{why}"); +} + #[test] fn png_sequence_import() { let Some(first) = png_sequence() else { return }; diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 16d782165..70a85a718 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1089,6 +1089,11 @@ impl FieldOrder { } } +/// File extensions of the movie containers footage is imported from (FilmCraft reads them). The +/// import and proxy file dialogs, the Media Browser and `effectcraft_media::is_importable` all go +/// by this list. +pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "m4v", "mov", "mkv", "webm", "mxf", "avi"]; + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum AlphaMode { #[default] From e26b4af5c07fe8dd29139c71fe5e0a4848d26a43 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 11:35:05 -0400 Subject: [PATCH 07/33] Media Browser: list movies by the importer's list (#1701) --- crates/engine/src/media_browser.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/engine/src/media_browser.rs b/crates/engine/src/media_browser.rs index b394ad937..775e3db98 100644 --- a/crates/engine/src/media_browser.rs +++ b/crates/engine/src/media_browser.rs @@ -197,7 +197,6 @@ fn sort_entries(entries: &mut [Entry]) { entries.sort_by_cached_key(|e| (!e.is_dir, e.name.to_lowercase())); } -const VIDEO: &[&str] = &["mp4", "m4v", "mov", "webm", "mkv", "avi", "mxf", "gif"]; const IMAGE: &[&str] = &["png", "jpg", "jpeg", "tif", "tiff", "exr", "bmp", "webp", "psd", "psb", "svg", "tga", "dpx", "cin", "hdr"]; const AUDIO: &[&str] = &["wav", "aif", "aiff", "mp3", "m4a", "aac", "flac", "ogg", "opus"]; const DATA: &[&str] = &["json", "csv", "tsv", "mgjson"]; @@ -207,7 +206,8 @@ const MODEL: &[&str] = &["gltf", "glb", "obj", "stl", "fbx"]; pub fn kind_of(path: &str) -> Option<&'static str> { let ext = std::path::Path::new(path).extension()?.to_str()?.to_ascii_lowercase(); let e = ext.as_str(); - if VIDEO.contains(&e) { + // (An animated GIF plays as a movie.) + if e == "gif" || effectcraft_project::VIDEO_EXTENSIONS.contains(&e) { Some("video") } else if IMAGE.contains(&e) { Some("image") @@ -420,6 +420,15 @@ mod tests { std::fs::remove_dir_all(&dir).unwrap(); } + /// Every movie type the importer reads lists as video, as an animated GIF does. + #[test] + fn movies_list_as_video() { + for ext in effectcraft_project::VIDEO_EXTENSIONS.iter().chain(&["gif"]) { + assert_eq!(kind_of(&format!("/clips/A.{}", ext.to_uppercase())), Some("video"), "{ext}"); + } + assert_eq!(kind_of("/clips/capture.avi"), Some("video")); + } + #[test] fn virtual_tree_lists_like_a_folder() { let mut t = VirtualTree::default(); From 93a9295125e5487c0b9e99cb904cd07bb340901b Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 11:35:05 -0400 Subject: [PATCH 08/33] File dialogs: offer every movie type the importer reads (#1701) Import and Set Proxy build their filters from the shared movie list, so they offer AVI; the web build's file picker gets the same list. --- crates/ui-egui/src/menus.rs | 19 ++++++++++++++----- crates/ui-egui/tests/ui_menus.rs | 20 ++++++++++++-------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 26a7b66f6..eda7ccf7e 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -1337,6 +1337,8 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("paths", Ask::Import), @@ -1366,7 +1368,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Open(&["json", "csv", "txt"])), "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), "essential.importTemplate" => ("path", Ask::Open(&["ectemplate"])), - "file.setProxy" => ("path", Ask::Open(&["mp4", "mov", "m4v", "mkv", "webm", "mxf", "png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"])), + "file.setProxy" => ("path", Ask::OpenFootage(&["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"])), _ => return None, }; if params.get(key).is_some() @@ -1377,19 +1379,22 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option>(); // A host whose modal dialogs would run a nested event loop inside this frame (macOS: a // drag over the window then re-entered winit's event handler and aborted, #511) shows the // dialog without blocking; the command continues when the answer arrives. if let Some(spawn) = app.hooks.file_dialog_async.as_ref() { let request = match ask { Ask::Folder => crate::FileDialog::Folder, - Ask::Import => crate::FileDialog::Open { exts: MEDIA.iter().map(|e| e.to_string()).collect(), many: key == "paths" }, + Ask::Import => crate::FileDialog::Open { exts: footage(MEDIA).iter().map(|e| e.to_string()).collect(), many: key == "paths" }, Ask::OpenProject => crate::FileDialog::OpenProject, Ask::Open(exts) => crate::FileDialog::Open { exts: exts.iter().map(|e| e.to_string()).collect(), many: false }, + Ask::OpenFootage(more) => crate::FileDialog::Open { exts: footage(more).iter().map(|e| e.to_string()).collect(), many: false }, Ask::Save(default) => crate::FileDialog::save(default), }; let tx = app.dialog_answers.0.clone(); @@ -1409,7 +1414,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option { let Some(f) = app.hooks.pick_files.as_ref() else { return Some(Err("no file dialog available (pass `paths`)".into())) }; - picked_value(key, f(MEDIA)) + picked_value(key, f(&footage(MEDIA))) } Ask::OpenProject => { let Some(f) = app.hooks.pick_open_project.as_ref() else { return Some(Err("no file dialog available (pass `path`)".into())) }; @@ -1419,6 +1424,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option { + let Some(f) = app.hooks.pick_files.as_ref() else { return Some(Err("no file dialog available (pass `path`)".into())) }; + f(&footage(more)).first().map(|s| json!(s)) + } Ask::Save(default) => { let Some(picked) = app.hooks.save_dialog(default) else { return Some(Err("no file dialog available (pass `path`)".into())) }; picked.map(|s| json!(s)) diff --git a/crates/ui-egui/tests/ui_menus.rs b/crates/ui-egui/tests/ui_menus.rs index ca4c8cabc..ce6b97800 100644 --- a/crates/ui-egui/tests/ui_menus.rs +++ b/crates/ui-egui/tests/ui_menus.rs @@ -205,10 +205,10 @@ fn relink_restores_decoded_pixels_and_invalidates_viewer_content() { std::fs::remove_dir_all(dir).unwrap(); } -/// #767: File ▸ Import ▸ File… offers MXF footage (OP1a / OP-Atom from broadcast cameras and -/// Avid), which the Media Browser lists and the importer reads. +/// #767, #1701: File ▸ Import ▸ File… and the proxy dialog offer every movie type the importer +/// reads (MXF from broadcast cameras and Avid, AVI, …), the list the Media Browser goes by too. #[test] -fn import_dialog_offers_mxf() { +fn import_and_proxy_dialogs_offer_every_movie_type() { use std::cell::RefCell; use std::rc::Rc; @@ -219,11 +219,15 @@ fn import_dialog_offers_mxf() { *seen.borrow_mut() = exts.iter().map(|e| e.to_string()).collect(); Vec::new() })); - let result = effectcraft_ui_egui::menus::invoke(&mut app, &egui::Context::default(), "file.import", json!({})).unwrap(); - assert!(result.is_null(), "cancelled: nothing imported"); - let offered = offered.borrow(); - assert!(offered.iter().any(|e| e == "mxf"), "offered {offered:?}"); - assert!(offered.iter().any(|e| e == "mov")); + for (command, still) in [("file.import", "psd"), ("file.setProxy", "png")] { + let result = effectcraft_ui_egui::menus::invoke(&mut app, &egui::Context::default(), command, json!({})).unwrap(); + assert!(result.is_null(), "{command} cancelled: nothing done"); + let offered = offered.borrow(); + for ext in effectcraft_project::VIDEO_EXTENSIONS.iter().chain([&still]) { + assert!(offered.iter().any(|e| e == ext), "{command} offers {ext}: {offered:?}"); + } + assert!(offered.iter().any(|e| e == "avi"), "{command}: {offered:?}"); + } } /// #1059: opening the demo from the start screen and immediately pressing Spacebar previews From 84d0f567656a0ba1b68da5ccbf804c64d2cdc74f Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 11:35:05 -0400 Subject: [PATCH 09/33] Docs: AVI import (#1701) --- docs/file-format-parity.md | 2 +- docs/gaps.md | 2 +- docs/target-app-parity.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 9e2249e9b..f6e8913c4 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -65,7 +65,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | Apple ProRes (422 family, 4444) | R/W | yes | yes | good; 4444 colour tag wrong (#340); no ProRes RAW | | WebM / Matroska (VP9, AV1, Opus) | R (limited) | yes, incl. VP9 alpha | yes | beyond After Effects | | MXF (OP1a / OP-Atom) | R/W | yes (FilmCraft's demuxer: AVC, MPEG-2 incl. XDCAM / D-10, VC-3 / DNxHD, ProRes; PCM audio) | none | partial: import (#767), checked against ffmpeg; no MXF output | -| AVI | R | none | none | Media Browser lists it | +| AVI | R | yes (FilmCraft's demuxer: AVI 1.0 / OpenDML; Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3 audio) | none | partial: import (#1701), checked against ffmpeg; DV, MPEG-4 Part 2 (DivX / Xvid) and other legacy codecs import but show why they have no picture; no AVI output | | DNxHD / DNxHR | R/W | unverified (FilmCraft decodes it; reachable only inside `.mov`) | none | — | | BRAW, R3D, ARRIRAW | R (plug-ins / SDKs) | none | none | proprietary SDKs | | Variable-frame-rate phone video | R | unverified | — | needs a real-media corpus | diff --git a/docs/gaps.md b/docs/gaps.md index a20e779cd..bda5eba76 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -73,7 +73,7 @@ are grouped by area; the issue tracker has the rest. | 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Since then: the Object Matte tool in the viewer (click / drag a box, hover crosshair), the box as the model's box prompt, background tracking with progress and cancel, Freeze, and segmentations kept in the disk cache (26.5). Missing: a comparison with After Effects ([plan](fidelity/object-matte.md)) | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | | 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it (answered: CUDA / NVENC need Nvidia's proprietary SDK through FFI, which the no-`unsafe` rule forbids); open PR #819 adds hardware decoding via `filmcraft-platform`; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 10 languages (ja, zh-hans, zh-hant, uk, es, pt-br, ru, fr, de, cs: 624 rows each); panels in 6 (ja, zh-hans, zh-hant, uk, pt-br, cs); effect names translated in Simplified Chinese (306, plus the generated Effect menu), their ~1,650 parameter names untranslated; Korean and Italian (After Effects ships them) absent; Hangul probe added (#599 closed) | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | -| 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists AVI; `media::is_importable` has none of them (Targa and Radiance HDR import since #745; DPX and Cineon since #756, log shown as stored: no Cineon Settings conversion yet; MXF OP1a / OP-Atom since #767; FilmCraft has no AVI demuxer); #431 and #324 closed | Confusing failures on common VFX formats | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | +| 17 | **Media Browser lists files import refuses** | The Media Browser, the import and proxy dialogs and `media::is_importable` share one movie list since #1701, AVI included (Targa and Radiance HDR import since #745; DPX and Cineon since #756, log shown as stored: no Cineon Settings conversion yet; MXF OP1a / OP-Atom since #767; AVI since #1701, without DV, MPEG-4 Part 2 or other legacy codecs yet); #431 and #324 closed | Confusing failures on common VFX formats | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | | 18 | **Relative paths and relinking** | Projects store absolute footage paths; no relink when a project moves | Moving or sharing a project breaks it | 6–12 h | G4 · [file-format-parity.md](file-format-parity.md) | | 19 | **Plug-in ecosystem** | No After Effects SDK hosting (#600, #327 OpenFX, #334, #376 plug-in manager); our WebAssembly API has no third-party plug-ins | Many pro projects depend on Trapcode, Element, Saber… | 40–80 h for API growth; SDK hosting needs an owner decision | G8 · [scripting-parity.md](scripting-parity.md) | | 20 | **Multi-monitor and video output** | #486: any panel can move into a native window of its own (panel menu ▸ Move Panel to New Window, `window.panelWindow`) and so onto another screen, restored with the workspace; still missing: dragging tabs between those windows, app shortcuts while one has the focus, named workspaces remembering them, Mercury Transmit to video hardware (we have a second preview window only) | Two-screen setups are standard for professionals | 10–20 h | G2 · [hardware-parity.md](hardware-parity.md) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index eac744b6f..74111b27d 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -114,7 +114,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for | Area | Weight | Breadth | Ready | Hours | Biggest gaps (details in [gaps.md](gaps.md)) | |---|---:|---:|---:|---:|---| -| Project, footage and import | 8% | 92% | 55% | 25–45 | No relative paths or relinking when a project moves; Media Browser lists AVI, which it can't import (TGA/HDR/DPX/MXF import since #745, #756, #767); #431, #324, #552 and #351 are closed | +| Project, footage and import | 8% | 92% | 55% | 25–45 | No relative paths or relinking when a project moves; TGA/HDR/DPX/MXF/AVI import since #745, #756, #767, #1701 (no DV or MPEG-4 Part 2 AVI yet); #431, #324, #552 and #351 are closed | | Compositions, layers and compositing | 13% | 98% | 60% | 30–55 | 32-bpc blend modes differed from After Effects (#494, closed); expand precomps in place (#601); #527, #510, #513, #539 and #540 are closed | | Animation: keyframes, graph editor, motion paths, puppet | 15% | 97% | 55% | 35–60 | Only scalar ease is measured against After Effects (168 samples); puppet lag (#586); #456, #457, #355, #553 and #318 are closed; Proportional Scrubbing (26.2) | | Expressions and scripting | 7% | 93% | 65% | 20–40 | `.jsxbin`, Startup/Shutdown folders, script shortcuts (`#include` landed, #759); external editor (#364); see [scripting-parity.md](scripting-parity.md) | From c7715a3f2d43d79e5d419c29aba736444443dd4d Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 14:02:18 -0400 Subject: [PATCH 10/33] Footage: a proxy in use that can't be decoded says why (#1701) The picture comes from an item's proxy while Use Proxy is on, but the Project panel, the Viewer, project.summary and footage.info asked only about the item's own file: a proxy that couldn't be decoded (an AVI with an unsupported codec, say) left the layer empty with no reason given. They now ask FootageSource::item_problem, which gives the proxy's reason, said to be the proxy's, for footage and compositions alike. --- crates/engine/src/commands/footage_panel.rs | 5 +-- crates/engine/src/commands/query.rs | 8 ++-- crates/engine/src/tests_panels.rs | 44 +++++++++++++++++++ crates/render/src/lib.rs | 12 +++++ crates/ui-egui/src/panels/project.rs | 9 ++-- crates/ui-egui/src/panels/viewer.rs | 11 ++--- crates/ui-egui/tests/ui_unreadable_footage.rs | 39 +++++++++++----- 7 files changed, 97 insertions(+), 31 deletions(-) diff --git a/crates/engine/src/commands/footage_panel.rs b/crates/engine/src/commands/footage_panel.rs index 1bd0d2236..7336d0c08 100644 --- a/crates/engine/src/commands/footage_panel.rs +++ b/crates/engine/src/commands/footage_panel.rs @@ -207,10 +207,7 @@ fn info(s: &mut Session, _: &Value) -> Result { let item = s.project.item(v.item); let name = item.map(|i| i.name.clone()).unwrap_or_default(); let frame = fr.frame_at(v.time); - let problem = match item.map(|i| &i.kind) { - Some(ItemKind::Footage(f)) => s.footage.problem(f), - _ => None, - }; + let problem = item.and_then(|i| s.footage.item_problem(i)); Ok( json!({"item": v.item.0, "name": name, "time": v.time.seconds(), "frame": frame, "timecode": timecode(item, fr, frame), "in": v.in_point.map(|t| t.seconds()), "out": v.out_point.map(|t| t.seconds()), "duration": dur.map(|d| d.seconds()), "frameRate": fr.as_f64(), "problem": problem}), ) diff --git a/crates/engine/src/commands/query.rs b/crates/engine/src/commands/query.rs index 2ea8048fa..ed5eeffd1 100644 --- a/crates/engine/src/commands/query.rs +++ b/crates/engine/src/commands/query.rs @@ -46,11 +46,9 @@ fn summary(s: &mut Session, _: &Value) -> Result { o["layers"] = json!(c.layers.len()); o["frameRate"] = json!(c.frame_rate.as_f64()); } - // Footage that shows no picture although its file is there (a codec or profile the - // decoders lack): why, so an agent can tell the user. - if let ItemKind::Footage(f) = &i.kind - && let Some(why) = s.footage.problem(f) - { + // Footage (or the proxy in use) that shows no picture although its file is there (a + // codec or profile the decoders lack): why, so an agent can tell the user. + if let Some(why) = s.footage.item_problem(i) { o["problem"] = json!(why); } o diff --git a/crates/engine/src/tests_panels.rs b/crates/engine/src/tests_panels.rs index b442d7c3a..622088745 100644 --- a/crates/engine/src/tests_panels.rs +++ b/crates/engine/src/tests_panels.rs @@ -672,3 +672,47 @@ fn footage_that_cannot_be_decoded_reports_why() { s.execute("footage.open", json!({"item": item.0})).unwrap(); assert_eq!(s.execute("footage.info", json!({})).unwrap()["problem"], it["problem"]); } + +/// A proxy in use that can't be decoded leaves its item without a picture although the item's +/// own file is fine: `project.summary` and `footage.info` give the proxy's reason, said to be the +/// proxy's, for footage and compositions alike, and nothing once Use Proxy is off. +#[test] +fn a_proxy_in_use_that_cannot_be_decoded_reports_why() { + struct BrokenProxies; + impl FootageSource for BrokenProxies { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn problem(&self, f: &Footage) -> Option { + f.path.ends_with(".avi").then(|| format!("{} isn't supported by the built-in decoder", f.codec)) + } + } + let mut s = Session { footage: Arc::new(BrokenProxies), ..Default::default() }; + let comp = s.execute("comp.new", json!({"name": "C", "width": 64, "height": 36, "duration": 2})).unwrap()["comp"].as_u64().unwrap(); + let still = Footage { kind: FootageKind::Still, path: "a.png".into(), width: 64, height: 36, has_video: true, ..Default::default() }; + let item = std::sync::Arc::make_mut(&mut s.project).add_item("Plate", Default::default(), None, ItemKind::Footage(still)); + let proxy = Footage { + kind: FootageKind::Video, + path: "plate.avi".into(), + codec: "MPEG-4 Part 2".into(), + width: 32, + height: 18, + has_video: true, + ..Default::default() + }; + let want = json!("Proxy: MPEG-4 Part 2 isn't supported by the built-in decoder"); + let problem = |s: &mut Session, id: u64| { + let sum = s.execute("project.summary", json!({})).unwrap(); + sum["items"].as_array().unwrap().iter().find(|i| i["id"] == json!(id)).unwrap().get("problem").cloned() + }; + for id in [item.0, comp] { + let p = std::sync::Arc::make_mut(&mut s.project); + p.items.get_mut(&ItemId(id)).unwrap().proxy = Some(Box::new(effectcraft_project::Proxy { footage: proxy.clone(), enabled: true })); + assert_eq!(problem(&mut s, id), Some(want.clone()), "item {id}"); + std::sync::Arc::make_mut(&mut s.project).items.get_mut(&ItemId(id)).unwrap().proxy.as_mut().unwrap().enabled = false; + assert_eq!(problem(&mut s, id), None, "item {id}: its own file shows, and is fine"); + } + std::sync::Arc::make_mut(&mut s.project).items.get_mut(&item).unwrap().proxy.as_mut().unwrap().enabled = true; + s.execute("footage.open", json!({"item": item.0})).unwrap(); + assert_eq!(s.execute("footage.info", json!({})).unwrap()["problem"], want); +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index cfefd02db..b5aada454 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -130,6 +130,18 @@ pub trait FootageSource: Send + Sync { fn problem(&self, _footage: &Footage) -> Option { None } + /// [`Self::problem`] of what `item` shows: its proxy's while Use Proxy shows the proxy + /// instead (the reason then says it is the proxy's), else the footage item's own. A + /// composition has one only through its proxy. + fn item_problem(&self, item: &effectcraft_project::Item) -> Option { + match item.proxy.as_deref().filter(|px| px.enabled && px.footage.has_video) { + Some(px) => self.problem(&px.footage).map(|why| format!("Proxy: {why}")), + None => match &item.kind { + ItemKind::Footage(f) => self.problem(f), + _ => None, + }, + } + } } /// A render's view of a [`FootageSource`] whose frame reads can be called off with `cancel` diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 76309a5d0..ab5936765 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -678,12 +678,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("file.useProxy".into(), json!({"item": id.0}))); } } - // Footage that can't be decoded (no decoder for its codec / profile): a warning marker - // at the end of the name cell, the reason on hover. - let problem = match &it.kind { - ItemKind::Footage(f) => app.session.footage.problem(f), - _ => None, - }; + // Footage (or the proxy in use) that can't be decoded (no decoder for its codec / + // profile): a warning marker at the end of the name cell, the reason on hover. + let problem = app.session.footage.item_problem(it); let name_clip = Rect::from_min_max(pos2(x0 + 30.0, r.min.y), pos2(r.min.x + name_w + 4.0 - if problem.is_some() { 16.0 } else { 0.0 }, r.max.y)).intersect(list); if let Some(why) = &problem { diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 51bab7fda..a50c30e53 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1155,13 +1155,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add("viewer.warpBanner", r, &text); } - // Footage that can't be decoded (no decoder for its codec / profile): say why on the layer - // instead of leaving a black picture. + // Footage (or the proxy in use for footage or a precomp) that can't be decoded (no decoder + // for its codec / profile): say why on the layer instead of leaving a black picture. let mut notices = 0; for layer in comp.layers.iter().filter(|l| l.is_active_at(time) && l.switches.video) { - let effectcraft_engine::project::LayerSource::Footage { item } = &layer.source else { continue }; - let Some(effectcraft_engine::project::ItemKind::Footage(f)) = app.session.project.item(*item).map(|i| &i.kind) else { continue }; - let Some(why) = app.session.footage.problem(f) else { continue }; + let (effectcraft_engine::project::LayerSource::Footage { item } | effectcraft_engine::project::LayerSource::Comp { item }) = &layer.source else { + continue; + }; + let Some(why) = app.session.project.item(*item).and_then(|i| app.session.footage.item_problem(i)) else { continue }; if notices >= 3 { break; } diff --git a/crates/ui-egui/tests/ui_unreadable_footage.rs b/crates/ui-egui/tests/ui_unreadable_footage.rs index 9b48b9531..b1b12d6d2 100644 --- a/crates/ui-egui/tests/ui_unreadable_footage.rs +++ b/crates/ui-egui/tests/ui_unreadable_footage.rs @@ -1,10 +1,11 @@ -//! Footage whose codec the decoders lack says so: a warning marker in the Project panel and a -//! notice on the layer in the Composition viewer, instead of a black picture with no reason. +//! Footage (or the proxy in use for it) whose codec the decoders lack says so: a warning marker in +//! the Project panel and a notice on the layer in the Composition viewer, instead of a black +//! picture with no reason. use std::sync::Arc; use effectcraft_engine::Session; -use effectcraft_engine::project::{Footage, FootageKind, ItemId, ItemKind}; +use effectcraft_engine::project::{Footage, FootageKind, ItemId, ItemKind, Proxy}; use effectcraft_engine::render::FootageSource; use effectcraft_render::Image; use effectcraft_time::Tick; @@ -14,19 +15,19 @@ use serde_json::json; const WHY: &str = "H.264 (High 4:2:2) isn't supported by the built-in decoder: chroma_format_idc 2 (only 4:2:0 is implemented)"; -/// A source that decodes nothing and, for `broken` movies, says why. -struct Stub(bool); +/// A source that decodes nothing and says why for the files `broken` picks. +struct Stub(fn(&Footage) -> bool); impl FootageSource for Stub { fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { None } - fn problem(&self, _: &Footage) -> Option { - self.0.then(|| WHY.to_string()) + fn problem(&self, f: &Footage) -> Option { + (self.0)(f).then(|| WHY.to_string()) } } -fn harness(broken: bool) -> (Harness<'static, EffectcraftApp>, ItemId) { +fn harness(broken: fn(&Footage) -> bool, proxy: Option) -> (Harness<'static, EffectcraftApp>, ItemId) { let mut s = Session { footage: Arc::new(Stub(broken)), ..Default::default() }; s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap(); let footage = Footage { @@ -38,7 +39,11 @@ fn harness(broken: bool) -> (Harness<'static, EffectcraftApp>, ItemId) { duration: Tick::from_seconds_f64(2.0), ..Default::default() }; - let clip = std::sync::Arc::make_mut(&mut s.project).add_item("Clip", Default::default(), None, ItemKind::Footage(footage)); + let p = std::sync::Arc::make_mut(&mut s.project); + let clip = p.add_item("Clip", Default::default(), None, ItemKind::Footage(footage)); + if let (Some(item), Some(footage)) = (p.items.get_mut(&clip), proxy) { + item.proxy = Some(Box::new(Proxy { footage, enabled: true })); + } s.execute("layer.addItem", json!({"item": clip.0})).unwrap(); s.execute("project.select", json!({"items": []})).unwrap(); let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| EffectcraftApp::new(s)); @@ -48,7 +53,7 @@ fn harness(broken: bool) -> (Harness<'static, EffectcraftApp>, ItemId) { #[test] fn undecodable_footage_is_marked_in_the_project_panel_and_the_viewer() { - let (h, clip) = harness(true); + let (h, clip) = harness(|_| true, None); let marker = h.state().auto.find(&format!("project.item.{}.problem", clip.0)).expect("a warning on the footage item").clone(); assert!(marker.label.contains("4:2:2"), "{marker:?}"); let notice = h.state().auto.query("viewer.footageProblem.").first().copied().expect("a notice in the viewer").clone(); @@ -57,7 +62,19 @@ fn undecodable_footage_is_marked_in_the_project_panel_and_the_viewer() { #[test] fn readable_footage_has_no_notice() { - let (h, clip) = harness(false); + let (h, clip) = harness(|_| false, None); assert!(h.state().auto.find(&format!("project.item.{}.problem", clip.0)).is_none()); assert!(h.state().auto.query("viewer.footageProblem.").is_empty()); } + +/// The clip's own file is fine, but the proxy in use for it can't be decoded: the item and the +/// layer say why, and that it is the proxy. +#[test] +fn an_undecodable_proxy_in_use_is_marked_as_the_proxy() { + let proxy = Footage { kind: FootageKind::Video, path: "clip_proxy.avi".into(), width: 32, height: 18, has_video: true, ..Default::default() }; + let (h, clip) = harness(|f| f.path.ends_with(".avi"), Some(proxy)); + let marker = h.state().auto.find(&format!("project.item.{}.problem", clip.0)).expect("a warning on the footage item").clone(); + assert!(marker.label.starts_with("Proxy: ") && marker.label.contains("4:2:2"), "{marker:?}"); + let notice = h.state().auto.query("viewer.footageProblem.").first().copied().expect("a notice in the viewer").clone(); + assert!(notice.label.contains(&format!("Proxy: {WHY}")), "{notice:?}"); +} From 0a57bb9fc21a909adb822c017f22a0fd497a840d Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 14:02:18 -0400 Subject: [PATCH 11/33] Docs: the AVI variants that don't import yet (#1701) --- docs/file-format-parity.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index f6e8913c4..8ca81f715 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -65,7 +65,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | Apple ProRes (422 family, 4444) | R/W | yes | yes | good; 4444 colour tag wrong (#340); no ProRes RAW | | WebM / Matroska (VP9, AV1, Opus) | R (limited) | yes, incl. VP9 alpha | yes | beyond After Effects | | MXF (OP1a / OP-Atom) | R/W | yes (FilmCraft's demuxer: AVC, MPEG-2 incl. XDCAM / D-10, VC-3 / DNxHD, ProRes; PCM audio) | none | partial: import (#767), checked against ffmpeg; no MXF output | -| AVI | R | yes (FilmCraft's demuxer: AVI 1.0 / OpenDML; Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3 audio) | none | partial: import (#1701), checked against ffmpeg; DV, MPEG-4 Part 2 (DivX / Xvid) and other legacy codecs import but show why they have no picture; no AVI output | +| AVI | R | yes (FilmCraft's demuxer: AVI 1.0 / OpenDML; Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3 audio) | none | partial: import (#1701), checked against ffmpeg; H.264 needs its parameter sets in the chunks (global headers and copies with an `avcC` record don't decode until storytold/filmcraft#2303), and a damaged video index loses the video (storytold/filmcraft#2302); DV, MPEG-4 Part 2 (DivX / Xvid) and other legacy codecs import but show why they have no picture; no AVI output | | DNxHD / DNxHR | R/W | unverified (FilmCraft decodes it; reachable only inside `.mov`) | none | — | | BRAW, R3D, ARRIRAW | R (plug-ins / SDKs) | none | none | proprietary SDKs | | Variable-frame-rate phone video | R | unverified | — | needs a real-media corpus | From f52ffb63ceb762638fb866a6074e8faae324b9c2 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sun, 18 Oct 2026 12:59:24 -0400 Subject: [PATCH 12/33] Viewer: no proxy notice on a collapsed precomp, which draws its own layers (#1701) --- crates/ui-egui/src/panels/viewer.rs | 28 +++++++++++-------- crates/ui-egui/tests/ui_unreadable_footage.rs | 26 ++++++++++++++++- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 795ac0650..bec2440d6 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -8,8 +8,8 @@ use effectcraft_engine::commands::shape_tool::PaintKind; use effectcraft_engine::effects::puppet::PinKind; use effectcraft_engine::geom::{Mat3, vec2 as gv2}; use effectcraft_engine::project::{Comp, Layer, LayerId, LayerSource}; -use effectcraft_engine::render::EvalCtx; use effectcraft_engine::render::three_d::{self, CameraState, View3D}; +use effectcraft_engine::render::{EvalCtx, Renderer}; use effectcraft_engine::time::Tick; use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::json; @@ -1194,16 +1194,21 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } // Footage (or the proxy in use for footage or a precomp) that can't be decoded (no decoder - // for its codec / profile): say why on the layer instead of leaving a black picture. - let mut notices = 0; - for layer in comp.layers.iter().filter(|l| l.is_active_at(time) && l.switches.video) { - let (effectcraft_engine::project::LayerSource::Footage { item } | effectcraft_engine::project::LayerSource::Comp { item }) = &layer.source else { - continue; - }; - let Some(why) = app.session.project.item(*item).and_then(|i| app.session.footage.item_problem(i)) else { continue }; - if notices >= 3 { - break; - } + // for its codec / profile): say why on the layer instead of leaving a black picture. A + // collapsed precomp draws its own layers, never the comp's proxy. + let problems: Vec<(&Layer, String)> = { + let r = Renderer::new(&snap_project, &*app.session.footage, Default::default()); + comp.layers + .iter() + .filter(|l| l.is_active_at(time) && l.switches.video && r.collapsed(&ectx, l).is_none()) + .filter_map(|layer| { + let (LayerSource::Footage { item } | LayerSource::Comp { item }) = &layer.source else { return None }; + Some((layer, snap_project.item(*item).and_then(|i| app.session.footage.item_problem(i))?)) + }) + .take(3) + .collect() + }; + for (notices, (layer, why)) in problems.into_iter().enumerate() { let (m, _) = l2c(&ectx, layer); let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); let c = m.apply(effectcraft_engine::geom::vec2(w as f64 / 2.0, h as f64 / 2.0)); @@ -1215,7 +1220,6 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { painter.rect_stroke(r, 3.0, Stroke::new(1.0, t.warning), StrokeKind::Inside); painter.galley(pos2(r.min.x + 12.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); app.auto.add(&format!("viewer.footageProblem.{}", layer.id.0), r, &text); - notices += 1; } // Free transform box of a path (double-click a path with the Selection tool). diff --git a/crates/ui-egui/tests/ui_unreadable_footage.rs b/crates/ui-egui/tests/ui_unreadable_footage.rs index b1b12d6d2..63e3a75c2 100644 --- a/crates/ui-egui/tests/ui_unreadable_footage.rs +++ b/crates/ui-egui/tests/ui_unreadable_footage.rs @@ -45,10 +45,14 @@ fn harness(broken: fn(&Footage) -> bool, proxy: Option) -> (Harness<'st item.proxy = Some(Box::new(Proxy { footage, enabled: true })); } s.execute("layer.addItem", json!({"item": clip.0})).unwrap(); + (open(s), clip) +} + +fn open(mut s: Session) -> Harness<'static, EffectcraftApp> { s.execute("project.select", json!({"items": []})).unwrap(); let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| EffectcraftApp::new(s)); h.run_steps(4); - (h, clip) + h } #[test] @@ -78,3 +82,23 @@ fn an_undecodable_proxy_in_use_is_marked_as_the_proxy() { let notice = h.state().auto.query("viewer.footageProblem.").first().copied().expect("a notice in the viewer").clone(); assert!(notice.label.contains(&format!("Proxy: {WHY}")), "{notice:?}"); } + +/// A precomp whose proxy can't be decoded: the layer says so while it renders through the +/// proxy, but not with Collapse Transformations on, which draws the precomp's own layers. +#[test] +fn a_collapsed_precomp_has_no_notice_for_its_proxy() { + for collapse in [false, true] { + let mut s = Session { footage: Arc::new(Stub(|f| f.path.ends_with(".avi"))), ..Default::default() }; + s.execute("comp.new", json!({"name": "Inner", "width": 64, "height": 36, "duration": 4})).unwrap(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap(); + let p = Arc::make_mut(&mut s.project); + let inner = p.items.values_mut().find(|i| i.name == "Inner").unwrap(); + let proxy = Footage { kind: FootageKind::Video, path: "inner_proxy.avi".into(), width: 64, height: 36, has_video: true, ..Default::default() }; + inner.proxy = Some(Box::new(Proxy { footage: proxy, enabled: true })); + let inner = inner.id; + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + s.execute("layer.setSwitch", json!({"layers": ["#1"], "switch": "collapse", "value": collapse})).unwrap(); + let h = open(s); + assert_eq!(h.state().auto.query("viewer.footageProblem.").is_empty(), collapse, "collapse {collapse}"); + } +} From 0378816923b0377d6ce3def7dc83006450556fa0 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Tue, 27 Oct 2026 19:54:42 -0400 Subject: [PATCH 13/33] Export: pin the H.264 movie bytes the FilmCraft encoder this branch uses writes (#1701) --- crates/export/tests/it/movie_bytes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index 5f1f8eb6b..c761f274c 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,7 +94,7 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); - assert_eq!((data.len(), fnv1a(&data)), (104_161, 10_466_446_817_305_548_344), "H.264 MP4"); + assert_eq!((data.len(), fnv1a(&data)), (104_322, 11_051_847_956_767_423_392), "H.264 MP4"); } #[test] From 8bb5dbb81e8651072f7e19220342225f8bd2d7f1 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 17:52:57 -0400 Subject: [PATCH 14/33] Media: downmix AAC in the order its channel configuration decodes in (#1701) FilmCraft reorders AAC configurations 3 to 6, 11 and 12 to WAVE/SMPTE order and leaves configuration 7 and program config elements as coded, so the channel count alone can't say which order arrives: ffmpeg writes 7.1 as configuration 7 (C L R first) and 6.1 as a PCE it fills in its own order (L R C LFE Cs Ls Rs), while FilmCraft's encoder writes a centre-first PCE. The order is now read from an MP4/MOV's AudioSpecificConfig when the source opens; other containers keep a channel-count guess. The ffmpeg oracle test gains configuration 12 (ffmpeg's 7.1 relabelled) and skips PCE files ffmpeg 4.2 marks as stereo, which FilmCraft reads as two channels. --- Cargo.lock | 1 + crates/media/Cargo.toml | 1 + crates/media/src/downmix.rs | 123 +++++++++++++++++++++++-------- crates/media/src/pool.rs | 77 ++++++++++++------- crates/media/src/probe.rs | 11 +++ crates/media/tests/it/downmix.rs | 72 ++++++++++++++---- 6 files changed, 211 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7b774557..627da7cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1632,6 +1632,7 @@ dependencies = [ "effectcraft-svg", "effectcraft-time", "exr", + "filmcraft-aac", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 923a28afa..d3f815bba 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -21,6 +21,7 @@ effectcraft-pdf = { workspace = true } effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } +filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 1ad9721ba..745f28065 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -12,9 +12,10 @@ //! //! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM, //! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs, -//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. FilmCraft's AAC-LC -//! decoder instead delivers AAC's own order (C L R first, LFE last), so AAC gets its own tables. -//! More than 8 channels carry no known layout: only the first two (front L/R) are used. +//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC arrives in either +//! that order or AAC's own (C L R first, LFE last) depending on its channel configuration +//! ([`aac_order`]). More than 8 channels carry no known layout: only the first two (front L/R) +//! are used. use std::f32::consts::FRAC_1_SQRT_2; @@ -64,28 +65,59 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } -/// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 -/// channels) and 12 (7.1, 8 channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs, -/// L R C LFE Lb Rb Ls Rs), everything else (mono, stereo, program-config layouts such as the 6.1 -/// ffmpeg writes) in AAC's element order. Configuration 11 (6.1) and 7 (7.1 with front wides) -/// can't be told from those by the channel count and are read in the order of the commoner one. -pub fn decoded_aac_gains(count: usize) -> Vec { - match count { - // C L R Cs, reordered to L R C Cs: the same gains, permuted. - 4 => { - let g = stereo_gains(4, true); - [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() - } - 3 | 5 | 6 | 8 => stereo_gains(count, false), - _ => stereo_gains(count, true), +/// The order a source's decoded channels arrive in. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Order { + /// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest. + #[default] + Wav, + /// AAC that FilmCraft reorders to WAVE / SMPTE (configurations 3 to 6, 11 and 12; 4 becomes + /// L R C Cs) or that ffmpeg wrote in it (a program config element listing the front pair + /// first: ffmpeg fills a PCE's elements in its own channel order and reads them back so). + AacWav, + /// AAC in its element order (C L R … LFE), which FilmCraft keeps: configuration 7 and a PCE + /// listing the centre first. + AacElements, + /// AAC whose configuration the container doesn't show us (only MP4 / MOV headers are read): + /// as [`Order::AacWav`] up to 7 channels (configurations 3 to 6, 11 and ffmpeg's 6.1 PCE), + /// as [`Order::AacElements`] above (configuration 7, ffmpeg's 7.1). + Aac, +} + +/// How FilmCraft's decoder hands over the AAC described by the AudioSpecificConfig `asc`. +pub fn aac_order(asc: &[u8]) -> Order { + let Ok(asc) = filmcraft_aac::AudioSpecificConfig::parse(asc) else { return Order::Aac }; + match (asc.channel_config, asc.pce) { + (3..=6 | 11 | 12, _) => Order::AacWav, + (0, Some(pce)) if pce.front.first().is_some_and(|&(cpe, _)| cpe) => Order::AacWav, + (0, None) => Order::Aac, + _ => Order::AacElements, } } -/// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a -/// channel is too short for count as silence, extra frames in `out` stay as they are. -pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = if aac { decoded_aac_gains(channels.len()) } else { stereo_gains(channels.len(), false) }; - mix(channels, &gains, out); +/// The gains for `count` channels arriving in `order`. +pub fn decoded_gains(count: usize, order: Order) -> Vec { + let order = match order { + Order::Aac if (3..=7).contains(&count) => Order::AacWav, + Order::Aac => Order::AacElements, + o => o, + }; + match (order, count) { + // C L R Cs, reordered to L R C Cs: the same gains, permuted. + (Order::AacWav, 4) => { + let g = stereo_gains(4, true); + [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() + } + (Order::AacElements, _) => stereo_gains(count, true), + _ => stereo_gains(count, false), + } +} + +/// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2` +/// frames); frames a channel is too short for count as silence, extra frames in `out` stay as +/// they are. +pub fn downmix_into(channels: &[Vec], order: Order, out: &mut [f32]) { + mix(channels, &decoded_gains(channels.len(), order), out); } fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { @@ -119,19 +151,46 @@ mod tests { } } + /// AudioSpecificConfigs as ffmpeg 8.1 and FilmCraft's encoder write them (48 kHz AAC-LC). + const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500"; + const PCE_FILMCRAFT_61: &str = "118004c845000108840000"; + + fn asc(hex: &str) -> Vec { + (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect() + } + #[test] - fn decoded_aac_is_in_wav_order_for_three_to_six_and_eight_channels() { - // FilmCraft hands AAC 3.0 to 5.1 and 7.1 over in the order of the plain layouts. - for n in [3, 5, 6, 8] { - assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); + fn aac_order_follows_the_channel_configuration() { + // AAC-LC at 48 kHz with channel configuration n + let config = |n: u8| aac_order(&[0x11, 0x80 | n << 3]); + for n in [3, 4, 5, 6, 11, 12] { + assert_eq!(config(n), Order::AacWav, "configuration {n}"); } - let g = decoded_aac_gains(4); + for n in [1, 2, 7] { + assert_eq!(config(n), Order::AacElements, "configuration {n}"); + } + assert_eq!(aac_order(&asc(PCE_FFMPEG_61)), Order::AacWav, "ffmpeg's 6.1 PCE lists L R first"); + assert_eq!(aac_order(&asc(PCE_FILMCRAFT_61)), Order::AacElements, "a PCE listing the centre first"); + assert_eq!(aac_order(&[]), Order::Aac); + } + + #[test] + fn decoded_aac_gains_follow_the_order() { + for n in [3, 5, 6, 7, 8] { + assert_eq!(decoded_gains(n, Order::AacWav), stereo_gains(n, false), "{n} channels"); + assert_eq!(decoded_gains(n, Order::AacElements), stereo_gains(n, true), "{n} channels"); + } + let g = decoded_gains(4, Order::AacWav); assert_eq!(g.len(), 4); assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - for n in [1, 2, 7, 9] { - assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); + assert_eq!(decoded_gains(4, Order::Wav), stereo_gains(4, false), "PCM 4 channels are quad"); + for n in [3, 4, 5, 6, 7] { + assert_eq!(decoded_gains(n, Order::Aac), decoded_gains(n, Order::AacWav), "{n} channels"); + } + for n in [1, 2, 8, 9] { + assert_eq!(decoded_gains(n, Order::Aac), stereo_gains(n, true), "{n} channels"); } } @@ -161,10 +220,10 @@ mod tests { #[test] fn ragged_and_empty_input_never_panics() { let mut out = vec![0.0; 8]; - downmix_into(&[], false, &mut out); - downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], true, &mut out); + downmix_into(&[], Order::Wav, &mut out); + downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], Order::Aac, &mut out); assert!(out.iter().all(|v| v.is_finite())); let mut none: [f32; 0] = []; - downmix_into(&vec![vec![1.0; 3]; 7], false, &mut none); + downmix_into(&vec![vec![1.0; 3]; 7], Order::Wav, &mut none); } } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index ed89bf7e8..193804447 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -12,6 +12,7 @@ use filmcraft_media::{FrameRequest, SharedSource}; use image::metadata::Orientation; use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned}; +use crate::downmix::Order; use crate::gif_anim::{self, Anim}; use crate::{MediaError, Result}; @@ -178,10 +179,37 @@ struct CacheState { last: HashMap, i64>, } -/// A movie/audio source by path and the turn its picture is shown with -/// ([`crate::probe::video_turn`]), opened by the first caller while the others wait for it -/// (`None`: failed to open; not retried until `forget`). -type SourceSlot = Arc>>; +/// A movie/audio source by path ([`Opened`]), opened by the first caller while the others wait +/// for it (`None`: failed to open; not retried until `forget`). +type SourceSlot = Arc>>; + +/// An opened movie/audio source, the turn its picture is shown with and the order its decoded +/// audio channels arrive in. +#[derive(Clone)] +struct Opened { + src: SharedSource, + turn: Orientation, + order: Order, +} + +impl Opened { + /// `src` with what its MP4/MOV `header` (ftyp and moov) says, if it has one. + fn new(src: SharedSource, header: Option<&[u8]>) -> Opened { + let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn); + let order = channel_order(&src, header.and_then(crate::probe::aac_config).as_deref()); + Opened { src, turn, order } + } +} + +/// The order `src`'s audio channels decode in, AAC's decided by its AudioSpecificConfig `asc` +/// when the container's header was read. +fn channel_order(src: &SharedSource, asc: Option<&[u8]>) -> Order { + match (src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")), asc) { + (false, _) => Order::Wav, + (true, Some(asc)) => crate::downmix::aac_order(asc), + (true, None) => Order::Aac, + } +} /// An animated GIF by path, as [`SourceSlot`] (the error message when it failed to load). type GifSlot = Arc, String>>>; @@ -514,7 +542,7 @@ impl MediaPool { /// [`MediaPool::decode_audio`], failing when the source can't be opened or read. fn try_decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> std::result::Result, String> { let path: Arc = footage.path.as_str().into(); - let src = self.inner.source(&path).ok_or("the file can't be opened")?; + let Opened { src, order, .. } = self.inner.opened(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); let native = src.info().audio().map_or(rate, |a| a.sample_rate); // The header's rate scales the samples read: 0xFFFFFFFF Hz made one conform chunk about @@ -523,14 +551,14 @@ impl MediaPool { return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz")); } if native == 0 || rate == 0 || native == rate { - return read_stereo(&src, s0, frames, rate); + return read_stereo(&src, order, s0, frames, rate); } // Output sample `n` sits at source position n × native / rate. let (native_i, rate_i) = (i128::from(native), i128::from(rate)); let first = (i128::from(s0) * native_i).div_euclid(rate_i); let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) }; - let buf = read_stereo(&src, first64, count, native)?; + let buf = read_stereo(&src, order, first64, count, native)?; let mut out = Vec::with_capacity(frames * 2); for k in 0..frames as i128 { let pos = (i128::from(s0) + k) * native_i; @@ -593,17 +621,12 @@ impl Inner { crate::read_regular_file(std::path::Path::new(path)).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}"))) } - /// The opened movie/audio source for `path`. The first caller opens it, outside the map's - /// lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF files are - /// decoded whole); callers asking for the same file meanwhile wait for that instead of opening - /// it again: a preview's frame workers, audio feeder, conform and waveform threads each - /// decoded a long FLAC in full at once (#408). - fn source(&self, path: &Arc) -> Option { - self.opened(path).map(|(src, _)| src) - } - - /// [`Self::source`] and the turn the movie's picture is shown with. - fn opened(&self, path: &Arc) -> Option<(SharedSource, Orientation)> { + /// The opened movie/audio source for `path` ([`Opened`]). The first caller opens it, outside + /// the map's lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF + /// files are decoded whole); callers asking for the same file meanwhile wait for that instead + /// of opening it again: a preview's frame workers, audio feeder, conform and waveform threads + /// each decoded a long FLAC in full at once (#408). + fn opened(&self, path: &Arc) -> Option { let slot = lock(&self.sources).entry(path.clone()).or_default().clone(); slot.get_or_init(|| { self.opened.fetch_add(1, Ordering::Relaxed); @@ -613,14 +636,15 @@ impl Inner { let file = std::path::Path::new(&**path); let in_place = if in_memory { None } else { crate::stream::open_source(file) }; let in_place = in_place.map(|src| { - let turn = crate::stream::movie_header(file).map_or(Orientation::NoTransforms, |h| crate::probe::video_turn(&h)); - src.map(|src| (src, turn)) + let header = crate::stream::movie_header(file); + src.map(|src| Opened::new(src, header.as_deref())) }); let opened = in_place.unwrap_or_else(|| { self.read(path).and_then(|bytes| { let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let turn = crate::probe::video_turn(&bytes); - filmcraft_codecs::open_bytes(&name, bytes).map(|src| (src, turn)).map_err(MediaError::from) + let (turn, asc) = (crate::probe::video_turn(&bytes), crate::probe::aac_config(&bytes)); + let src = filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)?; + Ok(Opened { order: channel_order(&src, asc.as_deref()), src, turn }) }) }); match opened { @@ -854,7 +878,7 @@ impl Inner { Ok(dynamic_to_image(frame, op)) } Some(mt) => { - let (src, turn) = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; + let Opened { src, turn, .. } = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src .video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))) .map_err(MediaError::from) @@ -953,8 +977,8 @@ impl FootageSource for MediaPool { /// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which /// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both -/// channels, multichannel sources are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. -fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { +/// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. +fn read_stereo(src: &SharedSource, order: Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { let mut out = vec![0.0; frames * 2]; let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); if skip >= frames { @@ -962,8 +986,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")); - crate::downmix::downmix_into(&buf.channels, aac, dst); + crate::downmix::downmix_into(&buf.channels, order, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 9a7ebfa1d..435f9ce57 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -259,6 +259,17 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im Ok(img) } +/// The AudioSpecificConfig of an MP4/MOV movie's first audio track (the one FilmCraft plays), if +/// that is AAC. +pub(crate) fn aac_config(bytes: &[u8]) -> Option> { + let file = filmcraft_isobmff::open(bytes).ok()?; + let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; + match &track.entries.first()?.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), + _ => None, + } +} + /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other diff --git a/crates/media/tests/it/downmix.rs b/crates/media/tests/it/downmix.rs index e2c95c251..c139676d6 100644 --- a/crates/media/tests/it/downmix.rs +++ b/crates/media/tests/it/downmix.rs @@ -164,24 +164,66 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() { for c in cases() { let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { continue }; - let s = ours(&p); - let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) - for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { - for (side, gain) in g.iter().enumerate() { - let got = amp(&s, side, freqs[i]); - let want = sine * gain; - assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); - } - } - let theirs = ffmpeg_ac2(&p); - let (a, b) = (rms(&s), rms(&theirs)); - let db = 20.0 * (a / b).log10(); - assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); - let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); - assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); + check(&c, &p); } } +/// AAC 7.1 in channel configuration 12 (back surrounds), which FilmCraft reorders where it +/// leaves ffmpeg's configuration 7 as coded: ffmpeg's 7.1 with its configuration rewritten (both +/// code the same elements). +#[test] +fn seven_one_aac_in_configuration_12() { + let c = cases().into_iter().find(|c| c.file == "dm71.m4a").expect("the 7.1 AAC case"); + let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); + let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { return }; + let mut bytes = std::fs::read(&p).expect("read"); + let (asc, _) = aac_entry(&bytes).expect("an AAC track"); + assert_eq!(asc.get(..2), Some(&[0x11, 0xB8][..]), "AAC-LC 48 kHz, configuration 7"); + let at = bytes.windows(asc.len()).position(|w| w == asc).expect("the configuration in the file"); + bytes[at + 1] = 0xE0; + let p12 = fixtures().join("dm71c12.m4a"); + std::fs::write(&p12, bytes).expect("write"); + check(&Case { file: "dm71c12.m4a", ..c }, &p12); +} + +/// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track. +fn aac_entry(bytes: &[u8]) -> Option<(Vec, u32)> { + let file = filmcraft_isobmff::open(bytes).ok()?; + file.tracks.iter().flat_map(|t| &t.entries).find_map(|e| match &e.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some((a.asc.clone(), e.audio.as_ref()?.channels)), + _ => None, + }) +} + +fn check(c: &Case, p: &Path) { + let pce_as_stereo = aac_entry(&std::fs::read(p).expect("read")).is_some_and(|(asc, entry)| { + let config = filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config); + config.ok() == Some(0) && entry == 2 + }); + if pce_as_stereo && c.names.len() > 2 { + // ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout (8.1 the real count), and + // that is all FilmCraft decodes. + eprintln!("{}: FilmCraft reads 2 of its {} channels: skipping", c.file, c.names.len()); + return; + } + let s = ours(p); + let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); + let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) + for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { + for (side, gain) in g.iter().enumerate() { + let got = amp(&s, side, freqs[i]); + let want = sine * gain; + assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); + } + } + let theirs = ffmpeg_ac2(p); + let (a, b) = (rms(&s), rms(&theirs)); + let db = 20.0 * (a / b).log10(); + assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); + let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); + assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); +} + /// Full-scale in every channel must not clip. #[test] fn loud_surround_does_not_clip() { From 393a7aa56529c7ff424116e9f02b27b9bf9bc5dd Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 17:57:08 -0400 Subject: [PATCH 15/33] Export tests: pin the H.264 bytes FilmCraft 09aaf9b writes (#1701) The encoder's first-frame user_data_unregistered SEI now lists its settings ("... - options: rc=vbr bitrate=10000 ... scenecut=1", 211 bytes where "filmcraft-h264enc 0.1.1 - clean-room H.264 encoder" took 50): +161 bytes. The same build gives (104322, 11051847956767423392), the pin Linux CI took on a base before the 19-byte nclx colr box, when that box is left out, so the bytes match across platforms. --- crates/export/tests/it/movie_bytes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index 1f48efd1e..7a3fba611 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,7 +94,7 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); - assert_eq!((data.len(), fnv1a(&data)), (104_180, 8_728_430_532_616_484_528), "H.264 MP4"); + assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); } #[test] From e675df7cabe9ccb4b6f0f1f159c7db784d8fac10 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:08 -0400 Subject: [PATCH 16/33] Media: downmix AAC by where each decoded channel plays (#1701) The gains of an AAC source now come from the configuration FilmCraft decodes it with, read from MP4/MOV (esds), Matroska (CodecPrivate) and LATM in MPEG TS/PS (its first StreamMuxConfig). Channel configurations 3 to 6, 11 and 12 arrive reordered to WAVE order, 4 as L R C Cs, 7 in element order. A program config element goes by its element positions, except the six structures in which ffmpeg's encoder fills its own channel order across them (an LFE in a side SCE, a back centre in a side pair), recognised by its "Lavc" comment. ADTS can only signal configurations 1 to 7 that FilmCraft decodes, which the channel count tells apart. A decode of fewer channels than the layout keeps its first ones (7 of configuration 7's 8 in MPEG-TS LATM). --- Cargo.lock | 2 + Cargo.toml | 5 + crates/media/Cargo.toml | 2 + crates/media/src/aac.rs | 85 ++++++++++ crates/media/src/downmix.rs | 272 ++++++++++++++++++------------- crates/media/src/lib.rs | 1 + crates/media/src/pool.rs | 36 ++-- crates/media/src/probe.rs | 11 -- crates/media/src/stream.rs | 11 ++ crates/media/tests/it/footage.rs | 2 +- 10 files changed, 283 insertions(+), 144 deletions(-) create mode 100644 crates/media/src/aac.rs diff --git a/Cargo.lock b/Cargo.lock index 79ceb885c..21adef411 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1637,7 +1637,9 @@ dependencies = [ "filmcraft-color", "filmcraft-frame", "filmcraft-isobmff", + "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpegts", "filmcraft-time", "gif", "image", diff --git a/Cargo.toml b/Cargo.toml index 57ea0d720..c101c489c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09a filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +# container headers read for AAC channel layouts (crates/media) +filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } @@ -128,6 +131,8 @@ sys-locale = "0.3" # filmcraft-prores = { path = "../filmcraft/crates/prores" } # filmcraft-aac = { path = "../filmcraft/crates/aac" } # filmcraft-isobmff = { path = "../filmcraft/crates/isobmff" } +# filmcraft-matroska = { path = "../filmcraft/crates/matroska" } +# filmcraft-mpegts = { path = "../filmcraft/crates/mpegts" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index d3f815bba..77bda60d0 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -23,6 +23,8 @@ filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +filmcraft-matroska = { workspace = true } +filmcraft-mpegts = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs new file mode 100644 index 000000000..a2d7de37b --- /dev/null +++ b/crates/media/src/aac.rs @@ -0,0 +1,85 @@ +//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry +//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG +//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`), +//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart +//! ([`crate::downmix::Order::Aac`]). + +use std::io; + +use filmcraft_aac::AudioSpecificConfig; +use filmcraft_media::ByteReader; + +/// How far into an MPEG TS / PS file its first LATM frame is looked for: the stream is indexed +/// as far as this, not through the whole file. +const MPEG_HEAD: u64 = 8 << 20; + +/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its +/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS. +pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option { + let asc = match header.and_then(mp4) { + Some(asc) => asc, + None => { + let file = file?; + let mut magic = [0u8; 4]; + file.read_at(0, &mut magic).ok()?; + if magic == [0x1A, 0x45, 0xDF, 0xA3] { matroska(file)? } else { latm(file)? } + } + }; + AudioSpecificConfig::parse(&asc).ok() +} + +fn mp4(bytes: &[u8]) -> Option> { + let file = filmcraft_isobmff::open(bytes).ok()?; + let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; + match &track.entries.first()?.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), + _ => None, + } +} + +fn matroska(file: &dyn ByteReader) -> Option> { + let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() }; + let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?; + match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec { + filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()), + _ => None, + } +} + +fn latm(file: &dyn ByteReader) -> Option> { + let head = Head(file, file.len().min(MPEG_HEAD)); + let mpeg = filmcraft_mpegts::open(&head).ok()?; + let stream = mpeg.find(filmcraft_mpegts::Kind::Audio, |c| *c == filmcraft_mpegts::Codec::AacLatm)?; + let frame = mpeg.read_unit(&head, stream, 0).ok()?; + Some(filmcraft_codecs::audio::latm_config(&frame)?.asc) +} + +/// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read. +struct Head<'a>(&'a dyn ByteReader, u64); + +impl Head<'_> { + fn read(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + match offset.checked_add(buf.len() as u64) { + Some(end) if end <= self.1 => self.0.read_at(offset, buf), + _ => Err(io::Error::new(io::ErrorKind::UnexpectedEof, "read past the end")), + } + } +} + +impl filmcraft_matroska::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + +impl filmcraft_mpegts::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 745f28065..09bb1e1f6 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -10,17 +10,19 @@ //! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and //! stereo is untouched. //! -//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM, -//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs, -//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC arrives in either -//! that order or AAC's own (C L R first, LFE last) depending on its channel configuration -//! ([`aac_order`]). More than 8 channels carry no known layout: only the first two (front L/R) -//! are used. +//! A decoded channel's gains come from where it plays. WAVE / SMPTE order by channel count is what +//! FilmCraft's AC-3, PCM, FLAC and Opus readers deliver: 3 = L R C, 4 = quad L R Ls Rs, +//! 5 = L R C Ls Rs, 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC +//! arrives in an order its configuration decides ([`aac_order`]). More than 8 channels in WAVE +//! order carry no known layout: only the first two (front L/R) are used. use std::f32::consts::FRAC_1_SQRT_2; +use std::sync::Arc; + +use filmcraft_aac::{AudioSpecificConfig, ProgramConfig}; /// A source channel's contribution to the left and right outputs, before normalisation. -type Pair = [f32; 2]; +pub type Pair = [f32; 2]; const FL: Pair = [1.0, 0.0]; const FR: Pair = [0.0, 1.0]; @@ -30,33 +32,18 @@ const SL: Pair = [FRAC_1_SQRT_2, 0.0]; const SR: Pair = [0.0, FRAC_1_SQRT_2]; const CS: Pair = [0.5, 0.5]; -/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised. -/// `aac` selects AAC's native channel order. Empty for zero channels. -pub fn stereo_gains(count: usize, aac: bool) -> Vec { - let raw: &[Pair] = match (count, aac) { - (0, _) => &[], - (1, _) => &[[1.0, 1.0]], - (2, _) => &[FL, FR], - (3, false) => &[FL, FR, FC], - (3, true) => &[FC, FL, FR], - (4, false) => &[FL, FR, SL, SR], - (4, true) => &[FC, FL, FR, CS], - (5, false) => &[FL, FR, FC, SL, SR], - (5, true) => &[FC, FL, FR, SL, SR], - (6, false) => &[FL, FR, FC, LFE, SL, SR], - (6, true) => &[FC, FL, FR, SL, SR, LFE], - (7, false) => &[FL, FR, FC, LFE, CS, SL, SR], - (7, true) => &[FC, FL, FR, SL, SR, CS, LFE], - (8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR], - (8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE], - _ => &[FL, FR], - }; +/// AAC channel configuration 7 as FilmCraft decodes it (element order, not reordered). +const CONFIG_7: [Pair; 8] = [FC, FL, FR, SL, SR, SL, SR, LFE]; +/// AAC channel configuration 4 (C L R Cs), which FilmCraft reorders to L R C Cs. +const CONFIG_4: [Pair; 4] = [FL, FR, FC, CS]; + +/// `raw` scaled by the reciprocal of its larger row sum when that exceeds 1 (more than two +/// channels): full-scale input can never exceed full scale. +fn normalised(raw: &[Pair]) -> Vec { let mut out = raw.to_vec(); - // Channels beyond a known layout (more than 8) are silent. - out.resize(count, [0.0, 0.0]); let peak = out.iter().map(|g| g[0]).sum::().max(out.iter().map(|g| g[1]).sum::()); // Mono is deliberately left at unity on both sides; so is anything already within range. - if count > 2 && peak > 1.0 { + if out.len() > 2 && peak > 1.0 { for g in &mut out { g[0] /= peak; g[1] /= peak; @@ -65,63 +52,109 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } -/// The order a source's decoded channels arrive in. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +/// The per-channel `[left, right]` gains for `count` channels in WAVE / SMPTE order, normalised. +/// Empty for zero channels. +pub fn stereo_gains(count: usize) -> Vec { + let raw: &[Pair] = match count { + 0 => &[], + 1 => &[[1.0, 1.0]], + 2 => &[FL, FR], + 3 => &[FL, FR, FC], + 4 => &[FL, FR, SL, SR], + 5 => &[FL, FR, FC, SL, SR], + 6 => &[FL, FR, FC, LFE, SL, SR], + 7 => &[FL, FR, FC, LFE, CS, SL, SR], + 8 => &[FL, FR, FC, LFE, SL, SR, SL, SR], + _ => &[FL, FR], + }; + let mut raw = raw.to_vec(); + // Channels beyond a known layout (more than 8) are silent. + raw.resize(count, [0.0, 0.0]); + normalised(&raw) +} + +/// Where a source's decoded channels play. +#[derive(Clone, Debug, Default, PartialEq)] pub enum Order { /// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest. #[default] Wav, - /// AAC that FilmCraft reorders to WAVE / SMPTE (configurations 3 to 6, 11 and 12; 4 becomes - /// L R C Cs) or that ffmpeg wrote in it (a program config element listing the front pair - /// first: ffmpeg fills a PCE's elements in its own channel order and reads them back so). - AacWav, - /// AAC in its element order (C L R … LFE), which FilmCraft keeps: configuration 7 and a PCE - /// listing the centre first. - AacElements, - /// AAC whose configuration the container doesn't show us (only MP4 / MOV headers are read): - /// as [`Order::AacWav`] up to 7 channels (configurations 3 to 6, 11 and ffmpeg's 6.1 PCE), - /// as [`Order::AacElements`] above (configuration 7, ffmpeg's 7.1). + /// AAC whose configuration wasn't read: by channel count as the configurations 1 to 7 decode + /// (exact for ADTS, which can signal no other configuration FilmCraft decodes), 7 channels as + /// configuration 11. Aac, + /// Each decoded channel's gains, before normalisation. + Speakers(Arc<[Pair]>), } -/// How FilmCraft's decoder hands over the AAC described by the AudioSpecificConfig `asc`. -pub fn aac_order(asc: &[u8]) -> Order { - let Ok(asc) = filmcraft_aac::AudioSpecificConfig::parse(asc) else { return Order::Aac }; - match (asc.channel_config, asc.pce) { - (3..=6 | 11 | 12, _) => Order::AacWav, - (0, Some(pce)) if pce.front.first().is_some_and(|&(cpe, _)| cpe) => Order::AacWav, - (0, None) => Order::Aac, - _ => Order::AacElements, +/// How FilmCraft's decoder hands over AAC with the configuration `asc`: it reorders channel +/// configurations 3 to 6, 11 and 12 to WAVE / SMPTE and leaves 7 and PCE layouts in element order. +pub fn aac_order(asc: &AudioSpecificConfig) -> Order { + match (asc.channel_config, &asc.pce) { + (4, _) => Order::Speakers(CONFIG_4.into()), + (1..=6 | 11 | 12, _) => Order::Wav, + (7, _) => Order::Speakers(CONFIG_7.into()), + (0, Some(pce)) => Order::Speakers(pce_speakers(pce).into()), + _ => Order::Aac, } } -/// The gains for `count` channels arriving in `order`. -pub fn decoded_gains(count: usize, order: Order) -> Vec { - let order = match order { - Order::Aac if (3..=7).contains(&count) => Order::AacWav, - Order::Aac => Order::AacElements, - o => o, - }; - match (order, count) { - // C L R Cs, reordered to L R C Cs: the same gains, permuted. - (Order::AacWav, 4) => { - let g = stereo_gains(4, true); - [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() +/// ffmpeg's encoder fills a program config element's elements with its layout's channels in its +/// own order (L R C LFE …), and for these structures that order doesn't match the element +/// positions: an LFE in a side SCE, a back centre in a side pair. Front, side and back element +/// types (`true` for a pair) and the channels of the ffmpeg layout with that structure (the first +/// ffmpeg 8.1 lists, where several share it: 6.1 for 6.1(back) and 6.1(front), 7.1(wide) for +/// 7.1(wide-side)). +const FFMPEG_PCE: [(&[bool], &[bool], &[bool], &[Pair]); 6] = [ + (&[true, false], &[false], &[false], &[FL, FR, FC, LFE, CS]), + (&[true, false], &[false], &[true], &[FL, FR, FC, LFE, SL, SR]), + (&[true, false], &[true], &[false], &[FL, FR, FC, CS, SL, SR]), + (&[true, false], &[false], &[true, false], &[FL, FR, FC, LFE, CS, SL, SR]), + (&[true, false], &[false], &[true, true], &[FL, FR, FC, LFE, SL, SR, FL, FR]), + (&[true, false], &[true], &[true, false], &[FL, FR, FC, SL, SR, CS, SL, SR]), +]; + +/// The channels of a PCE layout in decoded (element) order: ffmpeg's for its structures above +/// (it names itself in the PCE's comment), otherwise by position (ISO/IEC 14496-3's program config element): +/// front pairs left and right, a front single the centre, side and back pairs surrounds, a side +/// or back single a back centre. +fn pce_speakers(pce: &ProgramConfig) -> Vec { + let pairs = |els: &[(bool, u8)]| els.iter().map(|e| e.0).collect::>(); + let (front, side, back) = (pairs(&pce.front), pairs(&pce.side), pairs(&pce.back)); + if pce.comment.starts_with(b"Lavc") && pce.lfe.is_empty() { + let known = FFMPEG_PCE.iter().find(|(f, s, b, _)| *f == front.as_slice() && *s == side.as_slice() && *b == back.as_slice()); + if let Some((.., speakers)) = known { + return speakers.to_vec(); } - (Order::AacElements, _) => stereo_gains(count, true), - _ => stereo_gains(count, false), + } + let mut out = Vec::new(); + for pair in front { + out.extend_from_slice(if pair { &[FL, FR] } else { &[FC] }); + } + for pair in side.into_iter().chain(back) { + out.extend_from_slice(if pair { &[SL, SR] } else { &[CS] }); + } + out.extend(pce.lfe.iter().map(|_| LFE)); + out +} + +/// The gains for `count` channels arriving in `order`. FilmCraft decodes as many channels as the +/// container names, the first ones of the layout where that is fewer (MPEG-TS LATM names 7 of +/// configuration 7's 8, ffmpeg 4.2 MP4s 2 of a PCE's); more go by count. +pub fn decoded_gains(count: usize, order: &Order) -> Vec { + match order { + Order::Speakers(s) if count <= s.len() => normalised(s.get(..count).unwrap_or_default()), + Order::Aac if count == 4 => normalised(&CONFIG_4), + Order::Aac if count == 8 => normalised(&CONFIG_7), + _ => stereo_gains(count), } } /// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2` /// frames); frames a channel is too short for count as silence, extra frames in `out` stay as /// they are. -pub fn downmix_into(channels: &[Vec], order: Order, out: &mut [f32]) { - mix(channels, &decoded_gains(channels.len(), order), out); -} - -fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { - for (ch, g) in channels.iter().zip(gains) { +pub fn downmix_into(channels: &[Vec], order: &Order, out: &mut [f32]) { + for (ch, g) in channels.iter().zip(&decoded_gains(channels.len(), order)) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -138,13 +171,14 @@ mod tests { #[test] fn every_channel_count_has_a_bounded_defined_matrix() { - for aac in [false, true] { + let pce = Order::Speakers([FL, FR, FC, FC, SL, SR, SL, SR, CS, LFE].into()); + for order in [Order::Wav, Order::Aac, Order::Speakers(CONFIG_7.into()), pce] { for n in 0..40usize { - let g = stereo_gains(n, aac); + let g = decoded_gains(n, &order); assert_eq!(g.len(), n); let (l, r) = (g.iter().map(|p| p[0]).sum::(), g.iter().map(|p| p[1]).sum::()); if n > 2 { - assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}"); + assert!(l <= 1.0001 && r <= 1.0001, "{n} channels ({order:?}): {l} {r}"); assert!(l > 0.0 && r > 0.0); } } @@ -155,75 +189,87 @@ mod tests { const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500"; const PCE_FILMCRAFT_61: &str = "118004c845000108840000"; - fn asc(hex: &str) -> Vec { - (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect() + fn order(hex: &str) -> Order { + let asc: Vec = (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect(); + aac_order(&AudioSpecificConfig::parse(&asc).expect("an AudioSpecificConfig")) + } + + fn speakers(order: &Order) -> Vec { + match order { + Order::Speakers(s) => s.to_vec(), + o => panic!("{o:?} names no speakers"), + } } #[test] fn aac_order_follows_the_channel_configuration() { - // AAC-LC at 48 kHz with channel configuration n - let config = |n: u8| aac_order(&[0x11, 0x80 | n << 3]); - for n in [3, 4, 5, 6, 11, 12] { - assert_eq!(config(n), Order::AacWav, "configuration {n}"); + for n in [1, 2, 3, 5, 6, 11, 12] { + assert_eq!(aac_order(&AudioSpecificConfig::lc(3, n)), Order::Wav, "configuration {n}"); } - for n in [1, 2, 7] { - assert_eq!(config(n), Order::AacElements, "configuration {n}"); - } - assert_eq!(aac_order(&asc(PCE_FFMPEG_61)), Order::AacWav, "ffmpeg's 6.1 PCE lists L R first"); - assert_eq!(aac_order(&asc(PCE_FILMCRAFT_61)), Order::AacElements, "a PCE listing the centre first"); - assert_eq!(aac_order(&[]), Order::Aac); + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 4))), [FL, FR, FC, CS]); + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 7))), [FC, FL, FR, SL, SR, SL, SR, LFE]); + assert_eq!(speakers(&order(PCE_FFMPEG_61)), [FL, FR, FC, LFE, CS, SL, SR], "ffmpeg's 6.1 carries the LFE in a side SCE"); + assert_eq!(speakers(&order(PCE_FILMCRAFT_61)), [FC, FL, FR, SL, SR, CS, LFE], "a centre-first PCE by position"); } #[test] - fn decoded_aac_gains_follow_the_order() { - for n in [3, 5, 6, 7, 8] { - assert_eq!(decoded_gains(n, Order::AacWav), stereo_gains(n, false), "{n} channels"); - assert_eq!(decoded_gains(n, Order::AacElements), stereo_gains(n, true), "{n} channels"); - } - let g = decoded_gains(4, Order::AacWav); - assert_eq!(g.len(), 4); - assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); - assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); - assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - assert_eq!(decoded_gains(4, Order::Wav), stereo_gains(4, false), "PCM 4 channels are quad"); - for n in [3, 4, 5, 6, 7] { - assert_eq!(decoded_gains(n, Order::Aac), decoded_gains(n, Order::AacWav), "{n} channels"); - } - for n in [1, 2, 8, 9] { - assert_eq!(decoded_gains(n, Order::Aac), stereo_gains(n, true), "{n} channels"); + fn only_ffmpeg_pce_structures_it_misplaces_take_its_channel_order() { + let pce = |comment: &[u8], side: Vec<(bool, u8)>, lfe: Vec| ProgramConfig { + front: vec![(true, 0), (false, 0)], + side, + back: vec![(true, 1)], + lfe, + comment: comment.to_vec(), + ..Default::default() + }; + // 5.1(side): ffmpeg's LFE sits in the side SCE + assert_eq!(pce_speakers(&pce(b"Lavc58.54.100", vec![(false, 1)], vec![])), [FL, FR, FC, LFE, SL, SR]); + // the same structure from another encoder: a back centre by position + assert_eq!(pce_speakers(&pce(b"", vec![(false, 1)], vec![])), [FL, FR, FC, CS, SL, SR]); + // 7.0: ffmpeg's order matches the positions + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(true, 2)], vec![])), [FL, FR, FC, SL, SR, SL, SR]); + // a real LFE element is never ffmpeg's misplaced one + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(false, 1)], vec![0])), [FL, FR, FC, CS, SL, SR, LFE]); + } + + #[test] + fn a_decode_of_fewer_channels_takes_the_first_of_the_layout_and_more_go_by_count() { + assert_eq!(decoded_gains(2, &order(PCE_FILMCRAFT_61)), [FC, FL]); + assert_eq!(decoded_gains(7, &Order::Speakers(CONFIG_7.into())), normalised(&CONFIG_7[..7])); + assert_eq!(decoded_gains(9, &Order::Speakers(CONFIG_7.into())), stereo_gains(9)); + assert_eq!(decoded_gains(4, &Order::Aac), normalised(&[FL, FR, FC, CS])); + assert_eq!(decoded_gains(8, &Order::Aac), normalised(&CONFIG_7)); + for n in [3, 5, 6, 7] { + assert_eq!(decoded_gains(n, &Order::Aac), stereo_gains(n), "{n} channels"); } } #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); - let g = stereo_gains(6, false); + let g = stereo_gains(6); assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0); assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6); assert_eq!(g[3], [0.0, 0.0], "LFE is left out"); assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0); assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0); - // AAC order: the centre is first, LFE last - let a = stereo_gains(6, true); - assert_eq!((a[0], a[5]), (g[2], g[3])); - assert_eq!((a[1], a[2]), (g[0], g[1])); } #[test] fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() { - assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]); - assert_eq!(stereo_gains(2, true), vec![FL, FR]); - let g = stereo_gains(12, false); + assert_eq!(stereo_gains(1), vec![[1.0, 1.0]]); + assert_eq!(stereo_gains(2), vec![FL, FR]); + let g = stereo_gains(12); assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0])); } #[test] fn ragged_and_empty_input_never_panics() { let mut out = vec![0.0; 8]; - downmix_into(&[], Order::Wav, &mut out); - downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], Order::Aac, &mut out); + downmix_into(&[], &Order::Wav, &mut out); + downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], &Order::Aac, &mut out); assert!(out.iter().all(|v| v.is_finite())); let mut none: [f32; 0] = []; - downmix_into(&vec![vec![1.0; 3]; 7], Order::Wav, &mut none); + downmix_into(&vec![vec![1.0; 3]; 7], &Order::Speakers(CONFIG_7.into()), &mut none); } } diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 306f8d860..d9df8944a 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -15,6 +15,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod aac; mod convert; pub mod downmix; mod dpx; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 193804447..655ca4ed8 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -193,21 +193,20 @@ struct Opened { } impl Opened { - /// `src` with what its MP4/MOV `header` (ftyp and moov) says, if it has one. - fn new(src: SharedSource, header: Option<&[u8]>) -> Opened { + /// `src` with what its container says: `header` is an MP4/MOV's ftyp and moov (or the whole + /// file), `file` the file for other containers' AAC configuration. + fn new(src: SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Opened { let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn); - let order = channel_order(&src, header.and_then(crate::probe::aac_config).as_deref()); - Opened { src, turn, order } + Opened { order: channel_order(&src, header, file), src, turn } } } -/// The order `src`'s audio channels decode in, AAC's decided by its AudioSpecificConfig `asc` -/// when the container's header was read. -fn channel_order(src: &SharedSource, asc: Option<&[u8]>) -> Order { - match (src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")), asc) { - (false, _) => Order::Wav, - (true, Some(asc)) => crate::downmix::aac_order(asc), - (true, None) => Order::Aac, +/// The order `src`'s audio channels decode in; multichannel AAC's by its configuration, where +/// the container's header shows it ([`crate::aac::config`]). +fn channel_order(src: &SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Order { + match src.info().audio() { + Some(a) if a.codec.contains("AAC") && a.channels > 2 => crate::aac::config(header, file).map_or(Order::Aac, |asc| crate::downmix::aac_order(&asc)), + _ => Order::Wav, } } @@ -551,14 +550,14 @@ impl MediaPool { return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz")); } if native == 0 || rate == 0 || native == rate { - return read_stereo(&src, order, s0, frames, rate); + return read_stereo(&src, &order, s0, frames, rate); } // Output sample `n` sits at source position n × native / rate. let (native_i, rate_i) = (i128::from(native), i128::from(rate)); let first = (i128::from(s0) * native_i).div_euclid(rate_i); let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) }; - let buf = read_stereo(&src, order, first64, count, native)?; + let buf = read_stereo(&src, &order, first64, count, native)?; let mut out = Vec::with_capacity(frames * 2); for k in 0..frames as i128 { let pos = (i128::from(s0) + k) * native_i; @@ -636,15 +635,14 @@ impl Inner { let file = std::path::Path::new(&**path); let in_place = if in_memory { None } else { crate::stream::open_source(file) }; let in_place = in_place.map(|src| { - let header = crate::stream::movie_header(file); - src.map(|src| Opened::new(src, header.as_deref())) + let (header, reader) = (crate::stream::movie_header(file), crate::stream::file_reader(file)); + src.map(|src| Opened::new(src, header.as_deref(), reader.as_deref())) }); let opened = in_place.unwrap_or_else(|| { self.read(path).and_then(|bytes| { let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let (turn, asc) = (crate::probe::video_turn(&bytes), crate::probe::aac_config(&bytes)); - let src = filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)?; - Ok(Opened { order: channel_order(&src, asc.as_deref()), src, turn }) + let src = filmcraft_codecs::open_bytes(&name, bytes.clone()).map_err(MediaError::from)?; + Ok(Opened::new(src, Some(&bytes), Some(&filmcraft_media::reader::MemReader(bytes.clone())))) }) }); match opened { @@ -978,7 +976,7 @@ impl FootageSource for MediaPool { /// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which /// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both /// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. -fn read_stereo(src: &SharedSource, order: Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { +fn read_stereo(src: &SharedSource, order: &Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { let mut out = vec![0.0; frames * 2]; let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); if skip >= frames { diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 435f9ce57..9a7ebfa1d 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -259,17 +259,6 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im Ok(img) } -/// The AudioSpecificConfig of an MP4/MOV movie's first audio track (the one FilmCraft plays), if -/// that is AAC. -pub(crate) fn aac_config(bytes: &[u8]) -> Option> { - let file = filmcraft_isobmff::open(bytes).ok()?; - let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; - match &track.entries.first()?.codec { - filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), - _ => None, - } -} - /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other diff --git a/crates/media/src/stream.rs b/crates/media/src/stream.rs index 447e239d1..80db21fc4 100644 --- a/crates/media/src/stream.rs +++ b/crates/media/src/stream.rs @@ -85,6 +85,17 @@ pub(crate) fn open_source(_path: &Path) -> Option> { None } +/// `path` read in place (`None` when it can't be opened, or without a native file system). +#[cfg(any(unix, windows))] +pub(crate) fn file_reader(path: &Path) -> Option> { + Some(Box::new(FileReader::open(path).ok()?)) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn file_reader(_path: &Path) -> Option> { + None +} + /// The leading `ftyp` and the `moov` box of an ISO-BMFF movie on disk (all a track's display /// matrix needs), without reading the samples: top-level box headers are walked with positional /// reads. `None` when the file isn't ISO-BMFF or its `moov` is missing or implausibly large. diff --git a/crates/media/tests/it/footage.rs b/crates/media/tests/it/footage.rs index 4d539c4e5..c57cec7eb 100644 --- a/crates/media/tests/it/footage.rs +++ b/crates/media/tests/it/footage.rs @@ -915,7 +915,7 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() { // ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries // channels 0 and 1 at full level and the others at half, so a multichannel file sounds as // its stereo downmix (tests/it/downmix.rs checks that matrix channel by channel). - let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false); + let gains = effectcraft_media::downmix::stereo_gains(channels as usize); let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::(); let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt(); let rms = (sum / n.max(1) as f64).sqrt(); From ec86f24afeaf54da8a4ea661c00c5d7ee89b9f30 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 17/33] Media tests: every AAC layout mixes by speaker in MP4, Matroska and ADTS (#1701) Tones encoded with FilmCraft's AAC encoder, with the configuration rewritten to 11, 12 or ffmpeg's 6.1 PCE where the coded elements fit it, so nothing depends on ffmpeg or skips. Expected gains come from each channel's speaker. --- crates/media/tests/it/aac_layout.rs | 224 ++++++++++++++++++++++++++++ crates/media/tests/it/main.rs | 1 + 2 files changed, 225 insertions(+) create mode 100644 crates/media/tests/it/aac_layout.rs diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs new file mode 100644 index 000000000..9e5ddbb23 --- /dev/null +++ b/crates/media/tests/it/aac_layout.rs @@ -0,0 +1,224 @@ +//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel +//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no +//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded +//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and +//! 8-channel output, and a program config element can name any of them. + +use std::f64::consts::PI; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe, probe_bytes}; +use effectcraft_time::Tick; +use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig}; + +const RATE: u32 = 48_000; +/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry. +const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700]; +const LFE_TONE: u32 = 60; +const AMP: f32 = 0.25; +const K: f64 = std::f64::consts::FRAC_1_SQRT_2; + +/// A channel's content, by where it plays. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Ch { + L, + R, + C, + Lfe, + Ls, + Rs, + Cs, +} + +impl Ch { + /// ITU-R BS.775 gains into `[left, right]`, before normalisation. + fn gains(self) -> [f64; 2] { + match self { + Ch::L => [1.0, 0.0], + Ch::R => [0.0, 1.0], + Ch::C => [K, K], + Ch::Lfe => [0.0, 0.0], + Ch::Ls => [K, 0.0], + Ch::Rs => [0.0, K], + Ch::Cs => [0.5, 0.5], + } + } +} + +struct Case { + name: &'static str, + /// What the encoder's input channels carry (its element order: C, L R, …, LFE last). + content: &'static [Ch], + /// The AudioSpecificConfig written to the container instead of the encoder's, if any. + asc: Option, +} + +fn lc(config: u8) -> Option { + Some(AudioSpecificConfig::lc(3, config)) +} + +fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec, comment: &[u8]) -> Option { + let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() }; + Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }) +} + +use Ch::*; + +fn cases() -> Vec { + vec![ + Case { name: "config3", content: &[C, L, R], asc: None }, + Case { name: "config4", content: &[C, L, R, Cs], asc: None }, + Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None }, + Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None }, + // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds + Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None }, + Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) }, + Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) }, + // the encoder's own 6.1: a PCE listing the centre first + Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None }, + // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE + // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the + // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag + // 1) and the LFE in the side SCE left over (tag 5). + Case { + name: "pce61-ffmpeg", + content: &[C, L, R, Cs, Ls, Rs, Lfe], + asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"), + }, + ] +} + +fn freq(content: &[Ch], i: usize) -> u32 { + if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] } +} + +/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units. +fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { + let n = content.len(); + let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32); + cfg.adts = adts; + let mut enc = Encoder::new(cfg).expect("encoder"); + let planes: Vec> = + (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect(); + let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect(); + let mut units = enc.encode(&refs); + units.extend(enc.flush()); + (enc.audio_specific_config(), units) +} + +fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); + for u in units { + w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); + } + w.finish().expect("finish").into_inner() +} + +fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer"); + let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); + for (i, u) in units.iter().enumerate() { + w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + } + w.finish().expect("finish").into_inner() +} + +/// The fixture written to disk as `name`. +fn write(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join(name); + std::fs::write(&p, bytes).expect("write"); + p +} + +/// Amplitude of the tone at `freq` in one side of interleaved stereo. +fn amp(s: &[f32], side: usize, freq: u32) -> f64 { + let (mut re, mut im, mut n) = (0.0, 0.0, 0usize); + for (i, f) in s.as_chunks::<2>().0.iter().enumerate() { + let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE); + re += f64::from(f[side]) * w.cos(); + im += f64::from(f[side]) * w.sin(); + n += 1; + } + 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64 +} + +/// Stereo of one second from 0.5 s in, through the media pool as footage reads it. +fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec { + pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE) +} + +/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger +/// side's sum (an LFE nowhere). +fn check(label: &str, content: &[Ch], s: &[f32]) { + let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect(); + let norm = raw.iter().map(|g| g[0]).sum::().max(raw.iter().map(|g| g[1]).sum::()).max(1.0); + for (i, (ch, g)) in content.iter().zip(&raw).enumerate() { + for side in 0..2 { + let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm); + assert!( + (got - want).abs() <= f64::from(AMP) * 0.05, + "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}", + freq(content, i), + ["L", "R"][side] + ); + } + } +} + +/// The LFE tone survives encoding, so its absence from the mix means it was left out. +fn lfe_is_coded(case: &Case, bytes: Vec) { + if !case.content.contains(&Lfe) { + return; + } + let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open"); + let buf = src.audio(24_000, RATE as usize, RATE).expect("decode"); + let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::>(), 0, LFE_TONE)).fold(0.0, f64::max); + assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name); +} + +#[test] +fn every_aac_layout_mixes_by_speaker_in_mp4() { + for case in cases() { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let bytes = mp4(asc, case.content.len() as u32, &units); + lfe_is_coded(&case, bytes.clone()); + let p = write(&format!("{}.mp4", case.name), &bytes); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + // in memory (`add_bytes`) as well as in place + let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into()); + let pool = MediaPool::new(); + pool.add_bytes(&path, bytes.clone()); + check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe"))); + } +} + +#[test] +fn matroska_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units)); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// ADTS signals configurations 1 to 7 only, which the channel count tells apart. +#[test] +fn adts_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) { + let (_, units) = encode(case.content, true); + let p = write(&format!("{}.aac", case.name), &units.concat()); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 5f7fb8037..69bed4a9f 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -1,5 +1,6 @@ //! The crate's integration tests, one binary (each module is a former `tests/.rs`). +mod aac_layout; mod downmix; mod exif_orientation; mod footage; From 799303ad68bcaac91bc197ecc7335d371efe23d5 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 18/33] Media tests: read ffmpeg 4.2's PCE fixtures with their real channel count (#1701) ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout and FilmCraft decodes as many channels as that names, so the 6.1 check now reads a copy naming all seven instead of skipping. Configuration 12 moves to the synthetic layout tests. --- crates/media/tests/it/downmix.rs | 44 +++++++++++++------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/crates/media/tests/it/downmix.rs b/crates/media/tests/it/downmix.rs index c139676d6..05205ab6a 100644 --- a/crates/media/tests/it/downmix.rs +++ b/crates/media/tests/it/downmix.rs @@ -168,22 +168,23 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() { } } -/// AAC 7.1 in channel configuration 12 (back surrounds), which FilmCraft reorders where it -/// leaves ffmpeg's configuration 7 as coded: ffmpeg's 7.1 with its configuration rewritten (both -/// code the same elements). -#[test] -fn seven_one_aac_in_configuration_12() { - let c = cases().into_iter().find(|c| c.file == "dm71.m4a").expect("the 7.1 AAC case"); - let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); - let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { return }; - let mut bytes = std::fs::read(&p).expect("read"); - let (asc, _) = aac_entry(&bytes).expect("an AAC track"); - assert_eq!(asc.get(..2), Some(&[0x11, 0xB8][..]), "AAC-LC 48 kHz, configuration 7"); - let at = bytes.windows(asc.len()).position(|w| w == asc).expect("the configuration in the file"); - bytes[at + 1] = 0xE0; - let p12 = fixtures().join("dm71c12.m4a"); - std::fs::write(&p12, bytes).expect("write"); - check(&Case { file: "dm71c12.m4a", ..c }, &p12); +/// `p`, or a copy whose AAC sample entry names all `channels` of its PCE layout: ffmpeg 4.2 writes +/// 2 there for a PCE layout (8.1 the real count), and FilmCraft decodes as many channels as the +/// sample entry names, so the other five of a 6.1 would never reach the mix. +fn whole_pce_entry(p: &Path, channels: usize) -> PathBuf { + let mut bytes = std::fs::read(p).expect("read"); + let Some((asc, 2)) = aac_entry(&bytes) else { return p.to_path_buf() }; + if channels <= 2 || filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config).ok() != Some(0) { + return p.to_path_buf(); + } + // the `mp4a` entry's channelcount follows its 6 reserved bytes, data reference index and + // 8 reserved bytes + let at = bytes.windows(4).position(|w| w == b"mp4a").expect("an mp4a sample entry") + 4 + 6 + 2 + 8; + bytes[at..at + 2].copy_from_slice(&(channels as u16).to_be_bytes()); + eprintln!("{}: its sample entry names 2 of {channels} channels (ffmpeg 4.2): reading a copy naming all", p.display()); + let out = p.with_extension("entry.m4a"); + std::fs::write(&out, bytes).expect("write"); + out } /// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track. @@ -196,16 +197,7 @@ fn aac_entry(bytes: &[u8]) -> Option<(Vec, u32)> { } fn check(c: &Case, p: &Path) { - let pce_as_stereo = aac_entry(&std::fs::read(p).expect("read")).is_some_and(|(asc, entry)| { - let config = filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config); - config.ok() == Some(0) && entry == 2 - }); - if pce_as_stereo && c.names.len() > 2 { - // ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout (8.1 the real count), and - // that is all FilmCraft decodes. - eprintln!("{}: FilmCraft reads 2 of its {} channels: skipping", c.file, c.names.len()); - return; - } + let p = &whole_pce_entry(p, c.names.len()); let s = ours(p); let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) From 4daff9a74adc5c875273abcbc11bf4b53438b7cc Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 19/33] Export tests: say why the H.264 pin moved with FilmCraft 09aaf9b (#1701) --- crates/export/tests/it/movie_bytes.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index 7a3fba611..f72d6fbf7 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,6 +94,8 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); + // FilmCraft 09aaf9b's encoder lists its settings in the first frame's user-data SEI (211 + // bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin. assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); } From 0ca3f72ac2175521c1a109dabb3f8a2242528709 Mon Sep 17 00:00:00 2001 From: becauseno7 <95076423+becauseno7@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:34:30 +0600 Subject: [PATCH 20/33] CC Bubbles: bubbles refract the layer's edge pixels on padded buffers too (#2024) Refraction samples are clamped to the layer rectangle instead of the buffer, so a sample past the layer's edge in a buffer with transparent padding (a half-resolution preview's, a text or shape layer's) repeats the layer's edge pixels rather than reaching the pale colour that stands in for transparent ones. Transparent pixels inside the layer still show it. Same bounds on the GPU path. Re-pins the four half-resolution CC Bubbles golden hashes (aarch64 macOS); the full-resolution ones are unchanged. Adds a CPU test and a GPU-vs-CPU parity test on padded buffers. --- crates/effects/src/lib.rs | 2 +- crates/effects/src/sim.rs | 2 +- crates/effects/src/sim/tests.rs | 39 +++++++++++++++++++++++++++ crates/effects/src/sim/weather.rs | 14 +++++++++- crates/effects/tests/it/sim_golden.rs | 8 +++--- crates/gpu/src/fx_sim.rs | 1 + crates/gpu/src/shaders/fx_sim.wgsl | 7 +++-- crates/gpu/src/tests/fx_sim.rs | 28 +++++++++++++++++++ 8 files changed, 92 insertions(+), 9 deletions(-) diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 32e662fec..a5534bf86 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -117,7 +117,7 @@ pub use ocio::color_stabilizer_maps; // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes). pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, selection_hashes}; pub use ocio::{ColorOp, Straight, Tf, color_program}; -pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, drizzle_drops, mercury_blobs, sprite_plan}; +pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan}; pub use sim2::{ CausticsSetup, FoamDisc, FoamEnv, FoamPlan, Piece, PiecePlan, PieceTex, PixelPlan, WavePlan, Waves, caustics_setup, foam_full_plan, piece_plan, pixel_plan, wave_world_plan, diff --git a/crates/effects/src/sim.rs b/crates/effects/src/sim.rs index 79a225f75..333549e7d 100644 --- a/crates/effects/src/sim.rs +++ b/crates/effects/src/sim.rs @@ -38,7 +38,7 @@ use particle_fx::{PW_RATE, mercury_phys, ps2_phys, pw_phys}; pub use registry::{specs, sprite_plan}; pub use sprites::{Acc, Post, Shape, Sprite, SpritePlan, Tint}; pub(crate) use sprites::{combine, raster, splat}; -pub use weather::{Bubble, bubble_list, drizzle_drops}; +pub use weather::{Bubble, bubble_list, bubble_sample_bounds, drizzle_drops}; #[cfg(test)] mod tests; diff --git a/crates/effects/src/sim/tests.rs b/crates/effects/src/sim/tests.rs index d5de4cb74..06ce973b2 100644 --- a/crates/effects/src/sim/tests.rs +++ b/crates/effects/src/sim/tests.rs @@ -112,6 +112,45 @@ fn bubbles_near_edges_show_the_layer() { assert_eq!(pale, [0, 0], "pale bubble pixels (liquid, metal)"); } +/// CC Bubbles on a 40 × 40 layer `img` at `scale`, in a buffer padded by 6 transparent pixels +/// (a half-resolution preview's layout). +fn padded_bubbles(img: Image, scale: f64, reflection: u32) -> Image { + let size = 40.0; + let s = crate::find("ec.sim.ccbubbles").unwrap(); + let mut params = crate::Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [size; 2]))).collect() }; + for (k, v) in [("bubbleAmount", num(200.0)), ("bubbleSize", num(3.0)), ("reflectionType", Value::Enum(reflection))] { + params.values.insert(k.into(), v); + } + let ctx = EffectCtx { params: ¶ms, time: 0.3, layer_size: [size; 2], seed: 1, adjustment: false, env: EffectEnv::default() }; + let mut b = Buf { img, offset: [0.0; 2], scale }; + b.pad(6); + crate::apply(s, &ctx, b).img +} + +/// #2024: the same when the buffer has transparent padding around the layer: samples past the +/// layer's edges repeat its edge pixels, not the padding, and a clear hole inside the layer is +/// still refracted as the pale colour. +#[test] +fn bubbles_near_padded_edges_show_the_layer() { + let blue = |side| Image::filled(side, side, [0.0, 0.0, 1.0, 1.0]); + let pale = |img: &Image| img.data.iter().filter(|p| (p[2] - p[0] - p[3]).abs() > 1e-4).count(); + for (scale, side) in [(1.0, 40), (0.5, 20)] { + let counts = [0, 1].map(|reflection| { + let a = padded_bubbles(blue(side), scale, reflection); + assert!(a.data.iter().any(|p| p[3] > 0.5), "bubbles drawn"); + pale(&a) + }); + assert_eq!(counts, [0, 0], "pale bubble pixels at scale {scale} (liquid, metal)"); + } + let mut holed = blue(40); + for y in 12..28 { + for x in 12..28 { + holed.set(x, y, [0.0; 4]); + } + } + assert!(pale(&padded_bubbles(holed, 1.0, 0)) > 0, "the hole refracts as the pale colour"); +} + #[test] fn drizzle_ripples() { let a = run("ec.sim.ccdrizzle", &[], ramp(48, 48), 0.6); diff --git a/crates/effects/src/sim/weather.rs b/crates/effects/src/sim/weather.rs index b58b39b5e..347ab4939 100644 --- a/crates/effects/src/sim/weather.rs +++ b/crates/effects/src/sim/weather.rs @@ -1,6 +1,7 @@ use super::common::{h, hs}; use super::sprites::{Acc, Post, Shape, Sprite, SpritePlan, Tint, aa, raster}; use super::{Buf, EffectCtx, Image, IntoParallelIterator, ParallelIterator, unpremul}; +use crate::util::layer_rect_at; // ------------------------------------------------------------------ CC Rainfall / CC Snowfall @@ -160,11 +161,21 @@ pub fn bubble_list(ctx: &EffectCtx, b: &Buf) -> Vec { .collect() } +/// CC Bubbles' refraction sample bounds in the pixels of a buffer with `offset` and `scale`: the +/// layer rectangle's first and last pixel centres (x0, y0, x1, y1). Samples clamped to them repeat +/// the layer's edge pixels instead of reaching transparent padding around it (#2024). +pub fn bubble_sample_bounds(ctx: &EffectCtx, offset: [f64; 2], scale: f64) -> [f32; 4] { + let (lx, ly, lw, lh) = layer_rect_at(ctx, offset, scale); + let (x0, y0) = (lx + 0.5, ly + 0.5); + [x0 as f32, y0 as f32, (lx + lw - 0.5).max(x0) as f32, (ly + lh - 0.5).max(y0) as f32] +} + pub(super) fn bubbles(ctx: &EffectCtx, mut b: Buf) -> Buf { let pr = ctx.params; let metal = pr.e("reflectionType") == 1; let shading = pr.e("shadingType"); let list = bubble_list(ctx, &b); + let [x0, y0, x1, y1] = bubble_sample_bounds(ctx, b.offset, b.scale); let src = b.img.clone(); let mut out = Image::new(src.width, src.height); raster( @@ -182,7 +193,8 @@ pub(super) fn bubbles(ctx: &EffectCtx, mut b: Buf) -> Buf { // Liquid refracts (inverted, magnified), metal reflects (wide-angle mirror). let k = if metal { 1.6 } else { -0.7 }; let nz = (1.0 - d2).max(0.0).sqrt(); - let p = src.sample_bilinear_clamped((q.sx + dx * q.r * k) as f64, (q.sy + dy * q.r * k) as f64); + let (sx, sy) = ((q.sx + dx * q.r * k).max(x0).min(x1), (q.sy + dy * q.r * k).max(y0).min(y1)); + let p = src.sample_bilinear_clamped(sx as f64, sy as f64); let (c, sa) = unpremul(p); let mut c = if sa <= 0.0 { [0.6, 0.7, 0.8] } else { c }; let mut a = cov; diff --git a/crates/effects/tests/it/sim_golden.rs b/crates/effects/tests/it/sim_golden.rs index 3932418ad..388790901 100644 --- a/crates/effects/tests/it/sim_golden.rs +++ b/crates/effects/tests/it/sim_golden.rs @@ -441,10 +441,10 @@ const GOLDEN: &[(&str, u64)] = &[ ("ec.sim.ccbubbles/0/32bpc/t1.5", 0xd09565e124d8f5db), ("ec.sim.ccbubbles/0/32bpc/t0.8", 0x91242f8e1fd0ad84), ("ec.sim.ccbubbles/0/32bpc/t2.2", 0x1f9782131e62149d), - ("ec.sim.ccbubbles/0/8bpc/half/t0.5", 0x931b4ca707604220), - ("ec.sim.ccbubbles/0/8bpc/half/t1.5", 0x21c25d4555abebe9), - ("ec.sim.ccbubbles/0/8bpc/half/t0.8", 0x7e05972683359632), - ("ec.sim.ccbubbles/0/8bpc/half/t2.2", 0xadc4f7219dab4921), + ("ec.sim.ccbubbles/0/8bpc/half/t0.5", 0x020755e48bdf8f43), + ("ec.sim.ccbubbles/0/8bpc/half/t1.5", 0x4aac0f1332e6001e), + ("ec.sim.ccbubbles/0/8bpc/half/t0.8", 0xdc2ab5e04c4a95a2), + ("ec.sim.ccbubbles/0/8bpc/half/t2.2", 0x30f78adc5d1dd715), ("ec.sim.ccbubbles/1/8bpc/t0.5", 0x2f5415c679ff0980), ("ec.sim.ccbubbles/1/8bpc/t1.5", 0xfcfc354dcd22e6d0), ("ec.sim.ccbubbles/1/8bpc/t0.8", 0xdf90fd57c217019c), diff --git a/crates/gpu/src/fx_sim.rs b/crates/gpu/src/fx_sim.rs index a2d2c24dd..944ca6c07 100644 --- a/crates/gpu/src/fx_sim.rs +++ b/crates/gpu/src/fx_sim.rs @@ -253,6 +253,7 @@ fn bubbles(e: &mut Enc, ctx: &EffectCtx, b: GBuf, geo: &effectcraft_effects::Buf } let mut p = Params::default(); p.u[1] = [0, (ctx.params.e("reflectionType") == 1) as u32, ctx.params.e("shadingType"), 0]; + p.f[0] = effectcraft_effects::bubble_sample_bounds(ctx, b.offset, b.scale); let img = raster(e, items, 1, false, &b.img, None, p, w, h)?; Some(GBuf { img, ..b }) } diff --git a/crates/gpu/src/shaders/fx_sim.wgsl b/crates/gpu/src/shaders/fx_sim.wgsl index 8ef3035fe..0c6023f32 100644 --- a/crates/gpu/src/shaders/fx_sim.wgsl +++ b/crates/gpu/src/shaders/fx_sim.wgsl @@ -165,7 +165,8 @@ fn fxm_sprite(at: u32, x: f32, y: f32) -> FxmShade { return FxmShade(true, vec4(rgb * a, a)); } -// CC Bubbles' bubble (sim::bubbles). u[1].y = metal, u[1].z = shading type. +// CC Bubbles' bubble (sim::bubbles). u[1].y = metal, u[1].z = shading type, f[0] = the layer +// rectangle's first and last pixel centres (x0, y0, x1, y1), which refraction samples stay within. fn fxm_bubble(at: u32, x: f32, y: f32) -> FxmShade { let qx = data[at + 1u]; let qy = data[at + 2u]; @@ -180,7 +181,9 @@ fn fxm_bubble(at: u32, x: f32, y: f32) -> FxmShade { } let k = select(-0.7, 1.6, P.u[1].y != 0u); let nz = sqrt(max(1.0 - d2, 0.0)); - let s = fxm_unpremul(sample_bilinear_clamped(src, data[at + 4u] + dx * qr * k, data[at + 5u] + dy * qr * k)); + let sx = min(max(data[at + 4u] + dx * qr * k, P.f[0].x), P.f[0].z); + let sy = min(max(data[at + 5u] + dy * qr * k, P.f[0].y), P.f[0].w); + let s = fxm_unpremul(sample_bilinear_clamped(src, sx, sy)); var c = s.xyz; if (s.w <= 0.0) { c = vec3(0.6, 0.7, 0.8); diff --git a/crates/gpu/src/tests/fx_sim.rs b/crates/gpu/src/tests/fx_sim.rs index dc2510bb3..4f54e07b6 100644 --- a/crates/gpu/src/tests/fx_sim.rs +++ b/crates/gpu/src/tests/fx_sim.rs @@ -105,6 +105,34 @@ fn bubbles_drizzle_mercury() { case("ec.sim.ccmrmercury", &[("birthRate", n(3.0)), ("blobInfluence", n(60.0)), ("shading/metal", n(80.0)), ("gravity", n(2.0))], 1.5, 0.0); } +/// CC Bubbles on a buffer with transparent padding around the layer: both sides clamp the +/// refraction samples to the layer rectangle, not to the buffer (#2024). +#[test] +fn bubbles_on_padded_buffers() { + let Some(g) = gpu() else { return }; + let spec = effectcraft_effects::find("ec.sim.ccbubbles").unwrap(); + let size = [70.0, 44.0]; + for reflection in [0, 1] { + let mut params = + effectcraft_effects::Params { values: spec.params.iter().map(|ps| (ps.id.to_string(), effectcraft_effects::default_value(ps, size))).collect() }; + for (k, v) in [("bubbleAmount", n(80.0)), ("bubbleSize", n(4.0)), ("reflectionType", e(reflection))] { + params.values.insert(k.to_string(), v); + } + for scale in [1.0, 0.5] { + let img = pattern(7, (size[0] * scale) as u32, (size[1] * scale) as u32); + let mut buf = effectcraft_effects::Buf { img, offset: [0.0, 0.0], scale }; + buf.pad(6); + let ctx = || effectcraft_effects::EffectCtx { params: ¶ms, time: 0.5, layer_size: size, seed: 11, adjustment: false, env: Default::default() }; + let cpu = (spec.render)(&ctx(), buf.clone()); + let out = + effectcraft_render::Accelerator::effects(g, &[effectcraft_render::FxStep { spec, ctx: ctx() }], &buf, None).expect("the GPU runs the effect"); + assert_eq!((out.offset, out.scale), (cpu.offset, cpu.scale), "geometry"); + let d = diff(&cpu.img, &out.img, 1e-3); + assert!(d.over == 0, "reflection {reflection} scale {scale}: {} of {} pixels over 1e-3 (max {}); worst {:?}", d.over, d.total, d.max, d.worst); + } + } +} + #[test] fn hair_wave_world_foam() { case("ec.sim.cchair", &[("density", n(300.0)), ("length", n(10.0))], 0.0, 0.0); From 800b555e514e541e25aecc84585d06d2aeb62da0 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:44:05 -0400 Subject: [PATCH 21/33] Media: read the AAC configuration of the MP4 or Matroska track FilmCraft plays, not an empty one before it (#1701) --- crates/media/src/aac.rs | 40 ++++++++++++++++++------ crates/media/tests/it/aac_layout.rs | 48 ++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs index a2d7de37b..cd33eaf7b 100644 --- a/crates/media/src/aac.rs +++ b/crates/media/src/aac.rs @@ -1,4 +1,4 @@ -//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry +//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry //! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG //! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`), //! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart @@ -13,10 +13,11 @@ use filmcraft_media::ByteReader; /// as far as this, not through the whole file. const MPEG_HEAD: u64 = 8 << 20; -/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its -/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS. +/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header` +/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or +/// MPEG TS / PS. pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option { - let asc = match header.and_then(mp4) { + let asc = match header.and_then(|h| mp4(h, file)) { Some(asc) => asc, None => { let file = file?; @@ -28,19 +29,29 @@ pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Op AudioSpecificConfig::parse(&asc).ok() } -fn mp4(bytes: &[u8]) -> Option> { - let file = filmcraft_isobmff::open(bytes).ok()?; - let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; +/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample +/// description and a timescale, read from the whole file as it is (a fragmented movie's samples +/// are in its fragments, not the `moov`). +fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option> { + let mp4 = match file { + Some(f) => filmcraft_isobmff::open(Head(f, f.len())), + None => filmcraft_isobmff::open(header), + }; + let mp4 = mp4.ok()?; + let playable = + |t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0; + let track = mp4.tracks.iter().find(playable)?; match &track.entries.first()?.codec { filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), _ => None, } } +/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by +/// indexing the clusters as that source does. fn matroska(file: &dyn ByteReader) -> Option> { - let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() }; - let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?; - match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec { + let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?; + match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec { filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()), _ => None, } @@ -66,6 +77,15 @@ impl Head<'_> { } } +impl filmcraft_isobmff::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + impl filmcraft_matroska::ByteSource for Head<'_> { fn len(&self) -> u64 { self.1 diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs index 9e5ddbb23..c6403667b 100644 --- a/crates/media/tests/it/aac_layout.rs +++ b/crates/media/tests/it/aac_layout.rs @@ -109,8 +109,17 @@ fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { } fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mp4_after(None, asc, channels, units) +} + +/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no +/// samples, if any. +fn mp4_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + if let Some((asc, channels)) = empty { + w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track"); + } let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); for u in units { w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); @@ -119,14 +128,25 @@ fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { } fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mkv_after(None, asc, channels, units) +} + +/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no +/// frames, if any. +fn mkv_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; - let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); - t.codec_private = asc; - t.audio = Some((f64::from(RATE), u64::from(channels), None)); - let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer"); + let track = |asc: Vec, channels: u32| { + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + t + }; + let tracks: Vec = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect(); + let played = tracks.len() - 1; + let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer"); let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); for (i, u) in units.iter().enumerate() { - w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); } w.finish().expect("finish").into_inner() } @@ -222,3 +242,21 @@ fn adts_aac_layouts_mix_by_speaker() { check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); } } + +/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before +/// it lends it no configuration. Each case decodes in another order than the empty track's +/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE). +#[test] +fn an_empty_audio_track_lends_its_configuration_to_no_other() { + for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] { + let case = cases().into_iter().find(|c| c.name == name).expect("case"); + let n = case.content.len() as u32; + let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n)); + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units)); + check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units)); + check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} From aecdffecb09ef33c43d3e5350cf5537bfa2067aa Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:44:34 -0400 Subject: [PATCH 22/33] Media: read the LATM configuration only when FilmCraft plays that MPEG stream, not an ADTS one before it (#1701) --- crates/media/src/aac.rs | 21 +++- crates/media/tests/it/aac_layout.rs | 160 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 5 deletions(-) diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs index cd33eaf7b..625c37b43 100644 --- a/crates/media/src/aac.rs +++ b/crates/media/src/aac.rs @@ -9,8 +9,8 @@ use std::io; use filmcraft_aac::AudioSpecificConfig; use filmcraft_media::ByteReader; -/// How far into an MPEG TS / PS file its first LATM frame is looked for: the stream is indexed -/// as far as this, not through the whole file. +/// How far into an MPEG TS / PS file the audio streams are looked for: the stream is indexed as +/// far as this, not through the whole file. const MPEG_HEAD: u64 = 8 << 20; /// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header` @@ -57,12 +57,23 @@ fn matroska(file: &dyn ByteReader) -> Option> { } } +/// The configuration of the stream FilmCraft's MPEG source plays: the first audio stream whose +/// first unit it can read. Only AAC is asked about, so the streams before that one were refused +/// whatever their codec, and an ADTS stream played has no configuration outside its frames. fn latm(file: &dyn ByteReader) -> Option> { + use filmcraft_mpegts::Codec; let head = Head(file, file.len().min(MPEG_HEAD)); let mpeg = filmcraft_mpegts::open(&head).ok()?; - let stream = mpeg.find(filmcraft_mpegts::Kind::Audio, |c| *c == filmcraft_mpegts::Codec::AacLatm)?; - let frame = mpeg.read_unit(&head, stream, 0).ok()?; - Some(filmcraft_codecs::audio::latm_config(&frame)?.asc) + let audio = mpeg.streams.iter().enumerate().filter(|(_, s)| s.kind() == filmcraft_mpegts::Kind::Audio && !s.units.is_empty()); + let played = audio.take(filmcraft_media::MAX_AUDIO_STREAMS).find_map(|(i, s)| { + let first = mpeg.read_unit(&head, i, 0).ok()?; + match s.codec { + Codec::AacAdts => filmcraft_mpegts::frame_info(&s.codec, &first).map(|_| None), + Codec::AacLatm => filmcraft_codecs::audio::latm_config(&first).map(|c| Some(c.asc)), + _ => None, + } + }); + played? } /// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read. diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs index c6403667b..fb3e078a6 100644 --- a/crates/media/tests/it/aac_layout.rs +++ b/crates/media/tests/it/aac_layout.rs @@ -260,3 +260,163 @@ fn an_empty_audio_track_lends_its_configuration_to_no_other() { check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); } } + +/// A transport stream plays its first audio stream and mixes by that stream's configuration, +/// whatever follows it. ADTS of configuration 7 decodes in element order, LATM of configuration +/// 12 reordered to WAVE / SMPTE; LATM of configuration 7 names 7 channels, which mix as +/// configuration 7's rather than as 7 channels by count. (FilmCraft decodes LATM of configuration +/// 12 as 12 channels, so it isn't played here.) +#[test] +fn mpeg_aac_mixes_by_the_stream_played() { + let case = cases().into_iter().find(|c| c.name == "config7").expect("case"); + let (_, adts) = encode(case.content, true); + let (_, raw) = encode(case.content, false); + for (label, latm_first, latm_config) in [("ADTS before LATM", false, 12), ("LATM before ADTS", true, 7)] { + let (adts_pid, latm_pid) = if latm_first { (0x102, 0x101) } else { (0x101, 0x102) }; + let mut ts = Ts::default(); + let mut streams = [(0x0F, adts_pid), (0x11, latm_pid)]; + streams.sort_by_key(|s| s.1); + ts.psi(&streams); + for (i, (a, r)) in adts.iter().zip(&raw).enumerate() { + let pts = 1920 * i as i64; + let mut units = [(adts_pid, pes(pts, a)), (latm_pid, pes(pts, &loas(latm_config, r)))]; + units.sort_by_key(|u| u.0); + for (pid, unit) in units { + ts.pes(pid, &unit); + } + } + let p = write(&format!("latm-first-{latm_first}.ts"), &ts.out); + check(label, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// A LOAS frame of `payload` with its StreamMuxConfig: AAC LC, 48 kHz, channel configuration +/// `config`. +fn loas(config: u32, payload: &[u8]) -> Vec { + let mut bits = Bits::default(); + // useSameStreamMux, audioMuxVersion, allStreamsSameTimeFraming, numSubFrames, numProgram, numLayer + bits.put(0b001, 3); + bits.put(0, 13); + // AudioSpecificConfig and GASpecificConfig + bits.put(2, 5); + bits.put(3, 4); + bits.put(config, 4); + bits.put(0, 3); + // frameLengthType, latmBufferFullness, otherDataPresent, crcCheckPresent + bits.put(0, 3); + bits.put(0xFF, 8); + bits.put(0, 2); + let mut len = payload.len(); + while len >= 255 { + bits.put(255, 8); + len -= 255; + } + bits.put(len as u32, 8); + for &b in payload { + bits.put(u32::from(b), 8); + } + let body = bits.finish(); + let mut v = vec![0x56, 0xE0 | (body.len() >> 8) as u8, body.len() as u8]; + v.extend(body); + v +} + +/// A big-endian bit writer. +#[derive(Default)] +struct Bits { + out: Vec, + acc: u64, + n: u32, +} + +impl Bits { + fn put(&mut self, v: u32, n: u32) { + self.acc = self.acc << n | u64::from(v); + self.n += n; + while self.n >= 8 { + self.n -= 8; + self.out.push((self.acc >> self.n) as u8); + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.put(0, 8 - self.n); + } + self.out + } +} + +/// An audio PES packet (stream id 0xC0) presented at `pts` (90 kHz). +fn pes(pts: i64, payload: &[u8]) -> Vec { + let t = [0x21 | ((pts >> 29) & 0x0E) as u8, (pts >> 22) as u8, ((pts >> 14) & 0xFE) as u8 | 1, (pts >> 7) as u8, ((pts << 1) & 0xFE) as u8 | 1]; + let len = 8 + payload.len(); + let mut v = vec![0, 0, 1, 0xC0, (len >> 8) as u8, len as u8, 0x80, 0x80, 5]; + v.extend(t); + v.extend_from_slice(payload); + v +} + +/// A transport stream of one program (its PMT on PID 0x100), PES cut into 188-byte packets. +#[derive(Default)] +struct Ts { + out: Vec, + cc: std::collections::HashMap, +} + +impl Ts { + fn packet(&mut self, pid: u16, start: bool, payload: &[u8]) -> usize { + let cc = self.cc.entry(pid).or_insert(0); + let mut p = vec![0x47, u8::from(start) << 6 | (pid >> 8) as u8, pid as u8]; + let n = payload.len().min(184); + if n < 184 { + let stuffing = 183 - n; + p.extend([0x30 | *cc, stuffing as u8]); + if stuffing > 0 { + p.push(0); + p.extend(std::iter::repeat_n(0xFF, stuffing - 1)); + } + } else { + p.push(0x10 | *cc); + } + *cc = (*cc + 1) & 15; + p.extend_from_slice(&payload[..n]); + self.out.extend(p); + n + } + fn section(&mut self, pid: u16, body: &[u8]) { + let mut s = vec![0]; + s.extend_from_slice(body); + s.extend(crc32_mpeg(body).to_be_bytes()); + self.packet(pid, true, &s); + } + /// The PAT and a PMT of `streams` (stream type, PID). + fn psi(&mut self, streams: &[(u8, u16)]) { + self.section(0, &[0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00]); + let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x01, 0xF0, 0]; + for &(kind, pid) in streams { + pmt.extend([kind, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]); + } + pmt[2] = (pmt.len() + 1) as u8; + self.section(0x100, &pmt); + } + fn pes(&mut self, pid: u16, data: &[u8]) { + let (mut rest, mut start) = (data, true); + while !rest.is_empty() { + let n = self.packet(pid, start, rest); + rest = &rest[n..]; + start = false; + } + } +} + +/// CRC-32 of PSI sections (MSB first, polynomial 0x04C11DB7, initial value all ones). +fn crc32_mpeg(data: &[u8]) -> u32 { + let mut crc = u32::MAX; + for &b in data { + crc ^= u32::from(b) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { crc << 1 ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} From dcbb806139fc3ab2425bb77c52e4142741af52be Mon Sep 17 00:00:00 2001 From: Krueels <90205419+Krueels@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:44:05 -0300 Subject: [PATCH 23/33] #986: add Blender scene export for 3D Camera Tracker --- contributors/people.toml | 3 + crates/engine/src/cmd.rs | 1 + crates/engine/src/cmd/blender.rs | 295 ++++++++++++++++++ crates/engine/src/cmd/blender/validate.rs | 71 +++++ crates/engine/src/cmd/blender_scene.py | 206 ++++++++++++ crates/engine/src/cmd/camera_cmds.rs | 14 + crates/engine/src/tests.rs | 1 + crates/engine/src/tests/blender.rs | 261 ++++++++++++++++ crates/engine/tests/blender_scene_oracle.py | 61 ++++ crates/ui-egui/src/i18n/ui.rs | 1 + crates/ui-egui/src/menus/frontend.rs | 1 + .../ui-egui/src/panels/camera_tracker_ui.rs | 7 +- crates/ui-egui/tests/it/ui_camera_tracker.rs | 61 +++- docs/agents.md | 7 +- docs/blender-camera-export.md | 101 ++++++ 15 files changed, 1086 insertions(+), 5 deletions(-) create mode 100644 crates/engine/src/cmd/blender.rs create mode 100644 crates/engine/src/cmd/blender/validate.rs create mode 100644 crates/engine/src/cmd/blender_scene.py create mode 100644 crates/engine/src/tests/blender.rs create mode 100644 crates/engine/tests/blender_scene_oracle.py create mode 100644 docs/blender-camera-export.md diff --git a/contributors/people.toml b/contributors/people.toml index 8654c7a10..df9d7fd80 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -17,6 +17,9 @@ # To remove your names, delete your entry (or set fields empty) in a PR. The next # contributors.py run, and the next build, drop them. +[people.Krueels] +sync_github_name = true + [people.echelon] real_name = "Brandon Thomas" display_name = "Brandon Thomas" diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index cba6df6ef..bd089b52b 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -10,6 +10,7 @@ mod animation; pub(crate) mod app_more; mod autotrace; mod batch; +mod blender; mod camera_cmds; mod comp; pub(crate) mod comp_more; diff --git a/crates/engine/src/cmd/blender.rs b/crates/engine/src/cmd/blender.rs new file mode 100644 index 000000000..e6c678101 --- /dev/null +++ b/crates/engine/src/cmd/blender.rs @@ -0,0 +1,295 @@ +//! Camera-solve interchange through Blender's public Python API. The generated script is +//! original MIT OR Apache-2.0 code; no Blender implementation is read or embedded. + +use std::path::Path; + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Uid}; +use effectcraft_render::EvalCtx; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::{CameraSolve, linalg}; +use serde_json::{Value, json}; + +mod validate; + +use super::{b_p, bad, f_p, str_p}; +use crate::{EngineError, Result, Session}; + +const CMD: &str = "camera.exportBlender"; +const MAX_FRAMES: usize = 100_000; +const MAX_POINTS: usize = 100_000; +const MAX_JSON: usize = 64 * 1024 * 1024; +const MAX_FRAME: i64 = 1_048_574; + +fn number(p: &Value, key: &str, default: f64, min: f64, max: f64) -> Result { + let v = match p.get(key) { + None => default, + Some(_) => f_p(p, key).ok_or_else(|| bad(CMD, format!("{key}: expected a number")))?, + }; + if !v.is_finite() || !(min..=max).contains(&v) { + return Err(bad(CMD, format!("{key}: expected {min}..{max}"))); + } + Ok(v) +} + +fn finite(values: impl IntoIterator) -> Result<()> { + if values.into_iter().any(|v| !v.is_finite() || v.abs() > 1e12) { + return Err(bad(CMD, "invalid or excessive numeric camera/scene data; re-analyze the tracker")); + } + Ok(()) +} + +/// Right-handed EffectCraft world (right, down, forward) → Blender (right, forward, up). +fn position(p: [f64; 3], scale: f64) -> [f64; 3] { + [p[0] * scale, p[2] * scale, -p[1] * scale] +} + +fn media_path(s: &Session, path: &str) -> Result { + if path.is_empty() || path.contains('\0') { + return Err(bad(CMD, "the source footage has no valid file path; relink it before exporting")); + } + let path = Path::new(path); + if path.is_absolute() || cfg!(target_arch = "wasm32") { + return Ok(path.to_string_lossy().into_owned()); + } + let cwd = std::env::current_dir().map_err(|e| bad(CMD, format!("cannot resolve footage path: {e}")))?; + let base = s.path.as_deref().and_then(|p| Path::new(p).parent()).map(|p| cwd.join(p)).unwrap_or(cwd); + Ok(base.join(path).to_string_lossy().into_owned()) +} + +pub(super) fn export(s: &mut Session, p: &Value, cid: ItemId, lid: LayerId, uid: Uid) -> Result { + if s.running_for_script() && !s.prefs.scripting.allow_scripts_write_files { + return Err(bad(CMD, "scripts can't export files unless Settings ▸ Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network is on")); + } + let options = Options::parse(p)?; + let Options { path, sensor, units, start, include_points } = options; + if s.camera_target().is_some_and(|(c, l, u)| (c, l, u) == (cid, lid, uid)) && s.is_camera_analyzing() || s.camera_pending.contains(&(cid, lid, uid)) { + return Err(bad(CMD, "wait for this tracker's analysis to finish before exporting")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(CMD, "tracked layer missing"))?; + let g = layer.props.find_group(uid).ok_or_else(|| bad(CMD, "tracker missing"))?; + for name in [ct::SOLVE, ct::TRACKS] { + if g.prop(name).is_some_and(|p| matches!(&p.value, effectcraft_keyframe::Value::Str(v) if v.len() > MAX_JSON)) { + return Err(bad(CMD, "tracking data exceeds the 64 MiB export limit")); + } + } + let params = crate::camera_track::static_params(g); + let solve = ct::solve(¶ms).ok_or_else(|| bad(CMD, "the camera is not solved yet; analyze the tracker first"))?; + let (lo, end) = validate::timing(comp, layer, &solve, start)?; + let Source { footage, path: source_path, fps: source_fps, count: source_count } = source(s, layer, &solve, path)?; + validate::geometry(&solve, sensor)?; + let camera = params + .get(ct::CAMERA) + .map(|v| v.as_f64()) + .filter(|v| v.is_finite() && *v > 0.0 && *v <= u64::MAX as f64) + .and_then(|id| comp.layer(LayerId(id as u64))) + .filter(|l| l.is_camera()); + let warnings = scene_warnings(camera, &solve, footage); + let ctx = EvalCtx { expr: s.expr.as_deref(), ..EvalCtx::new(&s.project, cid, comp, lo) }; + let (frames, world) = bake_frames(&ctx, layer, &solve, camera, &options, footage, source_count)?; + let mut points = Vec::new(); + for point in &solve.points { + finite(point.pos.into_iter().chain([point.error as f64]))?; + if point.first > point.last || point.last as usize >= solve.frames.len() { + return Err(bad(CMD, "invalid solved point frame range; re-analyze the tracker")); + } + let location = position(world.apply(point.pos), units); + finite(location)?; + if include_points { + points.push(json!({"id": point.id, "location": location, "error": point.error, + "first": start.saturating_add(point.first as i64), "last": start.saturating_add(point.last as i64)})); + } + } + let tracks = ct::tracks(¶ms); + if tracks.as_ref().is_some_and(|t| t.tracks.len() > MAX_POINTS || t.tracks.iter().any(|t| t.pts.len() > MAX_FRAMES)) { + return Err(bad(CMD, "2D tracks exceed export limits")); + } + let markers: Vec = comp + .markers + .iter() + .filter_map(|m| { + let frame = start.saturating_add(comp.frame_rate.frame_at(m.time).saturating_sub(comp.frame_rate.frame_at(lo))); + if (start..=end).contains(&frame) { Some(json!({"name": m.comment, "frame": frame})) } else { None } + }) + .collect(); + let payload = json!({"version": 1, "name": format!("{} — {}", s.project.item(cid).map(|i| i.name.as_str()).unwrap_or("Shot"), layer.name), + "resolution": solve.size, "fpsNumerator": comp.frame_rate.num, "fpsDenominator": comp.frame_rate.den, + "startFrame": start, "endFrame": end, "sensorWidth": sensor, "unitScale": units, + "media": {"path": source_path, "kind": if footage.kind == FootageKind::Still {"FILE"} else {"MOVIE"}, + "fps": source_fps, "nativeRate": footage.native_rate, "pixelAspect": footage.pixel_aspect, "timecode": footage.start_timecode, + "frameCount": source_count, "loopCount": footage.loop_count}, + "frames": frames, "points": points, "markers": markers, "ground": solve.ground.is_some(), + "groundSize": world.s * units, "shutter": comp.shutter_angle / 360.0, "shutterPhase": comp.shutter_phase, + "solve": solve, "tracks": if include_points { tracks.as_deref() } else { None }, + "warnings": warnings}); + let data = serde_json::to_string(&payload).map_err(|e| bad(CMD, format!("cannot serialize scene: {e}")))?; + if data.len() > MAX_JSON { + return Err(bad(CMD, "scene exceeds the 64 MiB script limit; disable includePoints or split the shot")); + } + let quoted = serde_json::to_string(&data).map_err(|e| bad(CMD, format!("cannot encode scene: {e}")))?; + let script = include_str!("blender_scene.py").replace("__EFFECTCRAFT_DATA__", "ed); + s.services.write_file(path, script.as_bytes()).map_err(|e| bad(CMD, format!("cannot write {path}: {e}")))?; + Ok(json!({"path": path, "bytes": script.len(), "frames": frames.len(), "points": points.len(), "warnings": warnings})) +} + +struct Options<'a> { + path: &'a str, + sensor: f64, + units: f64, + start: i64, + include_points: bool, +} + +impl<'a> Options<'a> { + fn parse(p: &'a Value) -> Result { + let path = str_p(p, "path").filter(|p| !p.is_empty() && !p.contains('\0')).ok_or_else(|| bad(CMD, "missing `path` (.py)"))?; + if !Path::new(path).extension().is_some_and(|e| e.eq_ignore_ascii_case("py")) { + return Err(bad(CMD, "the export path must end in .py")); + } + let sensor = number(p, "sensorWidth", 36.0, 1.0, 1000.0)?; + let units = number(p, "unitScale", 0.01, 0.000_001, 1000.0)?; + let start = match p.get("startFrame") { + None => 1, + Some(v) => v.as_i64().filter(|n| (1..=MAX_FRAME).contains(n)).ok_or_else(|| bad(CMD, "startFrame: expected a positive Blender frame number"))?, + }; + let include_points = match p.get("includePoints") { + Some(v) if !v.is_boolean() => return Err(bad(CMD, "includePoints: expected a boolean")), + _ => b_p(p, "includePoints").unwrap_or(true), + }; + Ok(Self { path, sensor, units, start, include_points }) + } +} + +struct Source<'a> { + footage: &'a Footage, + path: String, + fps: f64, + count: i64, +} + +fn source<'a>(s: &'a Session, layer: &Layer, solve: &CameraSolve, path: &str) -> Result> { + let footage = match layer.source { + LayerSource::Footage { item } => s.project.item(item).and_then(|i| match &i.kind { + ItemKind::Footage(f) => Some(f), + _ => None, + }), + _ => None, + } + .filter(|f| f.has_video && matches!(f.kind, FootageKind::Video | FootageKind::Still)); + let footage = footage + .ok_or_else(|| bad(CMD, "Blender background export needs a video or still-footage layer; render precomps/sequences to a video and track that video"))?; + let source_path = media_path(s, &footage.path)?; + if Path::new(path) == Path::new(&source_path) { + return Err(bad(CMD, "the export path must differ from the source footage")); + } + if footage.width as f64 != solve.size[0] || footage.height as f64 != solve.size[1] { + return Err(bad(CMD, "the analysis image differs from the original footage dimensions; render the tracker's input to a video and track that video")); + } + let source_fps = footage.frame_rate.as_f64(); + if footage.frame_rate.num <= 0 || footage.frame_rate.den <= 0 || !source_fps.is_finite() || source_fps <= 0.0 { + return Err(bad(CMD, "invalid source footage frame rate")); + } + // Same frame selection as the media pool: interpreted FPS, rounded per-loop frame count, + // clamp to the total loop range, then wrap. Native FPS does not change the frame index. + let divisor = (effectcraft_time::TICKS_PER_SECOND as i128).saturating_mul(footage.frame_rate.den as i128); + let source_count = (footage.duration.0 as i128).saturating_mul(footage.frame_rate.num as i128).saturating_add(divisor / 2) / divisor; + let source_count = if footage.kind == FootageKind::Still { 1 } else { source_count }; + if !(1..=MAX_FRAME as i128).contains(&source_count) { + return Err(bad(CMD, "invalid footage duration or footage exceeds Blender's frame range; trim/render the source video first")); + } + let source_count = source_count as i64; + Ok(Source { footage, path: source_path, fps: source_fps, count: source_count }) +} + +fn scene_warnings(camera: Option<&Layer>, solve: &CameraSolve, footage: &Footage) -> Vec { + let mut warnings = vec![ + "Sensor width is assumed (36 mm by default); monocular tracking cannot recover physical sensor size or absolute scene scale.".to_string(), + "The original footage is referenced, not embedded or rendered: masks and effects before the tracker are not baked. Relink the video if moving this script.".to_string(), + ]; + if camera.is_none() { + warnings + .push("No linked tracker camera: focus uses the default scene depth, DOF is off; optical focus cannot be recovered from feature tracks.".into()); + } else { + warnings.push("Focus/iris settings are sampled from the linked camera; pose and lens come from the tracker solve. Blender's physical DOF is an approximation of EffectCraft's blur model.".into()); + } + if solve.distortion.is_some_and(|d| !d.is_none()) { + warnings.push("Radial distortion is preserved as metadata (Brown k1/k2, half-diagonal normalization), not applied to Blender's pinhole camera. Use an undistorted plate for exact alignment.".into()); + } + if (footage.pixel_aspect - 1.0).abs() > 1e-6 { + warnings.push("The tracker solves square pixels: export uses square pixels; original footage pixel aspect is retained in metadata.".into()); + } + warnings +} + +/// Bake the solved camera and the linked camera's optical settings at exact composition ticks. +fn bake_frames( + ctx0: &EvalCtx<'_>, + layer: &Layer, + solve: &CameraSolve, + camera: Option<&Layer>, + options: &Options<'_>, + footage: &Footage, + source_count: i64, +) -> Result<(Vec, linalg::Similarity)> { + let comp = ctx0.comp; + let lo = ctx0.time; + let fd = comp.frame_duration(); + let sensor = options.sensor; + let units = options.units; + let start = options.start; + let first = solve.frames.first().ok_or_else(|| bad(CMD, "the camera solve is empty"))?; + let m = ctx0.world_matrix(layer).0; + let layer_scale = m[0][0].hypot(m[1][0]); + finite([layer_scale])?; + if layer_scale <= 1e-9 { + return Err(bad(CMD, "the tracked layer has zero scale")); + } + let world = solve.world([comp.width as f64, comp.height as f64], first.focal * layer_scale); + finite([world.s * units])?; + let mut frames = Vec::with_capacity(solve.frames.len()); + for (i, frame) in solve.frames.iter().enumerate() { + let time = Tick(lo.0.saturating_add(fd.0.saturating_mul(i as i64))); + let ctx = ctx0.at(time); + let pose = linalg::mm(&world.r, &linalg::mt(&frame.r())); + let right = position(linalg::mv(&pose, [1.0, 0.0, 0.0]), 1.0); + let up = position(linalg::mv(&pose, [0.0, -1.0, 0.0]), 1.0); + let back = position(linalg::mv(&pose, [0.0, 0.0, -1.0]), 1.0); + let location = position(world.apply(frame.center), units); + let lens = frame.focal * sensor / solve.size[0]; + let opts = camera.and_then(|l| l.props.sub("cameraOptions").map(|g| (l, g))); + let value = |key: &str, fallback: f64| opts.map(|(l, g)| ctx.f(l, g, key, fallback)).unwrap_or(fallback); + let focus = value("focusDistance", world.s) * units; + let aperture = value("aperture", 0.0); + let zoom = value("zoom", frame.focal * layer_scale); + // Match Camera Settings' f-stop conversion: focal_for_zoom uses a 36 mm film width, + // and the aperture UI is mm scaled by comp_width / 1920 (not a world-space diameter). + let aperture_mm = aperture * 1920.0 / comp.width as f64; + let fstop = if aperture_mm > 0.0 { (zoom * 36.0 / comp.width as f64) / aperture_mm } else { 2.8 }; + let rotation = value("irisRotation", 0.0).to_radians(); + let ratio = value("irisAspectRatio", 1.0); + let shape = opts.map(|(l, g)| ctx.e(l, g, "irisShape")).unwrap_or(0); + let blades = match shape { + 0 => 4, + 1..=8 => shape.saturating_add(2), + _ => return Err(bad(CMD, "invalid iris shape")), + }; + let dof = opts.is_some_and(|(l, g)| ctx.b(l, g, "dof")) && aperture > 0.0 && value("blurLevel", 100.0) > 0.0; + let source_time = ctx.source_time(layer); + let source_frame = if footage.kind == FootageKind::Still { + 0 + } else { + footage.frame_rate.frame_at(source_time).clamp(0, source_count.saturating_mul(footage.loop_count.max(1) as i64).saturating_sub(1)) % source_count + }; + finite(location.into_iter().chain([focus, aperture, zoom, fstop, rotation, ratio, value("blurLevel", 100.0), value("irisRoundness", 0.0)]))?; + if focus <= 0.0 || !(0.01..=100.0).contains(&ratio) || !(0.01..=1e6).contains(&fstop) { + return Err(bad(CMD, "focus, f-stop or iris aspect is outside Blender's supported range")); + } + frames.push(json!({"frame": start + i as i64, "timeTicks": time.0, "sourceFrame": source_frame, + "location": location, "rotation": [right, up, back], "lens": lens, "focus": focus, "dof": dof, + "fstop": fstop, "blades": blades, "irisRotation": rotation, "irisRatio": ratio, + "sourceAperture": aperture, "sourceZoom": zoom, + "blurLevel": value("blurLevel", 100.0), "irisRoundness": value("irisRoundness", 0.0)})); + } + Ok((frames, world)) +} diff --git a/crates/engine/src/cmd/blender/validate.rs b/crates/engine/src/cmd/blender/validate.rs new file mode 100644 index 000000000..a9454861f --- /dev/null +++ b/crates/engine/src/cmd/blender/validate.rs @@ -0,0 +1,71 @@ +//! Validate solved geometry and its correspondence to the current layer timing. + +use effectcraft_project::{Comp, Layer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::{CameraSolve, linalg}; + +use super::{CMD, MAX_FRAME, MAX_FRAMES, MAX_POINTS, bad, finite}; +use crate::Result; + +pub(super) fn timing(comp: &Comp, layer: &Layer, solve: &CameraSolve, start: i64) -> Result<(Tick, i64)> { + if solve.frames.len() > MAX_FRAMES || solve.points.len() > MAX_POINTS { + return Err(bad(CMD, "export is limited to 100,000 frames and 100,000 points; split the shot")); + } + if solve.size.iter().any(|v| !v.is_finite() || *v < 4.0 || *v > 65_536.0 || v.fract() != 0.0) { + return Err(bad(CMD, "invalid tracking resolution (expected 4..65536 whole pixels)")); + } + let fps = comp.frame_rate.as_f64(); + if comp.frame_rate.num <= 0 || comp.frame_rate.den <= 0 || !fps.is_finite() || !(1.0..=1000.0).contains(&fps) { + return Err(bad(CMD, "invalid composition frame rate (expected 1..1000 fps)")); + } + let end = start + .checked_add(solve.frames.len() as i64) + .and_then(|n| n.checked_sub(1)) + .filter(|n| *n <= MAX_FRAME) + .ok_or_else(|| bad(CMD, "the baked animation exceeds Blender's frame range"))?; + finite([solve.start, solve.frame_duration, solve.average_error, layer.stretch, comp.shutter_angle, comp.shutter_phase])?; + if layer.stretch == 0.0 || comp.width == 0 || comp.height == 0 { + return Err(bad(CMD, "invalid composition dimensions or zero layer stretch")); + } + let lo = comp.frame_rate.snap_nearest(layer.in_point.max(Tick::ZERO)); + let hi = layer.out_point.min(comp.duration); + let fd = comp.frame_duration(); + if fd.0 <= 0 || lo >= hi { + return Err(bad(CMD, "the tracked layer has no active frames")); + } + let expected = ((hi.0 as i128 - lo.0 as i128 + fd.0 as i128 - 1) / fd.0 as i128) as u128; + if expected != solve.frames.len() as u128 { + return Err(bad(CMD, "the solve no longer matches the layer's frame range; re-analyze the tracker")); + } + let dt = (layer.layer_time(lo + fd) - layer.layer_time(lo)).seconds(); + if (solve.start - layer.layer_time(lo).seconds()).abs() > 1e-6 || solve.frames.len() > 1 && (solve.frame_duration - dt).abs() > 1e-6 { + return Err(bad(CMD, "the solve no longer matches the layer's timing; re-analyze the tracker")); + } + Ok((lo, end)) +} + +pub(super) fn geometry(solve: &CameraSolve, sensor: f64) -> Result<()> { + for frame in &solve.frames { + finite(frame.rot.into_iter().chain(frame.center).chain([frame.focal]))?; + if !frame.solved || frame.focal <= 0.0 { + return Err(bad(CMD, "the solve contains unsolved frames or invalid focal lengths; re-analyze the tracker")); + } + let mm = sensor * frame.focal / solve.size[0]; + if !(1.0..=5000.0).contains(&mm) { + return Err(bad(CMD, "lens outside Blender's 1..5000 mm range; choose another sensorWidth")); + } + } + if let Some(ground) = &solve.ground { + finite(ground.origin.into_iter().chain(ground.normal).chain(ground.x_axis))?; + if (linalg::norm(ground.normal) - 1.0).abs() > 1e-4 + || (linalg::norm(ground.x_axis) - 1.0).abs() > 1e-4 + || linalg::dot(ground.normal, ground.x_axis).abs() > 1e-4 + { + return Err(bad(CMD, "invalid ground-plane basis; set the ground plane again")); + } + } + if let Some(d) = &solve.distortion { + finite([d.k1, d.k2, d.radius])?; + } + Ok(()) +} diff --git a/crates/engine/src/cmd/blender_scene.py b/crates/engine/src/cmd/blender_scene.py new file mode 100644 index 000000000..7240cb1ca --- /dev/null +++ b/crates/engine/src/cmd/blender_scene.py @@ -0,0 +1,206 @@ +# EffectCraft camera tracker → Blender, original code under MIT OR Apache-2.0. +# Blender scene export contributor: Krueels — https://github.com/Krueels +# Run in Blender's Scripting workspace (Open, Run Script), or: +# blender --background --python "Tracked Scene.py" +# Creates a NEW scene; existing scenes and objects are preserved. Save the .blend yourself. +# Public API references: https://docs.blender.org/api/4.5/bpy.types.Camera.html +# https://docs.blender.org/api/4.5/bpy.types.ImageUser.html +import json +import math +import os + +import bpy +from mathutils import Matrix + +DATA = json.loads(__EFFECTCRAFT_DATA__) + + +def _curves(owner): + """Blender 4.0–4.3 legacy actions and 4.4+/5.x layered actions.""" + anim = owner.animation_data + if anim is None or anim.action is None: + return + action = anim.action + if getattr(action, "is_action_layered", False): + for layer in action.layers: + for strip in layer.strips: + if strip.type == 'KEYFRAME': + bag = strip.channelbag(anim.action_slot) + if bag is not None: + yield from bag.fcurves + else: + yield from action.fcurves + + +def _interpolation(owner): + for curve in _curves(owner): + # Footage indices and discrete switches must never interpolate between frames. + stepped = ("frame_offset" in curve.data_path or "use_dof" in curve.data_path + or "aperture_blades" in curve.data_path) + for key in curve.keyframe_points: + key.interpolation = 'CONSTANT' if stepped else 'LINEAR' + + +def _media_path(): + original = DATA["media"]["path"] + candidates = [original] + # When sharing, place the video alongside the script or the .blend and run again. + script_path = globals().get("__file__", "") + if script_path: + candidates.append(os.path.join(os.path.dirname(os.path.abspath(script_path)), + os.path.basename(original))) + if bpy.data.filepath: + candidates.append(os.path.join(os.path.dirname(bpy.data.filepath), + os.path.basename(original))) + return next((p for p in candidates if os.path.isfile(p)), None) + + +def import_scene(): + if bpy.app.version < (4, 0, 0): + raise RuntimeError("EffectCraft scene import requires Blender 4.0 or newer") + scene = bpy.data.scenes.new(DATA["name"]) + scene.frame_start = DATA["startFrame"] + scene.frame_end = DATA["endFrame"] + fps = DATA["fpsNumerator"] / DATA["fpsDenominator"] + scene.render.fps = max(1, round(fps)) + scene.render.fps_base = scene.render.fps / fps + scene.render.resolution_x, scene.render.resolution_y = map(int, DATA["resolution"]) + scene.render.resolution_percentage = 100 + # The tracker assumes square pixels, regardless of the source's interpretation metadata. + scene.render.pixel_aspect_x = scene.render.pixel_aspect_y = 1.0 + scene.render.film_transparent = True + scene.render.motion_blur_shutter = max(0.0, min(2.0, DATA["shutter"])) + scene.unit_settings.system = 'METRIC' + scene.unit_settings.scale_length = 1.0 + scene["effectcraft_unit_scale"] = DATA["unitScale"] + scene["effectcraft_scale_is_arbitrary"] = True + scene["effectcraft_shutter_phase_degrees"] = DATA["shutterPhase"] + + collection = bpy.data.collections.new("EffectCraft Tracking") + scene.collection.children.link(collection) + camera_data = bpy.data.cameras.new("3D Tracker Camera") + camera = bpy.data.objects.new("3D Tracker Camera", camera_data) + collection.objects.link(camera) + scene.camera = camera + camera.rotation_mode = 'QUATERNION' + camera_data.type = 'PERSP' + camera_data.sensor_fit = 'HORIZONTAL' + camera_data.sensor_width = DATA["sensorWidth"] + camera_data.sensor_height = (DATA["sensorWidth"] * DATA["resolution"][1] + / DATA["resolution"][0]) + camera_data.show_background_images = True + camera_data.show_limits = True + extent = max([DATA["groundSize"], 1.0] + + [math.dist(p["location"], f["location"]) + for p in DATA["points"][::max(1, len(DATA["points"]) // 100)] + for f in DATA["frames"][::max(1, len(DATA["frames"]) // 10)]]) + camera_data.clip_start = max(0.00001, min(0.1, extent / 10000.0)) + camera_data.clip_end = max(1000.0, extent * 10.0) + + previous = None + for sample in DATA["frames"]: + frame = sample["frame"] + camera.location = sample["location"] + # Exported right/up/back vectors are COLUMNS of Blender's local → world basis. + quaternion = Matrix(sample["rotation"]).transposed().to_quaternion() + if previous is not None and previous.dot(quaternion) < 0: + quaternion.negate() + camera.rotation_quaternion = quaternion + previous = quaternion.copy() + camera.keyframe_insert(data_path="location", frame=frame) + camera.keyframe_insert(data_path="rotation_quaternion", frame=frame) + camera_data.lens = sample["lens"] + camera_data.keyframe_insert(data_path="lens", frame=frame) + camera_data.dof.use_dof = sample["dof"] + camera_data.dof.focus_distance = sample["focus"] + camera_data.dof.aperture_fstop = sample["fstop"] + camera_data.dof.aperture_blades = sample["blades"] + camera_data.dof.aperture_rotation = (sample["irisRotation"] + math.pi) % math.tau - math.pi + camera_data.dof.aperture_ratio = sample["irisRatio"] + for prop in ("use_dof", "focus_distance", "aperture_fstop", "aperture_blades", + "aperture_rotation", "aperture_ratio"): + camera_data.keyframe_insert(data_path="dof." + prop, frame=frame) + + path = _media_path() + if path is not None: + try: + image = bpy.data.images.load(path, check_existing=False) + image.source = DATA["media"]["kind"] + background = camera_data.background_images.new() + background.source = 'IMAGE' + background.image = image + background.alpha = 1.0 + background.display_depth = 'BACK' + background.frame_method = 'STRETCH' + user = background.image_user + user.use_auto_refresh = True + user.use_cyclic = False + user.frame_start = DATA["startFrame"] + user.frame_duration = max(1, min(1048574, image.frame_duration)) + if image.source == 'MOVIE': + for sample in DATA["frames"]: + # Blender uses 1-based image frames; sourceFrame is 0-based. + user.frame_offset = sample["sourceFrame"] - (sample["frame"] - user.frame_start) + camera_data.keyframe_insert( + data_path="background_images[0].image_user.frame_offset", + frame=sample["frame"]) + except (RuntimeError, ValueError, OSError) as error: + DATA["warnings"].append("Blender could not load the background video: " + str(error)) + else: + DATA["warnings"].append("Video not found: " + DATA["media"]["path"] + + ". Place it beside this script or relink the camera background.") + + if DATA["points"]: + mesh = bpy.data.meshes.new("Solved Track Points") + mesh.from_pydata([p["location"] for p in DATA["points"]], [], []) + mesh.update() + cloud = bpy.data.objects.new("Solved Track Points", mesh) + collection.objects.link(cloud) + cloud.show_in_front = True + cloud.hide_render = True + for name, kind, key in (("track_id", 'INT', "id"), ("reprojection_error", 'FLOAT', "error"), + ("first_frame", 'INT', "first"), ("last_frame", 'INT', "last")): + attr = mesh.attributes.new(name=name, type=kind, domain='POINT') + for dest, point in zip(attr.data, DATA["points"]): + # Track IDs are u32 in EffectCraft; keep the full IDs in the JSON text too. + dest.value = point[key] if kind == 'FLOAT' else min(2147483647, point[key]) + + if DATA["ground"]: + size = DATA["groundSize"] + mesh = bpy.data.meshes.new("Solved Ground Plane") + mesh.from_pydata([(-size, -size, 0), (size, -size, 0), + (size, size, 0), (-size, size, 0)], [], [(0, 1, 2, 3)]) + mesh.update() + ground = bpy.data.objects.new("Solved Ground Plane", mesh) + collection.objects.link(ground) + ground.display_type = 'WIRE' + ground.hide_render = True + for marker in DATA["markers"]: + scene.timeline_markers.new(marker["name"], frame=marker["frame"]) + + _interpolation(camera) + _interpolation(camera_data) + metadata = bpy.data.texts.new("EffectCraft Tracking Metadata.json") + metadata.write(json.dumps(DATA, ensure_ascii=False, indent=2)) + scene["effectcraft_metadata_text"] = metadata.name + camera_data["effectcraft_average_error_pixels"] = DATA["solve"]["averageError"] + camera_data["effectcraft_solve_method"] = DATA["solve"]["methodUsed"] + camera_data["effectcraft_focus_source"] = "See warnings in " + metadata.name + distortion = DATA["solve"].get("distortion") + if distortion: + for key in ("k1", "k2", "radius"): + camera_data["effectcraft_distortion_" + key] = distortion[key] + if bpy.context.window is not None: + bpy.context.window.scene = scene + for area in bpy.context.window.screen.areas: + if area.type == 'VIEW_3D': + area.spaces.active.region_3d.view_perspective = 'CAMERA' + scene.frame_set(scene.frame_start) + for warning in DATA["warnings"]: + print("EffectCraft: " + warning) + print("EffectCraft: imported", len(DATA["frames"]), "baked camera frames into", scene.name) + return scene + + +if __name__ == "__main__": + EFFECTCRAFT_SCENE = import_scene() diff --git a/crates/engine/src/cmd/camera_cmds.rs b/crates/engine/src/cmd/camera_cmds.rs index 68cd14fb9..e717d51cc 100644 --- a/crates/engine/src/cmd/camera_cmds.rs +++ b/crates/engine/src/cmd/camera_cmds.rs @@ -42,11 +42,25 @@ pub fn specs() -> Vec { create_from_solve ), cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!( + "camera.exportBlender", + "Export Scene to Blender...", + [], + None, + "{path: .py, comp?, layer?, effect?, sensorWidth?: mm (default 36), unitScale?: Blender units per comp pixel (default 0.01), startFrame?: positive integer (default 1), includePoints?: true}", + has_comp, + export_blender + ), cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), ] } +fn export_blender(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.exportBlender")?; + super::blender::export(s, p, cid, lid, uid) +} + fn with_kind(p: &Value, kind: &str) -> Value { let mut q = p.clone(); if !q.is_object() { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 25d05b903..ac887f438 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,5 +1,6 @@ mod anim_presets; mod anim_tools; +mod blender; mod camera_track; mod codec_options; mod color_view; diff --git a/crates/engine/src/tests/blender.rs b/crates/engine/src/tests/blender.rs new file mode 100644 index 000000000..84e939055 --- /dev/null +++ b/crates/engine/src/tests/blender.rs @@ -0,0 +1,261 @@ +//! Synthetic camera interchange: projection, timing, DOF and hostile inputs. Optional public +//! Blender-API oracle: EFFECTCRAFT_BLENDER=/path/to/blender cargo test -p effectcraft-engine tests::blender -- --nocapture. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{Footage, FootageKind, ItemKind, LayerId, LayerSource, build}; +use effectcraft_time::{FrameRate, Tick}; +use effectcraft_track::camtrack::{CameraSolve, Distortion, Ground, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use serde_json::{Value, json}; + +use crate::Session; + +fn path(name: &str) -> String { + let dir = std::env::temp_dir().join(format!("effectcraft-blender-tests-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name).to_string_lossy().into_owned() +} + +fn fixture(ground: bool) -> (Session, LayerId, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot '\\ — câmera", "width": 640, "height": 360, "duration": 6})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let rate = FrameRate::new(30000, 1001); + let lid = s + .edit("fixture", None, |p, _| { + p.comp_mut(cid).unwrap().frame_rate = rate; + let item = p.add_item( + "Plate", + Default::default(), + None, + ItemKind::Footage(Footage { + path: std::env::var("EFFECTCRAFT_BLENDER_PLATE").unwrap_or_else(|_| path("plate '\\ — vídeo.mp4")), + width: 640, + height: 360, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(6.0), + kind: FootageKind::Video, + has_video: true, + ..Default::default() + }), + ); + let comp = p.comp(cid).unwrap().clone(); + let mut layer = build::layer(p, &comp, "Tracked Video", LayerSource::Footage { item }, (640, 360), None); + layer.in_point = rate.tick_of(60); + layer.out_point = rate.tick_of(63); + layer.start_time = rate.tick_of(30); + layer.stretch = 200.0; + let id = layer.id; + p.comp_mut(cid).unwrap().layers.push(layer); + Ok(id) + }) + .unwrap(); + let uid = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0]})).unwrap()["effects"][0].as_u64().unwrap(); + let solve = CameraSolve { + version: 1, + size: [640.0, 360.0], + start: rate.tick_of(30).seconds() / 2.0, + frame_duration: rate.frame_duration().seconds() / 2.0, + shot: ShotType::VariableZoom, + method_used: SolveMethod::Typical, + average_error: 0.12, + frames: (0..3) + .map(|i| SolvedFrame { + rot: [0.02 * i as f64, -0.03 * i as f64, 0.01 * i as f64], + center: [0.1 * i as f64, -0.01 * i as f64, 0.02 * i as f64], + focal: 600.0 + i as f64 * 25.0, + solved: true, + }) + .collect(), + points: vec![ + SolvedPoint { id: 7, pos: [0.1, 0.2, 2.0], error: 0.13, first: 0, last: 2 }, + SolvedPoint { id: 9, pos: [-0.3, -0.1, 1.2], error: 0.11, first: 0, last: 1 }, + ], + ground: ground.then_some(Ground { origin: [0.1, 0.2, 2.0], normal: [0.0, -1.0, 0.0], x_axis: [1.0, 0.0, 0.0] }), + distortion: ground.then_some(Distortion { k1: 0.01, k2: -0.002, radius: 320.0_f64.hypot(180.0) }), + }; + // Directly install a deterministic solve; no analysis or decoder is needed by these tests. + s.execute("prop.set", json!({"layer": lid.0, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.execute("layer.select", json!({"layers": [lid.0]})).unwrap(); + (s, lid, uid) +} + +fn data(script: &str) -> Value { + let line = script.lines().find_map(|l| l.strip_prefix("DATA = json.loads(")).unwrap().strip_suffix(')').unwrap(); + let encoded: String = serde_json::from_str(line).unwrap(); + serde_json::from_str(&encoded).unwrap() +} + +#[test] +fn blender_export_bakes_projection_focus_and_retimed_background() { + let (mut s, lid, uid) = fixture(true); + let result = s.execute("camera.create", json!({"layer": lid.0, "effect": uid})).unwrap(); + let camera = result["camera"].as_u64().unwrap(); + s.execute("layer.cameraSettings", json!({"layer": camera, "focusDistance": 800.0, "dof": true, "fStop": 4.0, "lockToZoom": false})).unwrap(); + for (frame, value) in [(60, 800.0), (62, 1000.0)] { + let time = FrameRate::new(30000, 1001).tick_of(frame).seconds(); + s.execute("prop.addKey", json!({"layer": camera, "path": "cameraOptions/focusDistance", "time": time, "value": value})).unwrap(); + } + let camera_layer = s.active_comp().unwrap().layer(LayerId(camera)).unwrap(); + assert!(camera_layer.props.prop("cameraOptions/dof").unwrap().value.as_bool()); + let output = path("oracle-scene.py"); + let before = s.project.to_json(); + let r = s.execute_checked("camera.exportBlender", json!({"layer": lid.0, "effect": uid, "path": output, "startFrame": 101})).unwrap(); + assert_eq!(s.project.to_json(), before, "export must not edit the document"); + assert_eq!(r["frames"], 3); + assert_eq!(r["points"], 2); + let script = std::fs::read_to_string(&output).unwrap(); + let d = data(&script); + assert_eq!(d["resolution"], json!([640.0, 360.0])); + assert_eq!(d["fpsNumerator"], 30000); + assert_eq!(d["fpsDenominator"], 1001); + assert_eq!(d["frames"][0]["sourceFrame"], 15, "trim + 200% stretch, not scene frame 101"); + assert_eq!(d["frames"][2]["sourceFrame"], 16); + assert_eq!(d["frames"][0]["focus"], 8.0); + assert_eq!(d["frames"][2]["focus"], 10.0); + assert_eq!(d["frames"][0]["dof"], true); + assert!((d["frames"][0]["fstop"].as_f64().unwrap() - 4.0).abs() < 1e-6); + assert_eq!(d["frames"][0]["lens"], 33.75); + assert_eq!(d["frames"][2]["lens"], 36.5625); + assert_eq!(d["ground"], true); + assert_eq!(d["solve"]["distortion"]["k1"], 0.01); + assert_eq!(d["points"][0]["location"][2], 0.0, "ground point lies on Blender XY plane"); + + // Execute the actual exported artifact in a separately installed Blender, when requested. + if let Ok(blender) = std::env::var("EFFECTCRAFT_BLENDER") { + let oracle = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/blender_scene_oracle.py"); + let output = std::process::Command::new(blender) + .args(["--background", "--factory-startup", "--python-exit-code", "1", "--python"]) + .arg(oracle) + .arg("--") + .arg(&output) + .output() + .unwrap(); + assert!(output.status.success(), "Blender oracle failed:\n{}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stdout).contains("EFFECTCRAFT_BLENDER_ORACLE_OK")); + } +} + +#[test] +fn blender_export_bakes_reverse_time_remap_and_footage_loops() { + let (mut s, lid, _) = fixture(false); + let cid = s.active_comp_id().unwrap(); + let rate = FrameRate::new(30000, 1001); + s.edit("retimed fixture", None, |p, _| { + let layer = p.comp(cid).unwrap().layer(lid).unwrap(); + let item = layer.source.item().unwrap(); + let keys = [(60, 8.0), (61, 6.0), (62, -1.0)] + .map(|(frame, time)| effectcraft_keyframe::Keyframe::new(layer.layer_time(rate.tick_of(frame)), KV::Scalar(time))); + let mut remap = build::Ids(&mut p.next_id).prop("timeRemap", "Time Remap", KV::Scalar(8.0)); + remap.keys = keys.to_vec(); + p.comp_mut(cid).unwrap().layer_mut(lid).unwrap().props.children.push(remap.into()); + if let ItemKind::Footage(f) = &mut p.item_mut(item).unwrap().kind { + f.loop_count = 2; + } + Ok(()) + }) + .unwrap(); + let output = path("remap.py"); + s.execute_checked("camera.exportBlender", json!({"layer": lid.0, "path": output})).unwrap(); + let d = data(&std::fs::read_to_string(output).unwrap()); + let indices: Vec = d["frames"].as_array().unwrap().iter().map(|f| f["sourceFrame"].as_i64().unwrap()).collect(); + assert_eq!(indices, vec![60, 0, 0], "reverse remap wraps the 6-second video and clamps negative source time, as the media pool does"); +} + +#[test] +fn blender_export_defaults_and_script_escaping() { + let (mut s, lid, _) = fixture(false); + let output = path("escaped.py"); + let r = s + .execute_checked("camera.exportBlender", json!({"layer": lid.0, "path": output, "includePoints": false, "sensorWidth": 24.0, "unitScale": 0.1})) + .unwrap(); + assert_eq!(r["points"], 0); + let script = std::fs::read_to_string(&output).unwrap(); + let d = data(&script); + assert!(d["name"].as_str().unwrap().contains("'\\ — câmera")); + assert_eq!(d["frames"][0]["lens"], 22.5); + assert_eq!(d["frames"][0]["dof"], false); + assert!(r["warnings"].as_array().unwrap().iter().any(|w| w.as_str().unwrap().contains("cannot be recovered"))); + match std::process::Command::new("python3").args(["-c", "import ast,sys; ast.parse(open(sys.argv[1], encoding='utf-8').read())", &output]).status() { + Ok(status) => assert!(status.success(), "generated script must parse even with quoted/Unicode paths"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => eprintln!("Python AST oracle skipped: python3 is not installed"), + Err(e) => panic!("Python AST oracle failed: {e}"), + } +} + +#[test] +fn blender_export_rejects_invalid_arguments_and_write_failures() { + let (mut s, lid, _) = fixture(false); + for params in [ + json!({"sensorWidth": 0}), + json!({"sensorWidth": "wide"}), + json!({"unitScale": -1}), + json!({"startFrame": -1}), + json!({"startFrame": 1.5}), + json!({"startFrame": 1048574}), + json!({"includePoints": "yes"}), + json!({"bogus": true}), + json!({"path": path("bad.blend")}), + ] { + let mut p = json!({"layer": lid.0, "path": path("invalid.py")}); + p.as_object_mut().unwrap().extend(params.as_object().unwrap().clone()); + assert!(s.execute_checked("camera.exportBlender", p.clone()).is_err(), "must reject {p}"); + } + let blocker = path("blocker"); + std::fs::write(&blocker, b"not a directory").unwrap(); + let p = format!("{blocker}/scene.py"); + assert!(s.execute_checked("camera.exportBlender", json!({"layer": lid.0, "path": p})).unwrap_err().to_string().contains("cannot write")); +} + +#[test] +fn blender_export_rejects_malformed_solves_without_panicking() { + for variant in ["empty", "unsolved", "zero-focal", "bad-ground", "huge-resolution", "mismatched-range"] { + let (mut s, lid, uid) = fixture(false); + let cid = s.active_comp_id().unwrap(); + let mut solve = ct::solve(&crate::camera_track::static_params(s.active_comp().unwrap().layer(lid).unwrap().props.find_group(uid).unwrap())) + .unwrap() + .as_ref() + .clone(); + match variant { + "empty" => solve.frames.clear(), + "unsolved" => solve.frames[0].solved = false, + "zero-focal" => solve.frames[0].focal = 0.0, + "bad-ground" => solve.ground = Some(Ground::default()), + "huge-resolution" => solve.size[0] = 1e10, + _ => { + solve.frames.pop(); + } + } + s.edit("corrupt solve", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(lid).unwrap().props.find_group_mut(uid).unwrap().prop_mut(ct::SOLVE).unwrap().value = KV::Str(solve.to_json()); + Ok(()) + }) + .unwrap(); + s.camera_pending.clear(); + let result = s.execute_checked("camera.exportBlender", json!({"layer": lid.0, "path": path("corrupt.py")})); + assert!(result.is_err(), "must reject {variant}"); + } +} + +#[test] +fn blender_export_respects_script_write_permission() { + let (mut s, lid, _) = fixture(false); + let output = path("script-permissions.py"); + let params = json!({"layer": lid.0, "path": output}); + for (from_script, allowed) in [(true, false), (true, true), (false, false), (false, true)] { + std::fs::write(&output, b"existing file").unwrap(); + s.prefs.scripting.allow_scripts_write_files = allowed; + let result = + if from_script { s.execute_for_script("camera.exportBlender", params.clone()) } else { s.execute_checked("camera.exportBlender", params.clone()) }; + assert!(!s.running_for_script(), "script origin must be restored after success or error"); + let contents = std::fs::read_to_string(&output).unwrap(); + if from_script && !allowed { + assert!(result.unwrap_err().to_string().contains("Allow Scripts to Write Files")); + assert_eq!(contents, "existing file", "blocked scripts must not replace an existing file"); + } else { + assert_eq!(result.unwrap()["frames"], 3); + assert!(contents.contains("def import_scene():")); + } + } +} diff --git a/crates/engine/tests/blender_scene_oracle.py b/crates/engine/tests/blender_scene_oracle.py new file mode 100644 index 000000000..3d1cbeda7 --- /dev/null +++ b/crates/engine/tests/blender_scene_oracle.py @@ -0,0 +1,61 @@ +"""Public-API oracle for scripts generated by src/tests/blender.rs; no Blender source dependency.""" +import json +import math +import runpy +import sys + +import bpy +from bpy_extras.object_utils import world_to_camera_view +from mathutils import Quaternion, Vector + +path = sys.argv[sys.argv.index("--") + 1] +before = set(bpy.data.scenes) +exported = runpy.run_path(path, run_name="__main__") +scene = exported["EFFECTCRAFT_SCENE"] +data = exported["DATA"] +assert before.issubset(set(bpy.data.scenes)), "existing scenes were removed" +assert scene not in before +assert scene.frame_start == 101 and scene.frame_end == 103 +assert abs(scene.render.fps / scene.render.fps_base - 30000 / 1001) < 1e-4 +assert (scene.render.resolution_x, scene.render.resolution_y) == (640, 360) +camera = scene.camera +assert camera.animation_data.action is not None +assert camera.data.animation_data.action is not None +assert camera.data.sensor_fit == 'HORIZONTAL' +metadata = json.loads(bpy.data.texts[scene["effectcraft_metadata_text"]].as_string()) +assert metadata["solve"]["averageError"] == 0.12 +cloud = scene.objects["Solved Track Points"] +assert len(cloud.data.vertices) == 2 +assert cloud.hide_render +assert cloud.data.attributes["track_id"].data[0].value == 7 +assert any(obj.name.startswith("Solved Ground Plane") for obj in scene.objects) + +for i, sample in enumerate(data["frames"]): + scene.frame_set(sample["frame"]) + bpy.context.view_layer.update() + assert abs(camera.data.lens - sample["lens"]) < 1e-5 + assert abs(camera.data.dof.focus_distance - sample["focus"]) < 1e-5 + assert abs(camera.data.dof.aperture_fstop - sample["fstop"]) < 1e-5 + assert camera.data.dof.use_dof + assert (camera.location - Vector(sample["location"])).length < 1e-5 + raw = data["solve"]["frames"][i] + rot = Vector(raw["rot"]) + rotation = Quaternion(rot.normalized(), rot.length).to_matrix() if rot.length else Quaternion().to_matrix() + for vertex, point in zip(cloud.data.vertices, data["solve"]["points"]): + relative = rotation @ (Vector(point["pos"]) - Vector(raw["center"])) + expected = (raw["focal"] * relative.x / relative.z + 320, + raw["focal"] * relative.y / relative.z + 180) + projected = world_to_camera_view(scene, camera, vertex.co) + actual = (projected.x * 640, (1.0 - projected.y) * 360) + assert math.dist(actual, expected) < 0.002, (sample["frame"], actual, expected) + + if "Video not found" not in " ".join(data["warnings"]): + background = camera.data.background_images[0] + assert background.image.source == 'MOVIE' + assert tuple(background.image.size) == (640, 360) + user = background.image_user + actual_source = sample["frame"] - user.frame_start + user.frame_offset + assert actual_source == sample["sourceFrame"], (actual_source, sample["sourceFrame"]) + assert user.use_auto_refresh + +print("EFFECTCRAFT_BLENDER_ORACLE_OK") diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index 061090871..23926864d 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -2996,6 +2996,7 @@ const UKRAINIAN: &[(&str, &str)] = &[ // Brazilian Portuguese. const PORTUGUESE: &[(&str, &str)] = &[ + ("Export Scene to Blender...", "Exportar cena para o Blender..."), (" OK ", " OK "), (" Delete ", " Excluir "), (" Open Auto-Save ", " Abrir salvamento automático "), diff --git a/crates/ui-egui/src/menus/frontend.rs b/crates/ui-egui/src/menus/frontend.rs index 5d65ffb8f..a5d616714 100644 --- a/crates/ui-egui/src/menus/frontend.rs +++ b/crates/ui-egui/src/menus/frontend.rs @@ -560,6 +560,7 @@ pub(super) fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> "anim.applyPreset" => ("path", Ask::Open(&["ecpreset", "json"])), "view.exportGuides" => ("path", Ask::Save("Guides.json")), "view.importGuides" => ("path", Ask::Open(&["json"])), + "camera.exportBlender" => ("path", Ask::Save("Tracked Scene.py")), "file.exportLottie" => ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "file.importProject" => ("path", Ask::Open(&["ecproj", "ecprojx"])), diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs index bca98ea4b..481577398 100644 --- a/crates/ui-egui/src/panels/camera_tracker_ui.rs +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -430,6 +430,11 @@ pub fn editor( actions.push(("camera.create".into(), me.clone())); } app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + let export = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 70.0), vec2(218.0, 22.0)); + if widgets::text_button(ui, export, tr("Export Scene to Blender..."), false, &t, egui::Id::unique(("ct-blender", g.uid))).clicked() && !busy { + actions.push(("camera.exportBlender".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.exportBlender", g.uid), export, "Export Scene to Blender..."); } let status = match app.session.camera_progress() { Some(pr) if running => pr.banner(), @@ -457,4 +462,4 @@ pub fn editor( } /// Height of [`editor`]. -pub const EDITOR_HEIGHT: f32 = 68.0; +pub const EDITOR_HEIGHT: f32 = 100.0; diff --git a/crates/ui-egui/tests/it/ui_camera_tracker.rs b/crates/ui-egui/tests/it/ui_camera_tracker.rs index f7d349342..8c4be61a7 100644 --- a/crates/ui-egui/tests/it/ui_camera_tracker.rs +++ b/crates/ui-egui/tests/it/ui_camera_tracker.rs @@ -16,8 +16,9 @@ const H: f64 = 360.0; fn app() -> (EffectcraftApp, u64, u64) { let mut s = Session::default(); - s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); - let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "Shot", "width": W as u32, "height": H as u32, "duration": 2, "frameRate": 30})).unwrap(); + let plate = + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": W as u32, "height": H as u32})).unwrap()["layer"].as_u64().unwrap(); let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); let uid = r["effects"][0].as_u64().unwrap(); // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. @@ -135,7 +136,11 @@ fn viewer_points_selection_and_effect_controls() { h.state_mut().show_panel(PanelKind::EffectControls); h.run_steps(4); let all = ids(&h); - for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + for want in [ + format!("effectControls.cameraTracker.{uid}.analyze"), + format!("effectControls.cameraTracker.{uid}.createCamera"), + format!("effectControls.cameraTracker.{uid}.exportBlender"), + ] { assert!(all.contains(&want), "missing {want}"); } // Tracker panel: Track Camera is available for the selected layer. @@ -145,6 +150,56 @@ fn viewer_points_selection_and_effect_controls() { assert!(h.state().session.is_enabled("track.camera")); } +#[test] +fn blender_export_button_opens_python_dialog_and_writes_scene() { + use effectcraft_engine::project::{Footage, FootageKind, ItemKind, LayerId, LayerSource}; + let (mut a, plate, uid) = app(); + let cid = a.session.active_comp_id().unwrap(); + a.session + .edit("video fixture", None, |p, _| { + let item = p.add_item( + "Video", + Default::default(), + None, + ItemKind::Footage(Footage { + path: "/tmp/effectcraft-ui-synthetic.mp4".into(), + kind: FootageKind::Video, + width: W as u32, + height: H as u32, + has_video: true, + duration: effectcraft_engine::time::Tick::from_seconds_f64(2.0), + ..Default::default() + }), + ); + p.comp_mut(cid).unwrap().layer_mut(LayerId(plate)).unwrap().source = LayerSource::Footage { item }; + Ok(()) + }) + .unwrap(); + a.session.camera_pending.clear(); + let output = std::env::temp_dir().join(format!("effectcraft-ui-blender-{}.py", std::process::id())); + let selected_path = output.clone(); + a.hooks.pick_save_file = Some(Box::new(move |name, ext| { + assert!(name.ends_with(".py")); + assert_eq!(ext, "py"); + Some(selected_path.to_string_lossy().into_owned()) + })); + let shot = std::env::var("EFFECTCRAFT_BLENDER_UI_SNAPSHOT").ok(); + let builder = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)); + let builder = if shot.is_some() { builder.wgpu() } else { builder }; + let mut h = builder.build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(5); + let button = rect(&h, &format!("effectControls.cameraTracker.{uid}.exportBlender")); + if let Some(path) = shot { + h.render().expect("render export button").save(path).unwrap(); + } + click(&mut h, button.center(), Default::default()); + assert!(output.exists(), "UI export failed: {}", h.state().ui.status); + let script = std::fs::read_to_string(output).expect("clicking the export button saves a Python scene"); + assert!(script.contains("def import_scene():")); + assert_eq!(h.state().session.active_comp().unwrap().layers.len(), 1, "export does not create a camera in the source project"); +} + /// Drag with `button` from `from` to `to` in steps, then release. fn drag_with(h: &mut Harness<'_, EffectcraftApp>, button: egui::PointerButton, from: egui::Pos2, to: egui::Pos2) { h.input_mut().events.push(egui::Event::PointerMoved(from)); diff --git a/docs/agents.md b/docs/agents.md index 172e5c4c6..8c2d49a45 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -446,7 +446,12 @@ and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, point {center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). -Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Each edit is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. + +`camera.exportBlender {path: ".py", layer?, effect?, sensorWidth?: 36, unitScale?: 0.01, +startFrame?: 1, includePoints?: true}` exports the solve as an executable Blender scene script: +baked camera/lens/focus, synced video background, points and ground. See +[Blender camera export](blender-camera-export.md) for framing, assumptions and limitations. Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, `effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. diff --git a/docs/blender-camera-export.md b/docs/blender-camera-export.md new file mode 100644 index 000000000..d102eef4f --- /dev/null +++ b/docs/blender-camera-export.md @@ -0,0 +1,101 @@ +# Exporting a tracked camera to Blender + +After the **3D Camera Tracker** finishes solving, click **Export Scene to Blender...** in its +Effect Controls and save the `.py` file. In Blender 4.0 or newer, open it in the Scripting +workspace's Text Editor and choose **Run Script**. The script creates and activates a new scene; +it preserves existing scenes and objects. Save the resulting `.blend` normally. + +The script uses Blender's public Python API, with no add-on or Python dependencies beyond +Blender itself. For unattended use: `blender --background --python "Tracked Scene.py"`. +That imports the scene in memory; it does not automatically save a `.blend`. + +## Transferred data + +- Camera position and quaternion rotation baked on every analyzed frame, with consistent + quaternion signs and linear interpolation; zoom/lens animation, including Variable Zoom. +- Tracking input/video resolution, horizontal sensor fit, exact rational composition FPS, + frame range and composition markers within the analyzed interval. The first analyzed frame + becomes Blender frame 1 by default; the original composition timestamps remain in metadata. +- The original video or still as the camera's background image, at full opacity, refreshed on + frame changes. Source-frame offsets are baked too, accounting for trim, stretch, source FPS + and Time Remap. This is a viewport background, not a render/compositor background. +- Focus distance, DOF switch, f-stop, iris blade count, rotation and aspect sampled from the + tracker's linked **3D Tracker Camera**, including keyframes and expressions. Create that + camera and adjust its Camera Options before exporting to transfer custom focus settings. +- Solved 3D points as a non-rendering vertex mesh, with point attributes for track ID, + reprojection error and first/last visible frame. A chosen ground plane becomes Blender's XY + plane, with its origin preserved and a non-rendering wire plane as a placement guide. +- Shutter duration (not automatically enabled), plus the original shutter phase in metadata. +- A Blender Text datablock named `EffectCraft Tracking Metadata.json` containing the complete + solve, optional original 2D tracks, per-frame timings, source interpretation, diagnostics and + export warnings. Radial distortion coefficients are also camera custom properties. + +Coordinates are converted from EffectCraft's right/down/forward world to Blender's +right/forward/up world, including the camera's local up and negative-Z viewing direction. +Export uses the tracking input's framing, not the footage layer's final 2D placement in the +composition. Camera pose and lens always come from the solve; subsequent manual pose/lens edits +on the linked camera are not substituted for the tracked motion. + +## What cannot be inferred + +Monocular tracking cannot recover absolute scene scale, physical sensor size, optical focus, +f-stop or exposure from feature tracks. The default sensor width is an assumption of **36 mm**; +the focal length in millimeters is derived from it while preserving the solved field of view. +The default scale is **0.01 Blender units per composition-world pixel**, not a measurement. +Without a linked camera, DOF is disabled and focus distance uses the default solved scene depth. +Blender's physical DOF approximates EffectCraft's blur model: Blur Level and Iris Roundness are +retained in metadata, but have no direct Blender equivalent. + +Brown-model radial `k1`/`k2` coefficients use radius normalized to half the image diagonal. +They are preserved, **not** assigned to Blender's different lens-distortion model. For a precise +match when distortion is nonzero, use an undistorted plate; the script prints a warning. +The tracker assumes square pixels, so Blender renders square pixels; any non-square source +interpretation is retained in metadata and flagged. + +The `.py` references the video; it does not contain a copy of the video's bytes. When moving it, +keep the video next to the script (same filename), or relink the background image in Blender. +Missing/unsupported media produces a warning while retaining the imported camera and geometry. +Masks and effects applied before the tracker are not baked into the original media. Export +requires a direct video/still footage layer at the analyzed size; render precomps, sequences or +size-changing preprocessing to a video and track that video first. + +## Agent / control-channel command + +```json +{ + "command": "camera.exportBlender", + "params": { + "layer": "#1", + "path": "/output/Tracked Scene.py", + "sensorWidth": 36, + "unitScale": 0.01, + "startFrame": 1, + "includePoints": true + } +} +``` + +`comp`, `layer` and `effect` select the tracker as with `camera.analyze`. Without explicit +selection, the selected tracker (or first tracker in the composition) is used. `sensorWidth`, +`unitScale`, `startFrame` and `includePoints` are optional. The reply contains file size, +exported frame/point counts and warnings. Writing uses the host's atomic file service. +Calls issued by scripts require **Allow Scripts to Write Files and Access Network** in +Settings ▸ Scripting & Expressions. Export from the UI or control channel remains available +with that preference off. + +Export is limited to 100,000 analyzed frames, 100,000 points and 64 MiB of serialized scene data; +invalid, stale or incomplete solves return an actionable error without modifying the document. + +## Verification + +Engine tests check argument validation, escaped/Unicode paths, fractional FPS, trim/stretch, +focus/f-stop, zoom, ground placement, malformed solves and write failures. To additionally run +the generated artifact against an installed Blender's public API: + +```sh +EFFECTCRAFT_BLENDER=/path/to/blender cargo test -p effectcraft-engine tests::blender -- --nocapture +``` + +Optionally set `EFFECTCRAFT_BLENDER_PLATE` to a local synthetic 640×360 video to exercise media +loading and source-frame offsets. The oracle checks every synthetic point's projection through +the imported camera on every frame, as well as animation, DOF, timing and scene preservation. From e2b56f1ba52b654c716107fdd75715f8a60d506a Mon Sep 17 00:00:00 2001 From: Krueels <90205419+Krueels@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:05:17 -0300 Subject: [PATCH 24/33] #986: remove personal credit from generated Blender scripts --- crates/engine/src/cmd/blender_scene.py | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/engine/src/cmd/blender_scene.py b/crates/engine/src/cmd/blender_scene.py index 7240cb1ca..9f8a522c5 100644 --- a/crates/engine/src/cmd/blender_scene.py +++ b/crates/engine/src/cmd/blender_scene.py @@ -1,5 +1,4 @@ # EffectCraft camera tracker → Blender, original code under MIT OR Apache-2.0. -# Blender scene export contributor: Krueels — https://github.com/Krueels # Run in Blender's Scripting workspace (Open, Run Script), or: # blender --background --python "Tracked Scene.py" # Creates a NEW scene; existing scenes and objects are preserved. Save the .blend yourself. From 73817dc3f0c37ccbf639add2f54925322dc40bea Mon Sep 17 00:00:00 2001 From: Krueels <90205419+Krueels@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:28:04 -0300 Subject: [PATCH 25/33] #986: confine Blender exports and trim generated header --- crates/automation/src/confine.rs | 1 + crates/automation/src/tests.rs | 27 ++++++++++++++++++++++++++ crates/engine/src/cmd/blender_scene.py | 3 +-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 67e5aef98..dc64084d3 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -97,6 +97,7 @@ const MIXED_WRITES: &[(&str, &[&str])] = &[("templates.create", &["projectPath", /// Commands that write files named by these parameters. const WRITES: &[(&str, &[&str])] = &[ + ("camera.exportBlender", &["path"]), ("file.save", &["path"]), ("file.saveAs", &["path"]), ("file.saveCopy", &["path"]), diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index 560a527c7..49f0389f6 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -1473,3 +1473,30 @@ fn automation_roots_refuse_running_script_code() { call_json(&mut s, "execute_command", json!({"command": "scriptui.list"})); let _ = std::fs::remove_dir_all(&base); } + +#[test] +fn blender_export_requires_an_absolute_path_in_a_write_root() { + use crate::confine::Confinement; + let base = std::env::temp_dir().join(format!("ec-blender-confine-{}", std::process::id())); + let (read, write) = (base.join("read"), base.join("write")); + for dir in [&read, &write] { + std::fs::create_dir_all(dir).unwrap(); + } + let confinement = Confinement::new(std::slice::from_ref(&read), std::slice::from_ref(&write)).unwrap(); + let check = |path: &std::path::Path| confinement.check_command("camera.exportBlender", &json!({"path": path})); + assert_eq!(check(&write.join("scene.py")).unwrap(), None); + for path in [read.join("scene.py"), base.join("outside.py"), write.join("../read/scene.py")] { + assert!(check(&path).is_err(), "must deny {}", path.display()); + } + for path in ["scene.py", "./scene.py", "write/scene.py", "../scene.py"] { + let error = check(std::path::Path::new(path)).unwrap_err().to_string(); + assert!(error.contains("give an absolute path"), "{path}: {error}"); + } + let read_only = Confinement::new(std::slice::from_ref(&write), &[]).unwrap(); + assert!(read_only.check_command("camera.exportBlender", &json!({"path": write.join("scene.py")})).is_err()); + for path in [json!(read.join("scene.py")), json!("scene.py"), json!("$1.path")] { + let batch = json!({"steps": [{"command": "camera.exportBlender", "params": {"path": path}}]}); + assert!(confinement.check_command("engine.batch", &batch).is_err()); + } + std::fs::remove_dir_all(&base).unwrap(); +} diff --git a/crates/engine/src/cmd/blender_scene.py b/crates/engine/src/cmd/blender_scene.py index 9f8a522c5..0a9700a00 100644 --- a/crates/engine/src/cmd/blender_scene.py +++ b/crates/engine/src/cmd/blender_scene.py @@ -2,8 +2,7 @@ # Run in Blender's Scripting workspace (Open, Run Script), or: # blender --background --python "Tracked Scene.py" # Creates a NEW scene; existing scenes and objects are preserved. Save the .blend yourself. -# Public API references: https://docs.blender.org/api/4.5/bpy.types.Camera.html -# https://docs.blender.org/api/4.5/bpy.types.ImageUser.html +# Public API types: bpy.types.Camera, bpy.types.ImageUser import json import math import os From d91b14db54848d3447b11bb5e36c29afdd904b03 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 19:30:31 -0400 Subject: [PATCH 26/33] Viewer: a precomp's undecodable proxy has its notice with Collapse on, now that a proxy keeps the flattened path (#1701) --- crates/ui-egui/src/panels/viewer.rs | 8 +++----- crates/ui-egui/tests/it/ui_unreadable_footage.rs | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index a255bd321..d084dd006 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -8,8 +8,8 @@ use effectcraft_engine::commands::shape_tool::PaintKind; use effectcraft_engine::effects::puppet::PinKind; use effectcraft_engine::geom::{Mat3, vec2 as gv2}; use effectcraft_engine::project::{Comp, Layer, LayerId, LayerSource}; +use effectcraft_engine::render::EvalCtx; use effectcraft_engine::render::three_d::{self, CameraState, View3D}; -use effectcraft_engine::render::{EvalCtx, Renderer}; use effectcraft_engine::time::Tick; use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::json; @@ -721,13 +721,11 @@ fn phase_scene(app: &mut EffectcraftApp, ectx: &EvalCtx, v: &mut Vw<'_>) { } /// Up to three layers active at `time` whose footage, or the proxy in use for footage or a -/// precomp, can't be decoded, each with the reason. A collapsed precomp draws its own layers, -/// never the comp's proxy. +/// precomp, can't be decoded, each with the reason. fn footage_problems<'c>(app: &EffectcraftApp, ectx: &EvalCtx, comp: &'c Comp, time: Tick) -> Vec<(&'c Layer, String)> { - let r = Renderer::new(ectx.project, &*app.session.footage, Default::default()); comp.layers .iter() - .filter(|l| l.is_active_at(time) && l.switches.video && r.collapsed(ectx, l).is_none()) + .filter(|l| l.is_active_at(time) && l.switches.video) .filter_map(|layer| { let (LayerSource::Footage { item } | LayerSource::Comp { item }) = &layer.source else { return None }; Some((layer, ectx.project.item(*item).and_then(|i| app.session.footage.item_problem(i))?)) diff --git a/crates/ui-egui/tests/it/ui_unreadable_footage.rs b/crates/ui-egui/tests/it/ui_unreadable_footage.rs index 63e3a75c2..a62884f43 100644 --- a/crates/ui-egui/tests/it/ui_unreadable_footage.rs +++ b/crates/ui-egui/tests/it/ui_unreadable_footage.rs @@ -83,10 +83,10 @@ fn an_undecodable_proxy_in_use_is_marked_as_the_proxy() { assert!(notice.label.contains(&format!("Proxy: {WHY}")), "{notice:?}"); } -/// A precomp whose proxy can't be decoded: the layer says so while it renders through the -/// proxy, but not with Collapse Transformations on, which draws the precomp's own layers. +/// A precomp whose proxy can't be decoded: the layer says so with Collapse Transformations off +/// or on, since a proxy in use keeps the precomp on the flattened path that draws the proxy. #[test] -fn a_collapsed_precomp_has_no_notice_for_its_proxy() { +fn a_precomp_with_an_undecodable_proxy_has_a_notice_collapsed_or_not() { for collapse in [false, true] { let mut s = Session { footage: Arc::new(Stub(|f| f.path.ends_with(".avi"))), ..Default::default() }; s.execute("comp.new", json!({"name": "Inner", "width": 64, "height": 36, "duration": 4})).unwrap(); @@ -99,6 +99,6 @@ fn a_collapsed_precomp_has_no_notice_for_its_proxy() { s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); s.execute("layer.setSwitch", json!({"layers": ["#1"], "switch": "collapse", "value": collapse})).unwrap(); let h = open(s); - assert_eq!(h.state().auto.query("viewer.footageProblem.").is_empty(), collapse, "collapse {collapse}"); + assert!(!h.state().auto.query("viewer.footageProblem.").is_empty(), "collapse {collapse}"); } } From 63ec5f2aad6d865cd3cb75428a1f222e32753252 Mon Sep 17 00:00:00 2001 From: erythrofr <20369086+erythrofr@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:39:56 +0100 Subject: [PATCH 27/33] Export: apply crop once when calculating resize dimensions Calculate frame_size from the original rendered dimensions and effective render scale before cropping, so locked-aspect resize does not apply the crop margins a second time in its dimension calculation. Add dimension and decoded-PNG regressions for crop, resize, padding, scaled ROI, both interpolation qualities, and zero or extreme requested sizes. AI assistance: OpenAI Codex (GPT-6). --- crates/export/src/pipeline.rs | 84 ++++++++++++++++++++++- crates/export/tests/it/render_settings.rs | 23 +++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index c639ebad1..7b9e4111c 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -431,12 +431,13 @@ impl<'a> Cx<'a> { fn crop_resize(&self, img: Image) -> Image { let om = self.job.output; let scale = self.comp().map(|c| self.job.settings.effective_scale(c)).unwrap_or_else(|| self.job.settings.resolution.clamp(0.01, 4.0)); + // frame_size applies crop itself: use the original frame and the same scale as edges. + let (w, h) = om.frame_size(img.width, img.height, scale); let (t, l, b, r) = om.crop.edges(img.width, img.height, scale); let img = if (t, l, b, r) == (0, 0, 0, 0) { img } else { crop(&img, t, l, b, r) }; if !om.resize.enabled { return img; } - let (w, h) = om.frame_size(img.width, img.height, 1.0); if (w, h) == (img.width, img.height) { return img; } @@ -1062,6 +1063,87 @@ mod tests { } } + #[test] + fn crop_resize_applies_margins_once() { + use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; + let output = OutputModule { + crop: Crop { enabled: true, left: 480, right: 480, ..Default::default() }, + resize: Resize { enabled: true, width: 1280, lock_aspect: true, quality: ResizeQuality::Low, ..Default::default() }, + ..Default::default() + }; + assert_eq!(output.frame_size(1920, 1080, 1.0), (1280, 1440)); + let img = run_crop_resize(Image::new(1920, 1080), &output, 1.0); + assert_eq!((img.width, img.height), (1280, 1440)); + assert_eq!(img.data.len(), 1280 * 1440); + } + + fn run_crop_resize(img: Image, output: &effectcraft_project::render_queue::OutputModule, scale: f64) -> Image { + use effectcraft_project::render_queue::RenderSettings; + let project = Project::default(); + let settings = RenderSettings { resolution: scale, ..Default::default() }; + let job = Job { + project: &project, + footage: &effectcraft_render::NoFootage, + expr: None, + accel: None, + comp: effectcraft_project::ItemId(0), + settings: &settings, + output, + path: "unused.png", + sink: None, + nested_switches: true, + options: Default::default(), + }; + Cx::new(&job).crop_resize(img) + } + + #[test] + fn crop_resize_dimensions_match_the_output_module() { + use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; + let edges = Crop { enabled: true, left: 2, right: 2, ..Default::default() }; + let padding = Crop { enabled: true, left: -2, bottom: -2, ..Default::default() }; + let roi = |rect| Crop { enabled: true, use_roi: true, roi: Some(rect), ..Default::default() }; + for quality in [ResizeQuality::Low, ResizeQuality::High] { + let resize = |width| Resize { enabled: true, width, height: 1, lock_aspect: true, quality }; + for (crop, resize, scale, expected) in [ + (Crop::default(), Resize::default(), 1.0, (8, 6)), + (edges, Resize::default(), 1.0, (4, 6)), + (Crop::default(), resize(4), 1.0, (4, 3)), + (edges, resize(8), 1.0, (8, 12)), + (padding, resize(5), 1.0, (5, 4)), + (roi([4.0, 2.0, 8.0, 6.0]), resize(8), 0.5, (8, 6)), + (roi([1.0, 0.5, 2.0, 1.5]), resize(8), 2.0, (8, 6)), + (edges, Resize { lock_aspect: false, height: 2, ..resize(5) }, 1.0, (5, 2)), + ] { + let output = OutputModule { crop, resize, ..Default::default() }; + assert_eq!(output.frame_size(8, 6, scale), expected); + let img = run_crop_resize(Image::filled(8, 6, [0.25, 0.0, 0.0, 0.5]), &output, scale); + assert_eq!((img.width, img.height), expected, "{crop:?}, {resize:?}, scale {scale}"); + assert_eq!(img.data.len(), expected.0 as usize * expected.1 as usize); + } + } + } + + #[test] + fn crop_resize_zero_and_extreme_sizes_stay_bounded() { + use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; + for quality in [ResizeQuality::Low, ResizeQuality::High] { + for (size, crop, width, height, expected) in [ + ((8, 6), Crop::default(), 0, 0, (1, 1)), + ((8, 1), Crop::default(), u32::MAX, 1, (30_000, 1)), + ((1, 8), Crop::default(), 1, u32::MAX, (1, 30_000)), + ((8, 6), Crop { enabled: true, left: 20_000, right: 20_000, ..Default::default() }, 2, 3, (2, 3)), + ((0, 0), Crop { enabled: true, left: 1, top: 1, ..Default::default() }, 2, 3, (2, 3)), + ] { + let output = OutputModule { crop, resize: Resize { enabled: true, width, height, lock_aspect: false, quality }, ..Default::default() }; + let img = run_crop_resize(Image::new(size.0, size.1), &output, 1.0); + assert_eq!((img.width, img.height), expected); + assert_eq!(img.data.len(), expected.0 as usize * expected.1 as usize); + assert!(img.data.iter().all(|p| *p == [0.0; 4])); + } + } + } + #[test] fn crop_and_pad() { let mut img = Image::new(4, 3); diff --git a/crates/export/tests/it/render_settings.rs b/crates/export/tests/it/render_settings.rs index 465d8e5fe..c2d7dcbfa 100644 --- a/crates/export/tests/it/render_settings.rs +++ b/crates/export/tests/it/render_settings.rs @@ -174,6 +174,29 @@ fn crop_roi_and_resize() { assert_eq!((z.width(), z.height()), (32, 24)); } +#[test] +fn crop_resize_preserves_pixels_when_the_cropped_size_is_requested() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + for (crop, scale, expected) in [ + (Crop { enabled: true, left: 16, right: 16, ..Default::default() }, 1.0, (32, 48)), + (Crop { enabled: true, left: -4, top: -2, ..Default::default() }, 1.0, (68, 50)), + (Crop { enabled: true, use_roi: true, roi: Some([0.0, 24.0, 32.0, 24.0]), ..Default::default() }, 0.5, (16, 12)), + (Crop { enabled: true, use_roi: true, roi: Some([0.0, 24.0, 32.0, 24.0]), ..Default::default() }, 2.0, (64, 48)), + ] { + let settings = RenderSettings { resolution: scale, ..custom(0.4, 0.44) }; + let mut output = OutputModule { crop, ..OutputModule::for_format(OutputFormat::PngSequence) }; + let reference = png(&run(&p, cid, &settings, &output, "/c/reference_[#####].png"), 0); + assert_eq!(reference.dimensions(), expected); + assert!(reference.pixels().any(|p| p.0[0] > 150), "the reference contains visible content"); + for quality in [ResizeQuality::Low, ResizeQuality::High] { + output.resize = Resize { enabled: true, width: expected.0, height: 1, lock_aspect: true, quality }; + let actual = png(&run(&p, cid, &settings, &output, "/c/resized_[#####].png"), 0); + assert_eq!(actual.dimensions(), expected); + assert!(actual == reference, "crop {crop:?}, scale {scale}, quality {quality:?}: pixels changed"); + } + } +} + #[test] fn alpha_only_and_premultiplied() { let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { From 3bc1caaedbf53dac566ecd677b84d9400bae18ae Mon Sep 17 00:00:00 2001 From: Camille Moine Date: Mon, 19 Oct 2026 19:54:11 +0200 Subject: [PATCH 28/33] Shapes: per-corner roundness for Rectangle Path (#1896) Add Per-Corner Roundness to Rectangle Path: a checkbox (off by default) and four Pixels properties (top-left, top-right, bottom-right, bottom-left). New rectangles initialise the four values from their Roundness, so ticking the box keeps the shape; the radii are animatable and expression-driven.\n\nrect_radii() builds the path clockwise from the top-left with kappa quarter-circle arcs. Overlapping radii scale down like CSS border-radius; negative and non-finite radii are square; rect() delegates to it, so uniform roundness clamps to half the shorter side as before.\n\nLottie rc has one r: four equal static radii export as that value; mixed or animated radii export roundness 0 with a warning. Import puts r into the four per-corner values so ticking the box on an imported rectangle keeps the shape.\n\nTests: path geometry and clamping, rendered per-corner pixels, Lottie equal/mixed/checkbox-off export. --- crates/engine/src/cmd/paths.rs | 65 +++++++++++----- crates/engine/src/tests/stubs.rs | 64 +++++++++++++++ crates/lottie/src/shapes.rs | 22 +++++- crates/lottie/src/tests.rs | 36 +++++++++ crates/path/src/lib.rs | 77 +++++++++++++++++++ crates/project/src/build.rs | 7 ++ crates/render/src/shapes.rs | 15 +++- crates/render/src/shapes/tests.rs | 30 ++++++++ crates/ui-egui/src/panels/timeline/tests.rs | 5 ++ crates/ui-egui/tests/it/ui_timeline_shapes.rs | 5 ++ 10 files changed, 304 insertions(+), 22 deletions(-) diff --git a/crates/engine/src/cmd/paths.rs b/crates/engine/src/cmd/paths.rs index 0bb2eaae2..7cd457640 100644 --- a/crates/engine/src/cmd/paths.rs +++ b/crates/engine/src/cmd/paths.rs @@ -5,6 +5,7 @@ use effectcraft_keyframe::{ShapePath, Value as KV}; use effectcraft_project::build::{self, Ids}; use effectcraft_project::{FeatherFalloff, GroupKind, ItemId, LayerId, LayerSource, MaskMotionBlur, Node, PropGroup, Uid}; +use effectcraft_render::EvalCtx; use serde_json::{Value, json}; use super::layer_menu::target_masks; @@ -111,18 +112,30 @@ fn roto_bezier(s: &mut Session, p: &Value) -> Result { /// The parametric shape paths that Convert To Bezier Path replaces with an editable path. const PARAMETRIC: [&str; 3] = ["rect", "ellipse", "star"]; -/// The path a parametric shape group draws, in the group's own space, read at `lt`. This is the -/// geometry the renderer builds from the same values (`render::shapes::collect`), so the converted -/// path is the shape that was on screen rather than an approximation of it; the defaults match -/// that code for a property a group doesn't carry (a polygon has no inner radius or roundness). -fn parametric_path(g: &PropGroup, lt: effectcraft_time::Tick) -> Option { - let f = |id: &str, d: f64| g.prop(id).map_or(d, |pr| pr.value_at(lt).as_f64()); - let v2 = |id: &str, d: [f64; 2]| g.prop(id).map_or(d, |pr| pr.value_at(lt).as_vec2()); +/// The path a parametric shape group draws, in the group's own space. This is the geometry the +/// renderer builds from the same values (`render::shapes::collect`), expressions included, so the +/// converted path is the shape that was on screen rather than an approximation of it; the defaults +/// match that code for a property a group doesn't carry (a polygon has no inner radius or +/// roundness). +fn parametric_path(ctx: &EvalCtx, layer: &effectcraft_project::Layer, g: &PropGroup) -> Option { + let f = |id: &str, d: f64| ctx.f(layer, g, id, d); + let v2 = |id: &str, d: [f64; 2]| ctx.v2(layer, g, id, d); let path = match g.match_id.as_str() { - "rect" => effectcraft_path::rect(v2("size", [100.0; 2]), v2("position", [0.0; 2]), f("roundness", 0.0)), + "rect" => { + let (size, pos) = (v2("size", [100.0; 2]), v2("position", [0.0; 2])); + if ctx.b(layer, g, "perCorner") { + effectcraft_path::rect_radii( + size, + pos, + [f("topLeftRoundness", 0.0), f("topRightRoundness", 0.0), f("bottomRightRoundness", 0.0), f("bottomLeftRoundness", 0.0)], + ) + } else { + effectcraft_path::rect(size, pos, f("roundness", 0.0)) + } + } "ellipse" => effectcraft_path::ellipse(v2("size", [100.0; 2]), v2("position", [0.0; 2])), "star" => effectcraft_path::polystar(effectcraft_path::Polystar { - star: g.prop("type").map_or(0, |pr| pr.value_at(lt).as_enum()) == 0, + star: ctx.e(layer, g, "type") == 0, points: f("points", 5.0), pos: v2("position", [0.0; 2]), rotation: f("rotation", 0.0), @@ -160,28 +173,40 @@ fn convert_to_bezier(s: &mut Session, p: &Value) -> Result { } let t = s.time(); let n = shapes.len(); + // Read the geometry before editing, with the same expression-aware context the renderer + // uses: an expression-driven corner radius, size or position must convert to what is on + // screen, not to the property's underlying value. + let expr = s.expr.clone(); + let prepared = { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let ctx = + EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: expr.as_deref(), footage: None, property_overrides: &[], media_overrides: &[] }; + let mut prepared = Vec::new(); + for (lid, uid) in &shapes { + let Some(l) = comp.layer(*lid) else { continue }; + let Some(old) = l.props.sub("contents").and_then(|c| c.find_group(*uid)) else { continue }; + let Some(path) = parametric_path(&ctx, l, old) else { continue }; + // Path Direction is the shape's own, not a default: reversing is a separate command. + prepared.push((*lid, *uid, path, ctx.e(l, old, "direction"), old.enabled)); + } + prepared + }; let made = s.edit("Convert To Bezier Path", None, |proj, st| { let mut next = proj.next_id; let mut made = Vec::new(); - for (lid, uid) in &shapes { - let l = layer_mut(proj, cid, *lid)?; - let lt = l.layer_time(t); + for (lid, uid, path, direction, enabled) in prepared { + let l = layer_mut(proj, cid, lid)?; let contents = l.props.sub_mut("contents").ok_or_else(|| bad("path.convertToBezier", "not a shape layer"))?; - let Some(old) = contents.find_group(*uid) else { continue }; - let Some(path) = parametric_path(old, lt) else { continue }; - // Path Direction is the shape's own, not a default: reversing is a separate command. - let direction = old.prop("direction").map_or(0, |pr| pr.value_at(lt).as_enum()); - let enabled = old.enabled; let mut g = build::shape_path(&mut Ids(&mut next), path); if let Some(pr) = g.prop_mut("direction") { pr.value = KV::Enum(direction); } g.enabled = enabled; let new_uid = g.uid; - let parent = contents.parent_of_mut(*uid).ok_or_else(|| bad("path.convertToBezier", "no such shape item"))?; - let Some(at) = parent.children.iter().position(|c| c.uid() == *uid) else { continue }; + let parent = contents.parent_of_mut(uid).ok_or_else(|| bad("path.convertToBezier", "no such shape item"))?; + let Some(at) = parent.children.iter().position(|c| c.uid() == uid) else { continue }; parent.children[at] = Node::Group(g); - made.push((*lid, new_uid)); + made.push((lid, new_uid)); } proj.next_id = next; // The new paths are what is selected, so the vertices can be edited straight away. diff --git a/crates/engine/src/tests/stubs.rs b/crates/engine/src/tests/stubs.rs index 9735f2c08..087a4ae1a 100644 --- a/crates/engine/src/tests/stubs.rs +++ b/crates/engine/src/tests/stubs.rs @@ -585,6 +585,70 @@ fn convert_to_bezier_reads_the_shape_as_it_is_set() { assert!((mid(&xs) - 10.0).abs() < 1e-6 && (mid(&ys) - 5.0).abs() < 1e-6, "centred on the position: {:?} {:?}", mid(&xs), mid(&ys)); } +/// #1896: Convert To Bezier Path reads Per-Corner Roundness, not just the uniform Roundness, so +/// the editable path is the shape that was on screen. +#[test] +fn convert_to_bezier_honours_per_corner_roundness() { + let mut s = comp(); + let l = s.execute("layer.newShape", json!({"kind": "rect", "size": [120, 80]})).unwrap()["layer"].as_u64().unwrap(); + let (uid, _) = first_shape(&s, l); + let uid_of = |s: &Session, id: &str| layer(s, l).props.sub("contents").unwrap().find_group(uid).unwrap().get(id).unwrap().uid; + let per_corner = uid_of(&s, "perCorner"); + let top_left = uid_of(&s, "topLeftRoundness"); + let bottom_right = uid_of(&s, "bottomRightRoundness"); + for (u, v) in [(per_corner, json!(true)), (top_left, json!(20.0)), (bottom_right, json!(10.0))] { + s.execute("prop.set", json!({"layer": l, "path": format!("@{u}"), "value": v})).unwrap(); + } + + s.state.selected_props = vec![(LayerId(l), uid)]; + s.execute("path.convertToBezier", json!({})).unwrap(); + let (new_uid, now) = first_shape(&s, l); + assert_eq!(now, "path"); + let path = shape_path_of(&s, l, new_uid); + let expected = effectcraft_path::from_kurbo(&effectcraft_path::rect_radii([120.0, 80.0], [0.0, 0.0], [20.0, 0.0, 10.0, 0.0])).into_iter().next().unwrap(); + assert_eq!(path, expected, "the converted path has the per-corner geometry"); +} + +/// #1896: Convert To Bezier Path evaluates the shape's expressions, so an expression-driven +/// Per-Corner checkbox and radius convert to the geometry on screen. +#[test] +fn convert_to_bezier_evaluates_corner_expressions() { + struct Corners; + impl effectcraft_render::ExprHost for Corners { + fn eval( + &self, + _: &effectcraft_render::EvalCtx, + _: &effectcraft_project::Layer, + prop: &effectcraft_project::Property, + value: &KV, + ) -> std::result::Result { + Ok(match prop.match_id.as_str() { + "perCorner" => KV::Bool(true), + "topLeftRoundness" => KV::Scalar(20.0), + _ => value.clone(), + }) + } + } + + let mut s = comp(); + s.expr = Some(Arc::new(Corners)); + let l = s.execute("layer.newShape", json!({"kind": "rect", "size": [120, 80]})).unwrap()["layer"].as_u64().unwrap(); + let (uid, _) = first_shape(&s, l); + let uid_of = |s: &Session, id: &str| layer(s, l).props.sub("contents").unwrap().find_group(uid).unwrap().get(id).unwrap().uid; + for (id, expression) in [("perCorner", "true"), ("topLeftRoundness", "20")] { + let u = uid_of(&s, id); + s.execute("prop.setExpression", json!({"layer": l, "path": format!("@{u}"), "expression": expression})).unwrap(); + } + + s.state.selected_props = vec![(LayerId(l), uid)]; + s.execute("path.convertToBezier", json!({})).unwrap(); + let (new_uid, now) = first_shape(&s, l); + assert_eq!(now, "path"); + let path = shape_path_of(&s, l, new_uid); + let expected = effectcraft_path::from_kurbo(&effectcraft_path::rect_radii([120.0, 80.0], [0.0, 0.0], [20.0, 0.0, 0.0, 0.0])).into_iter().next().unwrap(); + assert_eq!(path, expected, "the expression-evaluated corner is converted"); +} + /// #2201: a viewer drag on the free-transform box sends one `path.freeTransform` per frame with a /// shared merge key; one undo must revert the whole drag, not the last frame. #[test] diff --git a/crates/lottie/src/shapes.rs b/crates/lottie/src/shapes.rs index 990a0f28b..fc823b775 100644 --- a/crates/lottie/src/shapes.rs +++ b/crates/lottie/src/shapes.rs @@ -113,6 +113,9 @@ impl Out<'_, '_> { fn f(&self, m: &str) -> f64 { self.g.get(m).map(|p| p.value.as_f64()).unwrap_or(0.0) } + fn flag(&self, m: &str) -> bool { + self.g.get(m).map(|p| p.value.as_bool()).unwrap_or(false) + } fn animated(&self, m: &str) -> bool { self.g.get(m).is_some_and(|p| p.is_animated() || p.has_expression()) } @@ -232,7 +235,19 @@ pub(crate) fn export_items(ex: &mut Ex, contents: &PropGroup, label: &str) -> Ve dir(&mut o); o.put("s", "size", &anim::arr2); o.put("p", "position", &anim::arr2); - o.put("r", "roundness", &anim::num); + if o.flag("perCorner") { + let names = ["topLeftRoundness", "topRightRoundness", "bottomRightRoundness", "bottomLeftRoundness"]; + let radii = names.map(|m| o.f(m)); + // Lottie `rc` has one `r`: four equal static radii export as that value. + if radii.iter().all(|v| (v - radii[0]).abs() < 1e-6) && !names.iter().any(|m| o.animated(m)) && radii[0] >= 0.0 { + o.raw("r", json!(r(radii[0]))); + } else { + o.ex.warn(format!("{}: per-corner roundness has no Lottie rectangle equivalent (roundness 0 exported)", o.label)); + o.raw("r", json!(0)); + } + } else { + o.put("r", "roundness", &anim::num); + } } "el" => { dir(&mut o); @@ -447,6 +462,11 @@ pub(crate) fn import_items(im: &mut Im, ids: &mut Ids, items: &[Json]) -> Vec { diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 203f04c04..24b4690eb 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -896,3 +896,39 @@ fn muted_keys_are_omitted_from_the_export() { assert!((opacity_at(&p, cid, 1.0) - 150.0).abs() < 1e-6); assert!((opacity_at(&q, nc, 1.0) - 150.0).abs() < 1e-6); } + +#[test] +fn per_corner_roundness_exports_equal_radii_and_warns_when_mixed() { + let shape = |radii: [f64; 4], roundness: f64, per_corner: bool| { + let (mut p, cid) = setup(); + let l = shape_layer(&mut p, cid, move |ids| { + let mut g = build::shape_rect(ids, [100.0, 50.0], [0.0, 0.0], roundness); + g.prop_mut("perCorner").unwrap().value = Value::Bool(per_corner); + for (m, v) in + [("topLeftRoundness", radii[0]), ("topRightRoundness", radii[1]), ("bottomRightRoundness", radii[2]), ("bottomLeftRoundness", radii[3])] + { + g.prop_mut(m).unwrap().value = Value::Scalar(v); + } + let fill = build::shape_fill(ids, [1.0, 0.0, 0.0, 1.0]); + vec![build::shape_group(ids, "Rectangle 1", vec![g, fill])] + }); + push(&mut p, cid, l); + (p, cid) + }; + let rc = |json: &Json| -> Json { json["layers"][0]["shapes"][0]["it"].as_array().unwrap().iter().find(|i| i["ty"] == "rc").cloned().unwrap() }; + // Four equal static radii: `rc.r` carries the value, no warning. + let (p, cid) = shape([12.0; 4], 5.0, true); + let (_, _, warnings, json) = roundtrip(&p, cid, &ExportOptions::default()); + assert_eq!(rc(&json)["r"], json!(12.0)); + assert!(!warnings.iter().any(|w| w.contains("per-corner")), "{warnings:?}"); + // Mixed radii: not representable, roundness 0 exported with a warning. + let (p, cid) = shape([10.0, 20.0, 10.0, 20.0], 5.0, true); + let (_, _, warnings, json) = roundtrip(&p, cid, &ExportOptions::default()); + assert_eq!(rc(&json)["r"], json!(0)); + assert!(warnings.iter().any(|w| w.contains("per-corner roundness")), "{warnings:?}"); + // The checkbox off exports the single Roundness and ignores the four values. + let (p, cid) = shape([10.0, 20.0, 10.0, 20.0], 5.0, false); + let (_, _, warnings, json) = roundtrip(&p, cid, &ExportOptions::default()); + assert_eq!(rc(&json)["r"]["k"], json!(5.0)); + assert!(!warnings.iter().any(|w| w.contains("per-corner")), "{warnings:?}"); +} diff --git a/crates/path/src/lib.rs b/crates/path/src/lib.rs index 83e2102dc..61fd0e6c3 100644 --- a/crates/path/src/lib.rs +++ b/crates/path/src/lib.rs @@ -164,6 +164,52 @@ pub fn rect(size: [f64; 2], pos: [f64; 2], roundness: f64) -> BezPath { p } +/// Rectangle path centred at `pos` with independent corner radii `[top-left, top-right, +/// bottom-right, bottom-left]`, clockwise in screen coordinates. Negative and non-finite radii +/// are 0; radii that would overlap are scaled down as CSS `border-radius` does. +pub fn rect_radii(size: [f64; 2], pos: [f64; 2], radii: [f64; 4]) -> BezPath { + // Start at the end of the top-right arc on the right edge, like `rect`, and walk the corners + // clockwise: Trim Paths measures from the first point, so ticking per-corner roundness must + // not make a trim jump (#1896). + let w = if size[0].is_finite() { size[0].abs() } else { 0.0 }; + let h = if size[1].is_finite() { size[1].abs() } else { 0.0 }; + let (x0, y0) = (pos[0] - w / 2.0, pos[1] - h / 2.0); + let (x1, y1) = (pos[0] + w / 2.0, pos[1] + h / 2.0); + let mut r = radii.map(|v| if v.is_finite() { v.max(0.0) } else { 0.0 }); + // CSS border-radius: when two radii on an edge overlap, scale every radius by the smallest + // edge ratio (a uniform radius then clamps to half the shorter side, as before). + let ratio = |a: f64, b: f64, edge: f64| if a + b > edge && a + b > 0.0 { (edge / (a + b)).min(1.0) } else { 1.0 }; + let f = ratio(r[0], r[1], w).min(ratio(r[3], r[2], w)).min(ratio(r[0], r[3], h)).min(ratio(r[1], r[2], h)); + if f < 1.0 { + for v in &mut r { + *v *= f; + } + } + let [tl, tr, br, bl] = r; + // Quarter-circle Bezier control distance. + const K: f64 = 0.552_284_749_830_793_6; + let mut p = BezPath::new(); + p.move_to((x1, y0 + tr)); + p.line_to((x1, y1 - br)); + if br > 0.0 { + p.curve_to((x1, y1 - br + K * br), (x1 - br + K * br, y1), (x1 - br, y1)); + } + p.line_to((x0 + bl, y1)); + if bl > 0.0 { + p.curve_to((x0 + bl - K * bl, y1), (x0, y1 - bl + K * bl), (x0, y1 - bl)); + } + p.line_to((x0, y0 + tl)); + if tl > 0.0 { + p.curve_to((x0, y0 + tl - K * tl), (x0 + tl - K * tl, y0), (x0 + tl, y0)); + } + p.line_to((x1 - tr, y0)); + if tr > 0.0 { + p.curve_to((x1 - tr + K * tr, y0), (x1, y0 + tr - K * tr), (x1, y0 + tr)); + } + p.close_path(); + p +} + pub fn ellipse(size: [f64; 2], pos: [f64; 2]) -> BezPath { to_kurbo(&ShapePath::ellipse(pos, size[0], size[1])) } @@ -419,6 +465,11 @@ mod tests { let round = ops::trim(&[rect([400.0, 300.0], [0.0, 0.0], 40.0)], 0.0, 25.0, 0.0); let (a, b) = ends(&round[0]); assert!(near(a, (200.0, -110.0)) && near(b, (110.0, 150.0)), "{a:?} {b:?}"); + // Per-corner roundness starts and runs in the same order: ticking the checkbox must not + // move a Trim Paths contour. + let corners = ops::trim(&[rect_radii([400.0, 300.0], [0.0, 0.0], [40.0; 4])], 0.0, 25.0, 0.0); + let (a, b) = ends(&corners[0]); + assert!(near(a, (200.0, -110.0)) && near(b, (110.0, 150.0)), "per-corner: {a:?} {b:?}"); // Start > End: AE draws 30..70 % (plain: from x 80 on the bottom up to y −130 on the left). let swapped = ops::trim(&[rect([400.0, 300.0], [0.0, 0.0], 0.0)], 70.0, 30.0, 0.0); let (a, b) = ends(&swapped[0]); @@ -456,6 +507,32 @@ mod tests { assert!((area - 200.0).abs() < 1.0, "{area}"); } + #[test] + fn rect_radii_uniform_matches_a_rounded_rect() { + for r in [0.0, 5.0, 25.0, 100.0] { + let p = rect_radii([100.0, 50.0], [10.0, 20.0], [r; 4]); + let clamped = r.min(25.0); + let expect = 100.0 * 50.0 - (4.0 - std::f64::consts::PI) * clamped * clamped; + assert!((p.area().abs() - expect).abs() < 1.0, "{r}: {} != {expect}", p.area()); + assert!((p.area() - rect([100.0, 50.0], [10.0, 20.0], r).area()).abs() < 1e-6); + } + } + + #[test] + fn rect_radii_mixes_corners_and_scales_overlaps() { + // Only the top-left and bottom-right corners round. + let p = rect_radii([100.0, 50.0], [0.0, 0.0], [10.0, 0.0, 20.0, 0.0]); + let expect = 100.0 * 50.0 - (1.0 - std::f64::consts::PI / 4.0) * (10.0 * 10.0 + 20.0 * 20.0); + assert!((p.area().abs() - expect).abs() < 1.0, "{}", p.area()); + // Radii wider than the top edge scale down together, like CSS border-radius. + let p = rect_radii([50.0, 100.0], [0.0, 0.0], [40.0, 40.0, 0.0, 0.0]); + let scaled = rect_radii([50.0, 100.0], [0.0, 0.0], [25.0, 25.0, 0.0, 0.0]); + assert!((p.area() - scaled.area()).abs() < 1e-9); + // Negative and non-finite radii are square corners. + let p = rect_radii([50.0, 50.0], [0.0, 0.0], [-5.0, f64::NAN, f64::INFINITY, 0.0]); + assert!((p.area().abs() - 2500.0).abs() < 1e-9); + } + #[test] fn stroke_has_area() { let r = rect([40.0, 40.0], [50.0, 50.0], 0.0); diff --git a/crates/project/src/build.rs b/crates/project/src/build.rs index 3ff61ba7d..1aa8d7c90 100644 --- a/crates/project/src/build.rs +++ b/crates/project/src/build.rs @@ -385,11 +385,18 @@ pub fn shape_group(ids: &mut Ids, name: &str, items: Vec) -> PropGrou } pub fn shape_rect(ids: &mut Ids, size: [f64; 2], position: [f64; 2], roundness: f64) -> PropGroup { + // Per-corner radii start from the uniform Roundness, so ticking the box keeps the shape. + let r = if roundness.is_finite() { roundness.max(0.0) } else { 0.0 }; indexed(ids.group("rect", "Rectangle Path 1")) .with(ids.prop("direction", "Path Direction", Value::Enum(0)).with_ui(ParamUi::Hidden)) .with(ids.prop("size", "Size", Value::Vec2(size)).with_ui(ParamUi::Pixels)) .with(ids.prop("position", "Position", Value::Vec2(position)).with_ui(ParamUi::Point)) .with(ids.prop("roundness", "Roundness", Value::Scalar(roundness)).with_ui(ParamUi::Pixels)) + .with(ids.prop("perCorner", "Per-Corner Roundness", Value::Bool(false)).with_ui(ParamUi::Checkbox)) + .with(ids.prop("topLeftRoundness", "Top Left Roundness", Value::Scalar(r)).with_ui(ParamUi::Pixels)) + .with(ids.prop("topRightRoundness", "Top Right Roundness", Value::Scalar(r)).with_ui(ParamUi::Pixels)) + .with(ids.prop("bottomRightRoundness", "Bottom Right Roundness", Value::Scalar(r)).with_ui(ParamUi::Pixels)) + .with(ids.prop("bottomLeftRoundness", "Bottom Left Roundness", Value::Scalar(r)).with_ui(ParamUi::Pixels)) } pub fn shape_ellipse(ids: &mut Ids, size: [f64; 2], position: [f64; 2]) -> PropGroup { diff --git a/crates/render/src/shapes.rs b/crates/render/src/shapes.rs index bbc4283ff..2497ea807 100644 --- a/crates/render/src/shapes.rs +++ b/crates/render/src/shapes.rs @@ -269,7 +269,20 @@ fn collect_in(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup, arena: &mut Ve "rect" => { let size = ctx.v2(layer, g, "size", [100.0; 2]); let pos = ctx.v2(layer, g, "position", [0.0; 2]); - let p = effectcraft_path::rect(size, pos, ctx.f(layer, g, "roundness", 0.0)); + let p = if ctx.b(layer, g, "perCorner") { + effectcraft_path::rect_radii( + size, + pos, + [ + ctx.f(layer, g, "topLeftRoundness", 0.0), + ctx.f(layer, g, "topRightRoundness", 0.0), + ctx.f(layer, g, "bottomRightRoundness", 0.0), + ctx.f(layer, g, "bottomLeftRoundness", 0.0), + ], + ) + } else { + effectcraft_path::rect(size, pos, ctx.f(layer, g, "roundness", 0.0)) + }; push_shape(ctx, layer, g, p, arena, &mut live); } "ellipse" => { diff --git a/crates/render/src/shapes/tests.rs b/crates/render/src/shapes/tests.rs index e0c9086a7..ba7028a25 100644 --- a/crates/render/src/shapes/tests.rs +++ b/crates/render/src/shapes/tests.rs @@ -715,3 +715,33 @@ fn miter_corner_still_fits_its_buffer() { let b = shape_buf(&p, cid); assert!(b.img.height as f64 >= 50.0 + 20.0 * 4.0 * 0.5, "{}", b.img.height); } + +#[test] +fn rect_per_corner_roundness_rounds_the_named_corners() { + let render = |per_corner: bool, radii: [f64; 4], roundness: f64| { + let (p, cid) = shape_comp_sized(200, 100, move |ids| { + let mut g = build::shape_rect(ids, [100.0, 100.0], [0.0, 0.0], roundness); + g.prop_mut("perCorner").unwrap().value = Value::Bool(per_corner); + for (m, v) in + [("topLeftRoundness", radii[0]), ("topRightRoundness", radii[1]), ("bottomRightRoundness", radii[2]), ("bottomLeftRoundness", radii[3])] + { + g.prop_mut(m).unwrap().value = Value::Scalar(v); + } + vec![g, build::shape_fill(ids, RED)] + }); + frame(&p, cid) + }; + // Only the top-left corner rounds: its corner pixels fade, the other three stay filled. + let img = render(true, [30.0, 0.0, 0.0, 0.0], 0.0); + assert!(alpha(&img, 52, 2) < 0.1, "top-left rounds: {}", alpha(&img, 52, 2)); + for (x, y) in [(148, 2), (148, 98), (52, 98)] { + assert!(alpha(&img, x, y) > 0.9, "({x},{y}) stays square: {}", alpha(&img, x, y)); + } + // The checkbox off ignores the four values (the single Roundness still rules). + let off = render(false, [30.0; 4], 0.0); + assert!((coverage(&off) - 100.0 * 100.0).abs() < 20.0, "{}", coverage(&off)); + // Four equal per-corner radii render like the same Roundness. + let per = render(true, [25.0; 4], 0.0); + let uni = render(false, [0.0; 4], 25.0); + assert!((coverage(&per) - coverage(&uni)).abs() < 5.0, "{} vs {}", coverage(&per), coverage(&uni)); +} diff --git a/crates/ui-egui/src/panels/timeline/tests.rs b/crates/ui-egui/src/panels/timeline/tests.rs index 12259abf7..9a5c89f7d 100644 --- a/crates/ui-egui/src/panels/timeline/tests.rs +++ b/crates/ui-egui/src/panels/timeline/tests.rs @@ -195,6 +195,11 @@ fn shape_group_inner_contents_is_elided() { " Size", " Position", " Roundness", + " Per-Corner Roundness", + " Top Left Roundness", + " Top Right Roundness", + " Bottom Right Roundness", + " Bottom Left Roundness", " >Fill 1", " Composite", " Fill Rule", diff --git a/crates/ui-egui/tests/it/ui_timeline_shapes.rs b/crates/ui-egui/tests/it/ui_timeline_shapes.rs index 73686ee99..cd56db825 100644 --- a/crates/ui-egui/tests/it/ui_timeline_shapes.rs +++ b/crates/ui-egui/tests/it/ui_timeline_shapes.rs @@ -340,6 +340,11 @@ fn timeline_colour_eyedropper_samples_the_comp() { let fill = fill.expect("a fill colour"); fn groups(g: &effectcraft_engine::project::PropGroup, out: &mut Vec) { for c in g.groups() { + // The rectangle's path rows would push Fill below the rows the Timeline fits; the + // eyedropper only needs Fill, so leave that group collapsed. + if c.name == "Rectangle Path 1" { + continue; + } out.push(c.uid); groups(c, out); } From d249f255ae7852e23f285a473640b9720cf1819e Mon Sep 17 00:00:00 2001 From: Camille Moine Date: Mon, 19 Oct 2026 21:08:54 +0200 Subject: [PATCH 29/33] Shapes: export per-corner `rc.r` as a Lottie property The equal-radii and zero fallback branches wrote a bare number where the uniform branch (and every other Value field) writes `{a, k}`; external players may reject that form even though our importer reads it. Write `{"a": 0, "k": value}` in both branches and update the export assertions. --- crates/lottie/src/shapes.rs | 5 +++-- crates/lottie/src/tests.rs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/lottie/src/shapes.rs b/crates/lottie/src/shapes.rs index fc823b775..5b9d29970 100644 --- a/crates/lottie/src/shapes.rs +++ b/crates/lottie/src/shapes.rs @@ -239,11 +239,12 @@ pub(crate) fn export_items(ex: &mut Ex, contents: &PropGroup, label: &str) -> Ve let names = ["topLeftRoundness", "topRightRoundness", "bottomRightRoundness", "bottomLeftRoundness"]; let radii = names.map(|m| o.f(m)); // Lottie `rc` has one `r`: four equal static radii export as that value. + // `r` is a property (`{a, k}`) like the uniform branch writes, not a bare number. if radii.iter().all(|v| (v - radii[0]).abs() < 1e-6) && !names.iter().any(|m| o.animated(m)) && radii[0] >= 0.0 { - o.raw("r", json!(r(radii[0]))); + o.raw("r", json!({"a": 0, "k": r(radii[0])})); } else { o.ex.warn(format!("{}: per-corner roundness has no Lottie rectangle equivalent (roundness 0 exported)", o.label)); - o.raw("r", json!(0)); + o.raw("r", json!({"a": 0, "k": 0})); } } else { o.put("r", "roundness", &anim::num); diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 24b4690eb..83c81e242 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -919,12 +919,12 @@ fn per_corner_roundness_exports_equal_radii_and_warns_when_mixed() { // Four equal static radii: `rc.r` carries the value, no warning. let (p, cid) = shape([12.0; 4], 5.0, true); let (_, _, warnings, json) = roundtrip(&p, cid, &ExportOptions::default()); - assert_eq!(rc(&json)["r"], json!(12.0)); + assert_eq!(rc(&json)["r"], json!({"a": 0, "k": 12.0})); assert!(!warnings.iter().any(|w| w.contains("per-corner")), "{warnings:?}"); // Mixed radii: not representable, roundness 0 exported with a warning. let (p, cid) = shape([10.0, 20.0, 10.0, 20.0], 5.0, true); let (_, _, warnings, json) = roundtrip(&p, cid, &ExportOptions::default()); - assert_eq!(rc(&json)["r"], json!(0)); + assert_eq!(rc(&json)["r"], json!({"a": 0, "k": 0})); assert!(warnings.iter().any(|w| w.contains("per-corner roundness")), "{warnings:?}"); // The checkbox off exports the single Roundness and ignores the four values. let (p, cid) = shape([10.0, 20.0, 10.0, 20.0], 5.0, false); From 5bb66bad745fcec1f68e0f6a28a7697ba596aec1 Mon Sep 17 00:00:00 2001 From: bflatastic Date: Fri, 30 Oct 2026 07:37:19 +0700 Subject: [PATCH 30/33] ScriptUI: a docked panel gets its dock's size and onResize (#2585) A docked ScriptUI panel only ever saw its preferredSize: the host drew it in a scroll area at max(dock, preferred) and the script's window never learned the dock. scriptui.resize now gives a window a size: the script's window takes it, lays itself out again and runs onResizing / onResize. The dock host sends it whenever the panel's dock size changes, so window.size is the dock's, as in After Effects. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/scripts.rs | 9 ++++++ crates/engine/src/scriptui.rs | 11 ++++++++ crates/script/src/scriptui.js | 12 ++++++++ crates/ui-egui/src/panels/scriptui_view.rs | 11 ++++++++ crates/ui-egui/tests/it/ui_scriptui.rs | 32 ++++++++++++++++++++++ 5 files changed, 75 insertions(+) diff --git a/crates/engine/src/cmd/scripts.rs b/crates/engine/src/cmd/scripts.rs index a523770e3..a92d484a7 100644 --- a/crates/engine/src/cmd/scripts.rs +++ b/crates/engine/src/cmd/scripts.rs @@ -467,6 +467,15 @@ pub fn specs() -> Vec { always, scriptui::mouse ), + cmd!( + "scriptui.resize", + "Resize Script Window", + [], + None, + "{window?, size: [width, height] (pixels)} → the window takes the size, lays itself out again and its onResizing / onResize run (a docked ScriptUI panel gets its dock's size)", + always, + scriptui::resize + ), cmd!( "scriptui.close", "Close Script Window", diff --git a/crates/engine/src/scriptui.rs b/crates/engine/src/scriptui.rs index af89932fa..d8f9f1d1e 100644 --- a/crates/engine/src/scriptui.rs +++ b/crates/engine/src/scriptui.rs @@ -765,6 +765,17 @@ pub(crate) fn mouse(s: &mut Session, p: &Value) -> Result { dispatch(s, ScriptUiEvent { window, widget, kind: (*ty).to_string(), value }) } +/// `scriptui.resize`: the host gave the window a new size (a docked panel's dock, #2585): the +/// script's window takes it, lays itself out again and gets `onResizing` / `onResize`. +pub(crate) fn resize(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.resize"; + let window = window_p(s, p, c)?; + let size = p.get("size").and_then(Value::as_array).ok_or_else(|| crate::commands::bad(c, "missing `size` [width, height]"))?; + let side = |i: usize| size.get(i).and_then(Value::as_f64).filter(|v| v.is_finite()).map(|v| v.round().clamp(1.0, 16384.0)); + let (Some(w), Some(h)) = (side(0), side(1)) else { return Err(crate::commands::bad(c, "`size` must be [width, height] in pixels")) }; + dispatch(s, ScriptUiEvent { window, widget: 0, kind: "resize".into(), value: json!([w, h]) }) +} + /// `scriptui.close`: close a window (a dialog's `show()` returns `result`, default 2 = Cancel). pub(crate) fn close(s: &mut Session, p: &Value) -> Result { let window = window_p(s, p, "scriptui.close")?; diff --git a/crates/script/src/scriptui.js b/crates/script/src/scriptui.js index 178bd8be1..ddfbc1f3e 100644 --- a/crates/script/src/scriptui.js +++ b/crates/script/src/scriptui.js @@ -912,6 +912,18 @@ function __uiDispatch(winId, id, kind, valueJson) { w.close(v === null ? 2 : v); return; } + // The host gave the window a new size (a docked panel's dock): take it, lay out, tell the script. + if (kind === "resize") { + var nw = Number(v && v[0]), nh = Number(v && v[1]); + var b = w.__ub || w.__lb; + if (nw > 0 && nh > 0 && (b[2] - b[0] !== nw || b[3] - b[1] !== nh)) { + w.__ub = [b[0], b[1], b[0] + nw, b[1] + nh]; + __uiLayout(w); + w.__fire("onResizing"); + w.__fire("onResize"); + } + return; + } var c = __uiFind(w, id); if (!c || c.enabled === false) return; var radio = function () { diff --git a/crates/ui-egui/src/panels/scriptui_view.rs b/crates/ui-egui/src/panels/scriptui_view.rs index 14460ed28..753b2884e 100644 --- a/crates/ui-egui/src/panels/scriptui_view.rs +++ b/crates/ui-egui/src/panels/scriptui_view.rs @@ -26,6 +26,8 @@ enum Act { Close(u32), /// A mouse event (`scriptui.mouse` parameters without window and widget). Mouse(u32, u32, Value), + /// A docked panel's new size (`scriptui.resize`). + Resize(u32, [f32; 2]), } fn rect_of(origin: egui::Pos2, b: [f64; 4]) -> Rect { @@ -500,6 +502,7 @@ fn perform(app: &mut EffectcraftApp, ctx: &egui::Context, acts: Vec) { Act::Set(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v})), Act::Changing(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v, "changing": true})), Act::Close(w) => ("scriptui.close", json!({"window": w})), + Act::Resize(w, [x, y]) => ("scriptui.resize", json!({"window": w, "size": [x, y]})), Act::Mouse(w, id, mut v) => { if let Some(o) = v.as_object_mut() { o.insert("window".into(), json!(w)); @@ -582,6 +585,14 @@ pub fn panel(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, id: u32) { // A modal script dialog blocks the panels too. let modal = app.session.script_ui.windows.iter().any(|w| w.modal); let mut acts = vec![]; + // The script learns its dock's size when it changes (#2585), as After Effects tells a + // docked panel: its window takes the size and onResize runs. + let dock = [rect.width().round(), rect.height().round()]; + let told = egui::Id::unique(("scriptui-dock-size", id)); + if ui.data(|d| d.get_temp::<[f32; 2]>(told)) != Some(dock) && dock[0] >= 1.0 && dock[1] >= 1.0 { + ui.data_mut(|d| d.insert_temp(told, dock)); + acts.push(Act::Resize(id, dock)); + } let content = rect; egui::ScrollArea::both().id_salt(("scriptui-panel", id)).max_width(content.width()).max_height(content.height()).show(ui, |ui| { let (r, _) = ui.allocate_exact_size(vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32), Sense::hover()); diff --git a/crates/ui-egui/tests/it/ui_scriptui.rs b/crates/ui-egui/tests/it/ui_scriptui.rs index 2669a2631..8a248155b 100644 --- a/crates/ui-egui/tests/it/ui_scriptui.rs +++ b/crates/ui-egui/tests/it/ui_scriptui.rs @@ -305,6 +305,38 @@ fn a_drag_keeps_its_control_past_the_window_edge() { h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); } +/// #2585: a window the host resizes takes the size, lays out again and runs onResize; a docked +/// ScriptUI panel gets its dock's size that way. +#[test] +fn script_windows_learn_the_size_they_are_given() { + let mut h = harness(); + let code = r#" + var w = new Window("palette", "Sized"); + var log = w.add("statictext", undefined, "", { name: "log" }); + log.preferredSize = [200, 20]; + w.onResize = function () { log.text = "resized " + w.size.width + "x" + w.size.height; }; + w.show(); + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "sized.jsx"})).unwrap(); + assert_eq!(r["ok"], true, "{r}"); + h.run_steps(2); + h.state_mut().session.execute("scriptui.resize", json!({"window": "Sized", "size": [300, 180]})).unwrap(); + let win = h.state().session.script_ui.windows[0].clone(); + assert_eq!(win.root.lookup(&json!("log")).unwrap().text, "resized 300x180"); + let list = h.state_mut().session.execute("scriptui.list", json!({})).unwrap(); + assert_eq!(list[0]["size"], json!([300.0, 180.0]), "{list}"); + assert!(h.state_mut().session.execute("scriptui.resize", json!({"window": "Sized", "size": ["wide", 1]})).is_err()); + h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); + // A docked panel: its window is its dock's size, not its preferred size. + h.state_mut().session.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + h.run_steps(4); + let panel = h.state().session.script_ui.windows.iter().find(|w| w.script == "Layer Tools.jsx").unwrap().clone(); + let dock = rect(&h, &format!("scriptui.{}", panel.id)); + assert_eq!([panel.root.bounds[2].round(), panel.root.bounds[3].round()], [dock.width().round() as f64, dock.height().round() as f64]); + h.state_mut().close_panel(PanelKind::ScriptPanel(panel.id)); + h.run_steps(2); +} + fn click_at(h: &mut Harness<'_, EffectcraftApp>, c: egui::Pos2) { h.input_mut().events.push(Event::PointerMoved(c)); h.run_steps(1); From df21aff0d80ab36583cb3cae3a5cf447902cb95d Mon Sep 17 00:00:00 2001 From: bflatastic Date: Fri, 30 Oct 2026 07:46:50 +0700 Subject: [PATCH 31/33] Docs: ROADMAP progress log and ScriptUI parity for 30 October's features Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 11 ++++++++++- docs/scripting-parity.md | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index f3e6750ed..740ec6010 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~30 points of "ready for real work" and ~350–620 h away -> **Last reviewed:** 2026-10-22 · **Last updated:** 2026-10-30 · **Change:** minor (localization: 303 more panel/dialog rows in Japanese and the Chinese catalogs; recounted after #1774: Japanese 27% → 46%, Simplified Chinese 29% → 46%, Traditional Chinese 29% → 38%, overall 18% → 24%) · **Target:** Adobe After Effects 2026 (26.5.0.89) +> **Last reviewed:** 2026-10-22 · **Last updated:** 2026-10-30 · **Change:** minor (features of 30 October in the progress log: text Line Join, Timeline drag-reorder and multi-select of effects, viewer edge pan, ScriptUI drag capture and dock size; previously: localization recount after #1774) · **Target:** Adobe After Effects 2026 (26.5.0.89) EffectCraft aims to do what After Effects does, with the same panels, menus and behaviour, written from scratch in Rust. This page is the one-page summary; the evidence is in @@ -117,6 +117,14 @@ Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). ## Progress log +- 2026-10-30: Text strokes take a Line Join (Miter, Round or Bevel) in the Character panel, + Miter for new text as in After Effects (#2077). In the Timeline, effects, masks and shape items + drag to a new place in their stack (#2210) and Shift-click adds them to the selection, so one + animation preset can hold several (#2552). The Composition panel pans while a mask vertex is + held at its edge (#999). Effect buffers that pad a small layer stay within a memory budget + (#2261); an MP4 lasts as long as its edit list shows (#2592); the Render Queue's times tick + between frames (#2593). ScriptUI drags keep their control past the window edge with + `MouseEvent.buttons` (#2584), and docked panels get their dock's size and `onResize` (#2585). - 2026-10-28: A shape layer's Rectangle and Ellipse Size get the Constrain Proportions chain that Scale has, so a circle stays a circle while its size is typed, scrubbed or keyed; it starts off, so existing shapes behave as before (#2489). @@ -169,6 +177,7 @@ Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). | Date | Change | Summary | |---|---|---| +| 2026-10-30 | minor | Progress log: text Line Join, Timeline drag-reorder and Shift multi-select of effects, viewer edge pan, effect padding budget, MP4 edit-list duration, Render Queue clock, ScriptUI drag capture and docked-panel size | | 2026-10-22 | minor | Localization: 300 more panel/dialog rows in Japanese and the Chinese catalogs, plus the review's three missing rows (Basic, Advanced, Play/Stop) — panel tabs, form and dialog labels, context menus, tracker/graph/text-panel controls, File-panel footers, credits; language shares recounted after #1774 (Japanese 27% → 46% with 297 effect names, Simplified Chinese 29% → 46% with 307 effect names, Traditional Chinese 29% → 38%, overall 18% → 24%) | | 2026-10-12 | minor | Open-issue count (134 to 216), crash list and localization share (12% to 18%) updated to match docs/gaps.md | | 2026-10-12 | patch | Roto Brush span validation after trimming; command regressions for both layer edges | diff --git a/docs/scripting-parity.md b/docs/scripting-parity.md index 8d2934518..96e298ece 100644 --- a/docs/scripting-parity.md +++ b/docs/scripting-parity.md @@ -20,7 +20,7 @@ our own scripts; no real-world script library has been run against both apps. | Expression performance | Cached, multi-frame | Per-thread boa contexts | unknown | No benchmark | | Scripting object model | `app`, `Project`, items, layers, properties, keys, eases, render queue, `TextDocument`, `Shape`, markers, `File`/`Folder`, `Socket`, `app.settings` | Same classes, edits run through undoable engine commands; match names table; layers, groups and properties are callable (`layer("Effects")(1)` is `.property(...)`) | good | `crates/script` tests (≈ 10 test files) | | `$.evalFile`, `BridgeTalk` | Evaluate another script file; identify the host application and message other Adobe applications | `$.evalFile` evaluates a file relative to the calling file (nested at most 32 deep, under the scripts' file gate) and returns its last value; `BridgeTalk` names the host honestly (`appName` `effectcraft`, its version); messaging other applications throws "not supported" | partial | `tests_evalfile.rs` (#760); no inter-application messaging | -| ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics, mouse events | Same, dockable panels reopen at launch | good | `ui_scriptui`, `tests_ui*` | +| ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics, mouse events | Same, dockable panels reopen at launch; a docked panel gets its dock's size and `onResize` (`scriptui.resize`, #2585); a drag keeps reaching the control it started on past the window edge, and `MouseEvent.buttons` reports the held buttons (#2584) | good | `ui_scriptui`, `tests_ui*` | | `app.scheduleTask` / `cancelTask` | One-shot and repeating tasks (polling, panel refresh) until `cancelTask` | Same: repeating tasks run every `delay` ms (at least 50 ms, at most 64 pending per script) until `cancelTask`, until they throw (reported once, then cancelled), or until their owner goes: a script's last window closing, or the project closing for scripts without windows (a Startup script's tasks go on until EffectCraft quits); frontends tick them every frame, agents with `script.runTasks` | good | `tests_tasks.rs` (#773); one-shot tasks of a script file still run right after it, not after their delay | | Scripts menu, ScriptUI Panels folder | yes | yes; a subfolder (one level) of either folder is a submenu named after it | same | [plugins.md](plugins.md) | | Startup / Shutdown folders | yes | `Scripts/Startup` runs at launch and `Scripts/Shutdown` runs as the desktop app quits, after the project is closed, each in alphabetical order | partial | `crates/script/src/tests/startup.rs`; each script has its own global scope, so a Startup script's variables and functions are not available to later scripts ([plugins.md](plugins.md)) | From 1158022585666a11d046351cc5ca448509d83c72 Mon Sep 17 00:00:00 2001 From: bflatastic Date: Fri, 30 Oct 2026 07:53:12 +0700 Subject: [PATCH 32/33] Settings: a dragged UI Scale applies on release (#2582) Dragging an integer setting applied every step, so UI Scale resized the interface under the pointer mid-drag. A dragged value now shows in its field and applies when the button is released (a typed one already waited for Enter). Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/panels/settings.rs | 13 ++++++++---- crates/ui-egui/tests/it/ui_panels.rs | 30 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index f4dd89bfc..562a90132 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -117,15 +117,20 @@ fn page_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, page: &Page, cur: &Value ui.add_sized(vec2(label_w, 18.0), egui::Label::new(format!("{}:", tr(label))).truncate()); let r = match kind { Kind::Int(lo, hi, unit) => { - let mut x = v.as_i64().unwrap_or(0); - // A typed number applies once entered: typing 85 into UI Scale - // doesn't resize the interface to 75 % at its first digit. + // A dragged value applies on release, and a typed one once + // entered: UI Scale doesn't resize the interface under the + // pointer mid-drag (#2582) or at a typed number's first digit. + let held = egui::Id::unique(("settings-drag", key)); + let mut x = ui.data(|d| d.get_temp::(held)).unwrap_or_else(|| v.as_i64().unwrap_or(0)); let drag = egui::DragValue::new(&mut x).range(lo..=hi).speed(0.25).update_while_editing(false); let r = ui.add(crate::widgets::drag_value(drag)); if !unit.is_empty() { ui.label(RichText::new(tr(unit)).color(t.text_dim)); } - if r.changed() { + if r.dragged() { + ui.data_mut(|d| d.insert_temp(held, x)); + } else if r.changed() || r.drag_stopped() { + ui.data_mut(|d| d.remove::(held)); acts.push(Act::Set(key.into(), json!(x))); } r diff --git a/crates/ui-egui/tests/it/ui_panels.rs b/crates/ui-egui/tests/it/ui_panels.rs index db9ef7373..6d06c6bf7 100644 --- a/crates/ui-egui/tests/it/ui_panels.rs +++ b/crates/ui-egui/tests/it/ui_panels.rs @@ -664,3 +664,33 @@ fn transition_presets_apply_from_effects_and_presets() { assert_eq!(fx.name, "Radial Wipe"); assert_eq!(fx.prop("completion").unwrap().keys.len(), 2); } + +/// #2582: dragging Settings' UI Scale doesn't resize the interface under the pointer; the value +/// applies once the button is released. +#[test] +fn ui_scale_applies_when_the_drag_ends() { + let mut h = harness(); + open_settings(&mut h); + let e = h.state().auto.find("settings.appearance.uiScale").expect("the UI Scale field"); + let from = pos2(e.rect[0] + e.rect[2] / 2.0, e.rect[1] + e.rect[3] / 2.0); + h.input_mut().events.push(Event::PointerMoved(from)); + h.input_mut().events.push(Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for k in 1..=8 { + h.input_mut().events.push(Event::PointerMoved(from + egui::vec2(10.0 * k as f32, 0.0))); + h.step(); + assert_eq!(scale(&h), (100, 1.0), "nothing applies mid-drag"); + } + h.input_mut().events.push(Event::PointerButton { + pos: from + egui::vec2(80.0, 0.0), + button: egui::PointerButton::Primary, + pressed: false, + modifiers: Default::default(), + }); + h.run_steps(4); + let (pct, zoom) = scale(&h); + assert!(pct > 100 && (zoom - pct as f32 / 100.0).abs() < 1e-6, "applied on release: {pct} {zoom}"); + click(&mut h, "settings.cancel"); + h.run_steps(2); + assert_eq!(scale(&h), (100, 1.0)); +} From f6222ff5d5c53dc2847c47e61774b10ce4567533 Mon Sep 17 00:00:00 2001 From: erythrofr <20369086+erythrofr@users.noreply.github.com> Date: Fri, 30 Oct 2026 02:47:31 +0100 Subject: [PATCH 33/33] Tests: document crop/resize regressions Describe the dimension, pixel-preservation and extreme-input scenarios and their synthetic-frame helper to address the review's documentation warning. Test logic and the crop/resize implementation are unchanged. AI assistance: OpenAI Codex (GPT-6). --- crates/export/src/pipeline.rs | 4 ++++ crates/export/tests/it/render_settings.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 7b9e4111c..8e8c90561 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -1063,6 +1063,7 @@ mod tests { } } + /// Regress the locked-aspect example: cropping 1920x1080 must resize to 1280x1440. #[test] fn crop_resize_applies_margins_once() { use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; @@ -1077,6 +1078,7 @@ mod tests { assert_eq!(img.data.len(), 1280 * 1440); } + /// Apply output-module geometry to a synthetic frame without rendering or writing files. fn run_crop_resize(img: Image, output: &effectcraft_project::render_queue::OutputModule, scale: f64) -> Image { use effectcraft_project::render_queue::RenderSettings; let project = Project::default(); @@ -1097,6 +1099,7 @@ mod tests { Cx::new(&job).crop_resize(img) } + /// Match predicted dimensions across crop, padding, ROI scale, aspect locks and resize qualities. #[test] fn crop_resize_dimensions_match_the_output_module() { use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; @@ -1124,6 +1127,7 @@ mod tests { } } + /// Exercise zero/empty inputs, collapsed crops and existing caps on extreme single-axis resizes. #[test] fn crop_resize_zero_and_extreme_sizes_stay_bounded() { use effectcraft_project::render_queue::{Crop, OutputModule, Resize}; diff --git a/crates/export/tests/it/render_settings.rs b/crates/export/tests/it/render_settings.rs index c2d7dcbfa..0fd6d471b 100644 --- a/crates/export/tests/it/render_settings.rs +++ b/crates/export/tests/it/render_settings.rs @@ -174,6 +174,8 @@ fn crop_roi_and_resize() { assert_eq!((z.width(), z.height()), (32, 24)); } +/// Resizing to the cropped frame's own dimensions must preserve every decoded PNG pixel. +/// Compare against crop-only output for padding and scaled ROIs with both resize qualities. #[test] fn crop_resize_preserves_pixels_when_the_cropped_size_is_requested() { let (p, cid) = project(FrameRate::new(25, 1), |_, _| {});