From e7c208434b6a966f67731aa98abe625651f52778 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:27:45 +0100 Subject: [PATCH 01/35] #474: pin OpenEXR pictures and aux channels to exact hashes Synthetic single-layer, multi-layer, multi-part and windowed files in half, float and uint, with RLE, ZIP, PIZ and DWAA, decoded through the media pool picture first and channels first. The hashes are taken on main before the decode is reworked. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 338 ++++++++++++++++++++++++++++ crates/media/tests/it/main.rs | 1 + 2 files changed, 339 insertions(+) create mode 100644 crates/media/tests/it/exr_golden.rs diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs new file mode 100644 index 000000000..498944882 --- /dev/null +++ b/crates/media/tests/it/exr_golden.rs @@ -0,0 +1,338 @@ +//! OpenEXR pictures and aux channels pinned to exact hashes (#474). +//! +//! Synthetic files (single-layer, multi-layer with and without an unnamed RGB layer, multi-part, +//! data windows smaller and larger than the display window, half / float / uint channels, RLE, +//! ZIP, PIZ and DWAA) are decoded through the media pool as the renderer asks for them: the +//! picture in each alpha interpretation, and the channels EXtractoR and Cryptomatte read. The +//! hashes were taken before the picture and the channels started sharing one decode, so a change +//! to how either is decoded must leave every bit as it was. Only bit-exact float steps are +//! pinned (linear light, no `powf`), so the hashes hold on every platform; DWAA (whose decoder +//! uses `powf`) and the sRGB / gamma 2.4 curves are checked for agreement between the two ways a +//! frame can be decoded instead. Re-pin with `EXR_GOLDEN_PRINT=1` after an intended change. + +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe_bytes}; +use effectcraft_project::{AlphaMode, Footage, ItemId}; +use effectcraft_render::{FloatCurve, FootageSource}; +use effectcraft_time::Tick; +use exr::prelude::*; + +const W: usize = 37; +const H: usize = 23; + +/// A deterministic value for pixel (x, y) of channel `c`: negative, in range and above 1. +fn value(x: usize, y: usize, c: usize) -> f32 { + ((x * 7 + y * 13 + c * 29) % 97) as f32 / 40.0 - 0.3 +} + +/// A deterministic alpha in 0..=1 (0 included, for the premultiplied divide). +fn alpha(x: usize, y: usize) -> f32 { + ((x * 3 + y * 5) % 11) as f32 / 10.0 +} + +#[derive(Clone, Copy)] +enum Kind { + Half, + Float, + Uint, +} + +/// A channel `name` of `size` filled from [`value`] (or [`alpha`] for an `A` / `a` channel). +fn channel(name: &str, kind: Kind, size: (usize, usize), c: usize) -> AnyChannel { + let is_alpha = name.rsplit('.').next().is_some_and(|n| n.eq_ignore_ascii_case("a")); + let px = |i: usize| { + let (x, y) = (i % size.0, i / size.0); + if is_alpha { alpha(x, y) } else { value(x, y, c) } + }; + let n = size.0 * size.1; + let samples = match kind { + Kind::Half => FlatSamples::F16((0..n).map(|i| f16::from_f32(px(i))).collect()), + Kind::Float => FlatSamples::F32((0..n).map(px).collect()), + Kind::Uint => FlatSamples::U32((0..n).map(|i| (i as u32).wrapping_mul(2_654_435_761) >> 8).collect()), + }; + AnyChannel::new(name, samples) +} + +fn channels(list: &[(&str, Kind)], size: (usize, usize)) -> AnyChannels { + AnyChannels::sort(list.iter().enumerate().map(|(c, (n, k))| channel(n, *k, size, c)).collect::>().into()) +} + +fn encoding(compression: Compression) -> Encoding { + Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing } +} + +fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { + attrs.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + attrs.other.insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + attrs +} + +/// One part: `list` channels over a data window of `size` at `pos`. +fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) +} + +/// Encode `layers` with the display window `display` (position, size). +fn write(layers: Vec>>, display: ((i32, i32), (usize, usize))) -> Vec { + let mut attrs = ImageAttributes::new(IntegerBounds::new(display.0, display.1)); + attrs.pixel_aspect = 1.0; + let image = Image::from_layers(attrs, layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() +} + +const RGBA_HALF: [(&str, Kind); 4] = [("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half), ("A", Kind::Half)]; +const CRYPTO: [(&str, Kind); 4] = + [("CryptoObject00.r", Kind::Float), ("CryptoObject00.g", Kind::Float), ("CryptoObject00.b", Kind::Float), ("CryptoObject00.a", Kind::Float)]; + +/// The synthetic files: (name, bytes, whether the hashes are pinned). +fn files() -> Vec<(&'static str, Vec, bool)> { + let full = (W, H); + let origin = ((0, 0), full); + let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; + let layered: Vec<(&str, Kind)> = cat( + &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &CRYPTO, + ); + let unnamed_plus: Vec<(&str, Kind)> = + cat(&cat(&RGBA_HALF, &[("Diffuse.R", Kind::Half), ("Diffuse.G", Kind::Half), ("Diffuse.B", Kind::Half), ("Z", Kind::Float)]), &CRYPTO); + let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; + vec![ + ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), + ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), + ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), + ( + "multipart_zip16", + write( + vec![ + part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), + part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), + ( + // The first part with R, G and B is not the beauty pass. + "multipart_diffuse_then_beauty_zip1", + write( + vec![ + part(Some("diffuse"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], full, (0, 0), Compression::ZIP1), + part(Some("beauty"), &RGBA_HALF, (20, 10), (5, 7), Compression::ZIP1), + part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP1), + ], + origin, + ), + true, + ), + ( + "multipart_blender5_rle", + write( + vec![ + part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), + part( + Some("ViewLayer.Combined"), + &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + full, + (0, 0), + Compression::RLE, + ), + ], + origin, + ), + true, + ), + ("unnamed_rgb_window_smaller_zip16", write(vec![part(None, &unnamed_plus, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("rgb_float_window_larger_piz", write(vec![part(None, &rgb_float, (45, 30), (-1, -5), Compression::PIZ)], ((3, -2), full)), true), + ("layered_window_smaller_zip16", write(vec![part(None, &layered, (20, 10), (5, 7), Compression::ZIP16)], origin), true), + ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), + ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), + ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ] +} + +/// FNV-1a over 32-bit words. +fn fnv(words: impl IntoIterator) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for w in words { + for b in w.to_le_bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0100_0000_01b3); + } + } + h +} + +fn picture_hash(pool: &MediaPool, f: &Footage, float: FloatCurve, size: Option<(u32, u32)>) -> u64 { + let img = pool.frame_with(ItemId(1), f, Tick::ZERO, size, float).expect("picture"); + fnv([img.width, img.height].into_iter().chain(img.data.iter().flat_map(|p| p.map(f32::to_bits)))) +} + +fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { + let Some(aux) = pool.aux(ItemId(1), f, Tick::ZERO) else { return 0 }; + let mut words = vec![aux.width, aux.height, aux.scale.to_bits() as u32]; + for (name, plane) in &aux.channels { + words.extend(name.bytes().map(u32::from)); + words.extend(plane.iter().map(|v| v.to_bits())); + } + for (name, objects) in &aux.manifests { + words.extend(name.bytes().map(u32::from)); + for (o, h) in objects { + words.extend(o.bytes().map(u32::from)); + words.push(*h); + } + } + fnv(words) +} + +/// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). +fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { + let mut out = vec![]; + for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + let mut g = f.clone(); + if let Some(m) = mode { + g.alpha = m; + } + g.premul_color = [0.2, 0.4, 0.6]; + out.push((format!("{label}/linear"), g.clone(), FloatCurve::Linear, None, true)); + out.push((format!("{label}/srgb"), g.clone(), FloatCurve::Srgb, None, false)); + out.push((format!("{label}/gamma24"), g.clone(), FloatCurve::Gamma24, None, false)); + } + let preserve = Footage { preserve_rgb: true, ..f.clone() }; + out.push(("preserve_rgb".into(), preserve, FloatCurve::Srgb, None, true)); + out.push(("half_size/srgb".into(), f.clone(), FloatCurve::Srgb, Some((f.width.div_ceil(2), f.height.div_ceil(2))), false)); + out +} + +/// Every hash of `file`: the size and alpha import finds, each picture and the channels, decoded +/// picture first (the footage alone) or channels first (an effect reads them). +fn hashes(name: &str, bytes: &[u8], aux_first: bool) -> Vec<(String, u64, bool)> { + let order = if aux_first { "aux-first" } else { "picture-first" }; + let path = format!("/exr-golden-474/{order}/{name}.exr"); + let b: Arc<[u8]> = bytes.into(); + let pool = MediaPool::new(); + pool.add_bytes(&path, b.clone()); + let f = probe_bytes(&path, b).expect("probe"); + let mut out = vec![("probe".to_string(), fnv([f.width, f.height, f.alpha as u32]), true)]; + let aux = if aux_first { Some(aux_hash(&pool, &f)) } else { None }; + for (label, g, float, size, pinned) in variants(&f) { + out.push((label, picture_hash(&pool, &g, float, size), pinned)); + } + out.push(("aux".into(), aux.unwrap_or_else(|| aux_hash(&pool, &f)), true)); + out +} + +/// Hashes taken on main before #474's shared decode (`EXR_GOLDEN_PRINT=1`). +const PINNED: &[(&str, &str, u64)] = &[ + ("rgba_half_zip16", "probe", 0x0a32e074a28c1f66), + ("rgba_half_zip16", "probed/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("rgba_half_zip16", "premul/linear", 0xa84600283dce98f6), + ("rgba_half_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("rgba_half_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("rgba_half_zip16", "aux", 0xdd555e6cf67d75f3), + ("rgb_float_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_piz", "probed/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "straight/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "premul/linear", 0x515f476c176229c9), + ("rgb_float_piz", "ignore/linear", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "preserve_rgb", 0xa6e8fbbad371a2ae), + ("rgb_float_piz", "aux", 0x554623d237c2e5ef), + ("layered_rle", "probe", 0x0a32e074a28c1f66), + ("layered_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_zip1", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_zip1", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_zip1", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_zip1", "aux", 0x27cc9b53d0862db1), + ("multipart_zip16", "probe", 0x0a32e074a28c1f66), + ("multipart_zip16", "probed/linear", 0xa84600283dce98f6), + ("multipart_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_zip16", "premul/linear", 0xa84600283dce98f6), + ("multipart_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_zip16", "aux", 0x567ad99a6690d03d), + ("multipart_diffuse_then_beauty_zip1", "probe", 0x6a38346c4c5662d5), + ("multipart_diffuse_then_beauty_zip1", "probed/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "straight/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "premul/linear", 0x7ab18932e25c5cc8), + ("multipart_diffuse_then_beauty_zip1", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "preserve_rgb", 0xa90a57f9a67f7468), + ("multipart_diffuse_then_beauty_zip1", "aux", 0x6724d6dfd56a04cb), + ("multipart_blender5_rle", "probe", 0x0a32e074a28c1f66), + ("multipart_blender5_rle", "probed/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "straight/linear", 0xf028e1367a2fbb30), + ("multipart_blender5_rle", "premul/linear", 0xa84600283dce98f6), + ("multipart_blender5_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("multipart_blender5_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("multipart_blender5_rle", "aux", 0x522ae4819dd92e89), + ("unnamed_rgb_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_window_smaller_zip16", "aux", 0x8f78d3233450acf3), + ("rgb_float_window_larger_piz", "probe", 0x6a38346c4c5662d5), + ("rgb_float_window_larger_piz", "probed/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "straight/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "premul/linear", 0x0998eefc456dcdc9), + ("rgb_float_window_larger_piz", "ignore/linear", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "preserve_rgb", 0x8e3847d48f9967a6), + ("rgb_float_window_larger_piz", "aux", 0x7ccc9f86e6ff6140), + ("layered_window_smaller_zip16", "probe", 0x0a32e074a28c1f66), + ("layered_window_smaller_zip16", "probed/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "straight/linear", 0x2c73f9acaa936420), + ("layered_window_smaller_zip16", "premul/linear", 0x19eb3b1a88d108a7), + ("layered_window_smaller_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("layered_window_smaller_zip16", "preserve_rgb", 0x5b999014e4a64bb2), + ("layered_window_smaller_zip16", "aux", 0x443480a1d789e63c), + ("layered_window_larger_piz", "probe", 0x0a32e074a28c1f66), + ("layered_window_larger_piz", "probed/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "straight/linear", 0xb32b44880f746826), + ("layered_window_larger_piz", "premul/linear", 0x443afe4029748836), + ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), + ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), + ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), +]; + +#[test] +fn exr_pictures_and_channels_are_bit_identical() { + let print = std::env::var_os("EXR_GOLDEN_PRINT").is_some(); + let mut failures = vec![]; + for (name, bytes, pinned_file) in files() { + let first = hashes(name, &bytes, false); + let second = hashes(name, &bytes, true); + for ((label, h, pinned), (_, h2, _)) in first.iter().zip(&second) { + if h != h2 { + failures.push(format!("{name} {label}: picture first {h:#018x}, channels first {h2:#018x}")); + } + if print { + println!(" (\"{name}\", \"{label}\", {h:#018x}),{}", if *pinned && pinned_file { "" } else { " // not pinned" }); + continue; + } + if !(*pinned && pinned_file) { + continue; + } + match PINNED.iter().find(|(n, l, _)| *n == name && l == label) { + Some((_, _, want)) if want == h => {} + Some((_, _, want)) => failures.push(format!("{name} {label}: {h:#018x}, pinned {want:#018x}")), + None => failures.push(format!("{name} {label}: {h:#018x} is not pinned")), + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 69bed4a9f..08c5ec389 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -3,6 +3,7 @@ mod aac_layout; mod downmix; mod exif_orientation; +mod exr_golden; mod footage; mod open_in_place; mod tga; From 684e05859fb4e1ad3b5ba45cc92720ae010ea12e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 01:47:24 +0100 Subject: [PATCH 02/35] EXR export: no compand / tone mapping, and the output space's own curve OpenEXR renders (Render Queue sequences and Save Frame As ProEXR) received the frame after Project Settings > HDR, so Compand and Tone Map (the default of every OCIO project) compressed highlights and changed values below 1.0 before the writer decoded them, always with the sRGB curve. - RenderOpts::scene_output: the top-level comp converts to the output space without the HDR step (CPU and GPU walks, via Renderer::output_pipe). - Pipe::output_linear: the writers decode with the output space's curve (the Working Gamma's when unmanaged) instead of a fixed sRGB curve. - Render Queue EXR jobs and ProEXR set the flag; PSD layers, 8-bit stills, movies and the viewer keep the HDR step. The flag is part of the viewer frame and disk cache keys (opts_hash). Golden hashes, taken before the change, pin PNG/TIFF/Targa bytes for 8/16/32 bpc x Clip/Compand/ToneMap, Clip-mode EXR samples with an sRGB output, and the layered PSD bytes. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/frame_export.rs | 28 +- crates/engine/src/tests/frame_export.rs | 108 ++++++++ crates/export/src/lib.rs | 14 +- crates/export/src/pipeline.rs | 2 + crates/export/tests/it/exr_scene_linear.rs | 293 +++++++++++++++++++++ crates/export/tests/it/main.rs | 1 + crates/gpu/src/tests.rs | 28 ++ crates/gpu/src/walk.rs | 5 +- crates/render/src/accel.rs | 6 + crates/render/src/color.rs | 14 + crates/render/src/opts.rs | 6 + crates/render/src/pipeline.rs | 5 +- crates/render/src/tests/color.rs | 46 ++++ crates/ui-egui/src/app/viewer_frames.rs | 1 + crates/ui-egui/src/frames/tests.rs | 8 + docs/architecture.md | 6 +- 16 files changed, 548 insertions(+), 23 deletions(-) create mode 100644 crates/export/tests/it/exr_scene_linear.rs diff --git a/crates/engine/src/cmd/frame_export.rs b/crates/engine/src/cmd/frame_export.rs index d8b49159c..24b6f0c3e 100644 --- a/crates/engine/src/cmd/frame_export.rs +++ b/crates/engine/src/cmd/frame_export.rs @@ -27,12 +27,15 @@ struct Isolated { image: Image, } -fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64) -> Image { +/// `scene_output`: a float output (ProEXR), rendered without the HDR step +/// ([`RenderOpts::scene_output`]). +fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, scene_output: bool) -> Image { let opts = RenderOpts { scale, backend: Backend::Auto, nested_switches: s.prefs.general.switches_affect_nested_comps, draft_shadows: s.prefs.three_d.realtime_shadows, + scene_output, ..Default::default() }; let mut r = Renderer::new(project, s.footage.as_ref(), opts); @@ -41,10 +44,11 @@ fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64 r.comp_frame(comp, t) } -/// The comp frame and each visible layer rendered alone (top layer first). -fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Image, Vec)> { +/// The comp frame and each visible layer rendered alone (top layer first). `scene_output`: as +/// [`render_with`]. +fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> Result<(Image, Vec)> { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; - let full = render_with(s, &s.project, cid, t, scale); + let full = render_with(s, &s.project, cid, t, scale, scene_output); let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.has_video()); let ids: Vec = comp .layers @@ -72,7 +76,7 @@ fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Image, Vec< } } let Some((name, blend, opacity)) = info else { continue }; - let image = render_with(s, &p, cid, t, scale); + let image = render_with(s, &p, cid, t, scale, scene_output); out.push(Isolated { name, blend, opacity, image }); } Ok((full, out)) @@ -111,7 +115,7 @@ fn alpha_bounds(img: &Image) -> Option<(u32, u32, u32, u32)> { pub fn layered_psd(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec, usize, u32, u32)> { use effectcraft_psd::Rect; use effectcraft_psd::write::{WDoc, WLayer, write}; - let (full, layers) = isolate(s, cid, t, scale)?; + let (full, layers) = isolate(s, cid, t, scale, false)?; let mut doc = WDoc::new(full.width, full.height); let n = layers.len(); // File order is bottom first. @@ -136,10 +140,6 @@ pub fn layered_psd(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec Ok((write(&doc), n, full.width, full.height)) } -fn srgb_to_linear(c: f32) -> f32 { - if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } -} - /// A unique EXR layer name (no dots: they separate the layer from the channel). Printable ASCII only: /// the `exr` crate panics on characters it cannot store in a channel name (#1702). fn exr_name(name: &str, used: &mut Vec) -> String { @@ -158,15 +158,19 @@ fn exr_name(name: &str, used: &mut Vec) -> String { /// The multi-layer EXR bytes of a frame and the channel names. pub fn layered_exr(s: &Session, cid: ItemId, t: Tick, scale: f64) -> Result<(Vec, Vec, u32, u32)> { use exr::prelude::*; - let (full, layers) = isolate(s, cid, t, scale)?; + // The composite and every layer without the HDR step, decoded with the output space's + // curve (linear light in the output space's primaries). + let (full, layers) = isolate(s, cid, t, scale, true)?; + let dec = effectcraft_render::color::Pipe::of(&s.project.settings).output_linear(); let (w, h) = (full.width as usize, full.height as usize); let planes = |img: &effectcraft_raster::Image| -> [Vec; 4] { let mut out: [Vec; 4] = Default::default(); for p in &img.data { let sp = straight(*p); let a = sp[3]; + let lin = dec.apply([sp[0].max(0.0), sp[1].max(0.0), sp[2].max(0.0)]); for c in 0..3 { - out[c].push(srgb_to_linear(sp[c].max(0.0)) * a); + out[c].push(lin[c] * a); } out[3].push(a); } diff --git a/crates/engine/src/tests/frame_export.rs b/crates/engine/src/tests/frame_export.rs index a8dc938fb..2686e810e 100644 --- a/crates/engine/src/tests/frame_export.rs +++ b/crates/engine/src/tests/frame_export.rs @@ -230,3 +230,111 @@ fn save_frame_as_queued_respects_frame_default_template() { s.execute_checked("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "still", "name": "JPEG Sequence"})).unwrap(); assert!(s.execute_checked("comp.saveFrameAs", json!({"queue": true})).is_ok()); } + +fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ *b as u64).wrapping_mul(0x0100_0000_01b3)) +} + +/// A 32×32 comp in a managed project (sRGB working space, linearized): a full-frame over-range +/// solid under a 16×16 in-range Multiply solid at 50%. +fn hdr_frame(bit_depth: u64, hdr: &str) -> Session { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Frame", "width": 32, "height": 32, "frameRate": 24, "duration": 1})).unwrap(); + s.execute("file.projectSettings", json!({"bitDepth": bit_depth, "workingSpace": "srgb", "linearize": true, "hdr": hdr})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Bright", "color": [4.0, 2.0, 0.5]})).unwrap(); + let box_ = s.execute("layer.newSolid", json!({"name": "Box", "color": [0.85, 0.9, 0.2], "width": 16, "height": 16})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.setBlendMode", json!({"layers": [box_], "mode": "Multiply"})).unwrap(); + s.execute("prop.set", json!({"layer": box_, "path": "transform/opacity", "value": 50})).unwrap(); + s +} + +/// Photoshop Layers keeps the HDR step (it is a display-referred output): its bytes for each bit +/// depth × HDR mode. +#[test] +fn photoshop_layers_bytes_are_unchanged() { + let dir = out_dir("frame-psd-golden"); + let mut got = vec![]; + for depth in [8, 16, 32] { + for hdr in ["clip", "compand", "toneMap"] { + let mut s = hdr_frame(depth, hdr); + let path = dir.join(format!("{depth}-{hdr}.psd")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsPsd", json!({"path": path})).unwrap(); + got.push(fnv(&std::fs::read(&path).unwrap())); + } + } + // The hashes before ProEXR skipped the HDR step. + let want: [u64; 9] = [ + 0x3b921d4d9fc3119f, + 0x1886d026edf36a7f, + 0x6b564aa7fb50139f, + 0xabd585992079fc9f, + 0x88ca38726eaa557f, + 0xa3636606d7a4927f, + 0x7385ed211e9815ff, + 0x7385ed211e9815ff, + 0xfb59fa3f3bdfac3f, + ]; + assert_eq!(got, want); +} + +/// Every sample of a ProEXR file, channels in name order. +fn proexr_samples(path: &str) -> Vec<(String, Vec)> { + use exr::prelude::*; + let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_file(path).unwrap(); + let mut out: Vec<(String, Vec)> = img.layer_data[0] + .channel_data + .list + .iter() + .map(|c| match &c.sample_data { + FlatSamples::F32(v) => (c.name.to_string(), v.clone()), + o => panic!("{o:?}"), + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +/// ProEXR samples in Clip mode (sRGB output) for each bit depth. +#[test] +fn proexr_clip_samples_are_unchanged() { + let dir = out_dir("frame-exr-clip-golden"); + let mut got = vec![]; + for depth in [8, 16, 32] { + let mut s = hdr_frame(depth, "clip"); + let path = dir.join(format!("{depth}.exr")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsExr", json!({"path": path})).unwrap(); + let bytes: Vec = + proexr_samples(&path).iter().flat_map(|(n, v)| n.bytes().chain(v.iter().flat_map(|x| x.to_bits().to_le_bytes())).collect::>()).collect(); + got.push(fnv(&bytes)); + } + // The hashes before ProEXR skipped the HDR step. + let want: [u64; 3] = [0x4b9ecc87e6db7d03, 0xe06a35d302e71503, 0x694ca552f3efc903]; + assert_eq!(got, want); +} + +/// ProEXR ignores compand and tone mapping: the composite and every isolated layer hold the Clip +/// mode's samples, and over-range values stay. +#[test] +fn proexr_layers_are_not_tone_mapped() { + let dir = out_dir("frame-exr-hdr"); + let samples = |depth: u64, hdr: &str| { + let mut s = hdr_frame(depth, hdr); + let path = dir.join(format!("{depth}-{hdr}.exr")).to_string_lossy().to_string(); + s.execute("comp.saveFrameAsExr", json!({"path": path})).unwrap(); + proexr_samples(&path) + }; + for depth in [8, 16, 32] { + let clip = samples(depth, "clip"); + assert!(clip.iter().any(|(n, _)| n == "Bright.R") && clip.iter().any(|(n, _)| n == "Box.R"), "{:?}", clip.iter().map(|c| &c.0).collect::>()); + for hdr in ["compand", "toneMap"] { + assert!(samples(depth, hdr) == clip, "{depth} bpc {hdr}"); + } + } + let tone = samples(32, "toneMap"); + let at = |n: &str| tone.iter().find(|c| c.0 == n).unwrap().1[0]; + // The solid's colour is sRGB-encoded in the working space (4.0 is about 25.3 in linear light). + let lin = effectcraft_color::srgb_to_linear; + for (n, want) in [("Bright.R", lin(4.0)), ("Bright.G", lin(2.0)), ("Bright.B", lin(0.5)), ("R", lin(4.0))] { + assert!((at(n) - want).abs() <= 1e-4 * want, "{n}: {} vs {want}", at(n)); + } +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 1b69a87bd..2d0d38071 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -302,7 +302,10 @@ fn write_still(cx: &Cx, file: &mut W, img: &I OutputFormat::ExrSequence => { // Linear light, premultiplied (the EXR convention); RGB keeps alpha = 1 over the background. // Colour is floored at 0 but not capped at 1: a 32 bpc frame's over-range values survive (#339). - let lin = |v: f32| if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) }; + // The frame was rendered without the HDR step (`RenderOpts::scene_output`) and is + // decoded with the output space's curve, in the output space's primaries. + let dec = effectcraft_render::color::Pipe::of(&cx.project.settings).output_linear(); + let lin3 = |c: [f32; 3]| dec.apply(c.map(|v| v.max(0.0))); let mut data = Vec::with_capacity((w * h * 4) as usize); for y in 0..h { for x in 0..w { @@ -310,17 +313,16 @@ fn write_still(cx: &Cx, file: &mut W, img: &I let (c, a) = match channels { Channels::Alpha => { let a = p[3].clamp(0.0, 1.0); - ([lin(a); 3], 1.0) + (lin3([a; 3]), 1.0) } Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); - let s = |c: f32| if a > 0.0 { lin((c / a).max(0.0)) * a } else { 0.0 }; - ([s(p[0]), s(p[1]), s(p[2])], a) + let c = if a > 0.0 { lin3([p[0] / a, p[1] / a, p[2] / a]).map(|v| v * a) } else { [0.0; 3] }; + (c, a) } Channels::Rgb => { let k = 1.0 - p[3].clamp(0.0, 1.0); - let c = |i: usize| lin((p[i] + comp.background[i] * k).max(0.0)); - ([c(0), c(1), c(2)], 1.0) + (lin3([0, 1, 2].map(|i| p[i] + comp.background[i] * k)), 1.0) } }; data.extend_from_slice(&[c[0], c[1], c[2], a]); diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 8e8c90561..3742ae0a2 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -251,6 +251,8 @@ impl<'a> Cx<'a> { roi: None, nested_switches: self.job.nested_switches, draft_shadows: true, + // OpenEXR keeps over-range values: no HDR compand / tone mapping on its frames. + scene_output: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, proxy: s.proxy_use, }; let mut r = Renderer::new(&self.project, self.job.footage, opts); diff --git a/crates/export/tests/it/exr_scene_linear.rs b/crates/export/tests/it/exr_scene_linear.rs new file mode 100644 index 000000000..99a419894 --- /dev/null +++ b/crates/export/tests/it/exr_scene_linear.rs @@ -0,0 +1,293 @@ +//! OpenEXR output holds the comp's values in linear light without Project Settings ▸ HDR +//! (compand / tone map), and decodes with the output space's own curve; 8-bit writers keep the +//! HDR step and their exact bytes. + +use std::path::{Path, PathBuf}; + +use effectcraft_color::{ColorSpace, Conversion, Label}; +use effectcraft_export::render_queue::{Channels, OutputFormat, OutputModule, RenderSettings, TimeSpan}; +use effectcraft_export::{Job, export}; +use effectcraft_project::{BitDepth, Comp, HdrMode, ItemId, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; + +const W: u32 = 48; +const H: u32 = 16; +/// Three nested bands, outermost first: in range, inside the compand knee, over range. +const BANDS: [[f32; 3]; 3] = [[0.2, 0.5, 0.9], [0.85, 0.9, 0.95], [4.0, 2.0, 0.5]]; + +fn out_dir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-exr-scene-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("mkdir"); + d +} + +/// A one-frame comp of [`BANDS`] (full width, 2/3, 1/3, centred) with the given colour settings. +fn project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: bool, out: Option) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + p.settings.hdr = hdr; + p.settings.working_space = ws; + p.settings.linearize = linearize; + p.settings.output_space = out; + let mut comp = Comp::new(W, H, FrameRate::new(10, 1), Tick::from_seconds_f64(0.1)); + comp.background = [0.0, 0.0, 0.0]; + for (i, color) in BANDS.into_iter().enumerate() { + let w = W - W / 3 * i as u32; + let sid = p.add_item(&format!("Band {i}"), Label::Red, None, ItemKind::Solid(Solid { color, width: w, height: H, pixel_aspect: 1.0 })); + let l = build::layer(&mut p, &comp, &format!("Band {i}"), LayerSource::Solid { item: sid }, (w, H), None); + comp.layers.insert(0, l); + } + let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +/// Export the comp's one frame as a `fmt` sequence; the frame file's path. +fn render(p: &Project, cid: ItemId, fmt: OutputFormat, channels: Channels, dir: &Path, ext: &str) -> PathBuf { + let mut om = OutputModule::for_format(fmt); + om.channels = channels; + let s = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let path = dir.join(format!("f_[#####].{ext}")).to_string_lossy().to_string(); + let job = Job { + project: p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + nested_switches: true, + }; + export(&job, &mut |_| true).expect("export"); + dir.join(format!("f_00000.{ext}")) +} + +fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ *b as u64).wrapping_mul(0x0100_0000_01b3)) +} + +const DEPTHS: [BitDepth; 3] = [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32]; +const MODES: [HdrMode; 3] = [HdrMode::Clip, HdrMode::Compand, HdrMode::ToneMap]; +const STILLS: [(OutputFormat, Channels, &str); 3] = [ + (OutputFormat::PngSequence, Channels::Rgba, "png"), + (OutputFormat::TiffSequence, Channels::Rgb, "tif"), + (OutputFormat::TargaSequence, Channels::Rgba, "tga"), +]; + +/// The bytes of every 8-bit still for each bit depth × HDR mode, in a managed linear sRGB +/// project with a Display P3 output. +fn still_hashes() -> Vec { + let mut out = vec![]; + for depth in DEPTHS { + for mode in MODES { + let (p, cid) = project(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); + for (fmt, ch, ext) in STILLS { + let d = out_dir(&format!("golden-{depth:?}-{mode:?}-{ext}")); + let bytes = std::fs::read(render(&p, cid, fmt, ch, &d, ext)).expect("read"); + out.push(fnv(&bytes)); + let _ = std::fs::remove_dir_all(&d); + } + } + } + out +} + +/// [`still_hashes`] before float outputs skipped the HDR step: PNG, TIFF and Targa (8-bit +/// writers, which keep compand / tone mapping) are byte for byte the same. +const STILL_GOLDEN: [u64; 27] = [ + 0xd0cf67f45d6b6b68, + 0x712d42f3e204cbdb, + 0x021dc457f259eb87, + 0x88a16b1ca2db8805, + 0xc6fb221b89179bdb, + 0x965636561aaabb87, + 0x85c831b7694662db, + 0x8875a5c5232a33db, + 0x42d8f3c110bd0987, + 0xd555a6850fe66f04, + 0x920b49ee0f9900e4, + 0x94e4f43742f2c8ce, + 0xab0792254da43348, + 0x647a60c0a33237ec, + 0x65cba80bc3ecff08, + 0x0726be1b60eb7276, + 0xffd0315633cb88db, + 0x4f6a13131fb7a175, + 0x07a706a7e100b51d, + 0x718600eaa65a6f80, + 0x53489230b732f638, + 0x13024d498aa2552c, + 0x3a3dae30c9e25f83, + 0xc3191a3e860aa0c5, + 0xe5d7d08ecc3ddece, + 0x29a748f729fe5c8a, + 0x78d3b9aa9e107774, +]; + +#[test] +fn eight_bit_stills_are_unchanged() { + assert_eq!(still_hashes(), STILL_GOLDEN); +} + +/// The decoded samples of a still EXR frame, row-major RGBA. +fn exr_samples(path: &Path) -> Vec { + image::open(path).expect("exr").to_rgba32f().into_raw() +} + +fn bits_hash(v: &[f32]) -> u64 { + fnv(&v.iter().flat_map(|x| x.to_bits().to_le_bytes()).collect::>()) +} + +/// Clip-mode EXR samples with an sRGB output, for each colour setup × bit depth × channels. +fn exr_clip_hashes() -> Vec { + let mut out = vec![]; + for (ws, linearize) in [(None, false), (Some(ColorSpace::Srgb), false), (Some(ColorSpace::Srgb), true), (Some(ColorSpace::AcesCg), true)] { + for depth in DEPTHS { + let (p, cid) = project(depth, HdrMode::Clip, ws, linearize, None); + for ch in [Channels::Rgba, Channels::Rgb, Channels::Alpha] { + let d = out_dir(&format!("clip-{ws:?}-{linearize}-{depth:?}-{ch:?}")); + out.push(bits_hash(&exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")))); + let _ = std::fs::remove_dir_all(&d); + } + } + } + out +} + +/// [`exr_clip_hashes`] before: Clip-mode EXRs with an sRGB output keep every sample. +const EXR_CLIP_GOLDEN: [u64; 36] = [ + 0x9f54fea3d3abcb25, + 0x9f54fea3d3abcb25, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x9f54fea3d3abcb25, + 0x9f54fea3d3abcb25, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x27858400549ac325, + 0x27858400549ac325, + 0x7d40ad9df8600325, + 0x379f5ef9e35ce325, + 0x379f5ef9e35ce325, + 0x7d40ad9df8600325, + 0xb60de6a90263b525, + 0xb60de6a90263b525, + 0x7d40ad9df8600325, + 0x31d03697119d9325, + 0x31d03697119d9325, + 0x7d40ad9df8600325, + 0xff61dc513c834b25, + 0xff61dc513c834b25, + 0x7d40ad9df8600325, + 0x0c57d24a44f45925, + 0x0c57d24a44f45925, + 0x7d40ad9df8600325, +]; + +#[test] +fn exr_clip_with_srgb_output_is_unchanged() { + assert_eq!(exr_clip_hashes(), EXR_CLIP_GOLDEN); +} + +/// Compand and tone mapping leave EXR output alone: the file holds the Clip mode's samples, for +/// every bit depth, channel set and colour setup (wide working spaces, a Display P3 output). +#[test] +fn exr_ignores_compand_and_tone_mapping() { + let setups = [ + (Some(ColorSpace::Srgb), true, None), + (Some(ColorSpace::Srgb), false, Some(ColorSpace::DisplayP3)), + (Some(ColorSpace::AcesCg), true, None), + (Some(ColorSpace::Rec2020), true, Some(ColorSpace::Rec709)), + ]; + for (ws, linearize, out) in setups { + for depth in DEPTHS { + for ch in [Channels::Rgba, Channels::Rgb] { + let samples = |mode: HdrMode| { + let (p, cid) = project(depth, mode, ws, linearize, out); + let d = out_dir(&format!("modes-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); + let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + v + }; + let clip = samples(HdrMode::Clip); + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + assert!(samples(mode) == clip, "{ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); + } + } + } + } +} + +/// The centre pixel (the innermost band) of a one-frame EXR render. +fn exr_centre(p: &Project, cid: ItemId, name: &str) -> [f32; 4] { + let d = out_dir(name); + let v = exr_samples(&render(p, cid, OutputFormat::ExrSequence, Channels::Rgba, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + let o = ((H / 2 * W + W / 2) * 4) as usize; + [v[o], v[o + 1], v[o + 2], v[o + 3]] +} + +fn assert_near(got: [f32; 4], want: [f32; 3], what: &str) { + for i in 0..3 { + assert!((got[i] - want[i]).abs() <= 1e-4 * want[i].abs().max(1.0), "{what}: channel {i}: got {got:?}, want {want:?}"); + } + assert_eq!(got[3], 1.0, "{what}: {got:?}"); +} + +/// In a 32 bpc linear project with compand or tone mapping, the EXR holds the comp's scene-linear +/// values in the output space's primaries: 4.0 stays 4.0. (Solid colours are authored in the +/// working space's encoding; ACES spaces are linear.) +#[test] +fn exr_holds_scene_linear_values_in_output_primaries() { + let [_, _, band] = BANDS; + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + let cases = [ + // ACEScg in and out: nothing converts. + (ColorSpace::AcesCg, ColorSpace::AcesCg, band), + // ACEScg working space, Display P3 output: the primaries convert, the values stay linear. + (ColorSpace::AcesCg, ColorSpace::DisplayP3, Conversion::new(ColorSpace::AcesCg, true, ColorSpace::DisplayP3, true).expect("matrix").apply(band)), + // Linearized Rec. 2020 working space, sRGB output. + (ColorSpace::Rec2020, ColorSpace::Srgb, Conversion::new(ColorSpace::Rec2020, false, ColorSpace::Srgb, true).expect("matrix").apply(band)), + // Linearized sRGB in and out. + (ColorSpace::Srgb, ColorSpace::Srgb, band.map(effectcraft_color::srgb_to_linear)), + ]; + for (ws, out, want) in cases { + let (p, cid) = project(BitDepth::Bpc32, mode, Some(ws), true, Some(out)); + let what = format!("{mode:?} {ws:?} to {out:?}"); + assert_near(exr_centre(&p, cid, &format!("lin-{ws:?}-{out:?}-{mode:?}")), want.map(|v| v.max(0.0)), &what); + } + } + let (p, cid) = project(BitDepth::Bpc32, HdrMode::ToneMap, Some(ColorSpace::AcesCg), true, Some(ColorSpace::AcesCg)); + assert_eq!(exr_centre(&p, cid, "lin-exact")[..3], band, "4.0 is exactly 4.0"); +} + +/// The EXR is decoded with the output space's curve, not always sRGB's: a Rec. 709 output (pure +/// 2.4 power curve) and an unmanaged project at Working Gamma 2.4 give back the linear values. +#[test] +fn exr_decodes_with_the_output_curve() { + let [_, _, band] = BANDS; + // A linearized sRGB working space (same primaries as Rec. 709): the authored colour, decoded. + let (p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, Some(ColorSpace::Srgb), true, Some(ColorSpace::Rec709)); + assert_near(exr_centre(&p, cid, "curve-709"), band.map(effectcraft_color::srgb_to_linear), "Rec. 709 output"); + // Unmanaged at 2.4: the solid's colour is encoded with the 2.4 power curve. + let (mut p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, None, false, None); + p.settings.working_gamma = 2.4; + assert_near(exr_centre(&p, cid, "curve-gamma24"), band.map(|v| v.powf(2.4)), "Working Gamma 2.4"); + // Working Gamma 2.2 keeps the sRGB curve. + let (p, cid) = project(BitDepth::Bpc32, HdrMode::Clip, None, false, None); + assert_near(exr_centre(&p, cid, "curve-gamma22"), band.map(effectcraft_color::srgb_to_linear), "Working Gamma 2.2"); +} diff --git a/crates/export/tests/it/main.rs b/crates/export/tests/it/main.rs index a5b9fa99a..87548f9e5 100644 --- a/crates/export/tests/it/main.rs +++ b/crates/export/tests/it/main.rs @@ -2,6 +2,7 @@ mod audio_sample_positions; mod banding; +mod exr_scene_linear; mod h264_gpu_yuv; mod hevc_av1_roundtrip; #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 44e8922a6..4c09faa29 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -1315,3 +1315,31 @@ fn owned_devices_favour_memory_usage() { assert_eq!(d.label, Some("effectcraft gpu")); assert_eq!(d.required_limits.max_texture_dimension_2d, wgpu::Limits::default().max_texture_dimension_2d); } + +/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR on the GPU as on the +/// CPU: both compositors agree, and the frame is the Clip frame with its over-range values. +#[test] +fn scene_output_skips_the_hdr_step_on_the_gpu() { + use effectcraft_project::HdrMode; + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + let mut s = blend_scene(BitDepth::Bpc32, BlendMode::Normal); + let mut bright = s.solid([3.0, 1.5, 0.5], 30, 20); + bright.blend_mode = BlendMode::Add; + s.push(bright); + s.p.settings.working_space = Some(ColorSpace::Srgb); + s.p.settings.linearize = true; + s.p.settings.hdr = mode; + let scene = RenderOpts { scene_output: true, ..opts() }; + check(&format!("scene output {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); + let Some(g) = gpu() else { continue }; + let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..scene }); + r.accel = Some(g); + let img = g.render(&r, s.cid, Tick::ZERO).expect("the GPU renders the frame"); + let mut clip = s.p.clone(); + clip.settings.hdr = HdrMode::Clip; + let want = Renderer::new(&clip, &Pattern, RenderOpts { backend: Backend::Cpu, ..opts() }).comp_frame_cpu(s.cid, Tick::ZERO); + let d = diff(&want, &img, tolerance(BitDepth::Bpc32)); + assert_eq!(d.over, 0, "{mode:?}: max {} at {:?}", d.max, d.worst); + assert!(img.data.iter().any(|p| p[3] > 0.0 && p[0] / p[3] > 1.0), "{mode:?}: over-range values survive"); + } +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index fdcb58dd4..1b6687395 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -45,8 +45,9 @@ pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick if let Some(l) = pipe.levels { canvas = ops::quantize(e, &canvas, l); } + let out = r.output_pipe(); if r.depth() == 0 - && let Some(c) = pipe.output() + && let Some(c) = out.output() { canvas = ops::convert(e, &canvas, &c); if let Some(l) = pipe.levels { @@ -54,7 +55,7 @@ pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick } } if r.depth() == 0 - && let Some((lin, mode, enc)) = pipe.output_hdr() + && let Some((lin, mode, enc)) = out.output_hdr() { on_cpu(e, &mut canvas, |img| { effectcraft_render::color::output_hdr(img, lin, mode, enc); diff --git a/crates/render/src/accel.rs b/crates/render/src/accel.rs index e6bbd2113..613853e69 100644 --- a/crates/render/src/accel.rs +++ b/crates/render/src/accel.rs @@ -174,6 +174,12 @@ impl<'a> Renderer<'a> { self.pipe } + /// The colour pipeline of the top-level output steps: the project's, without the HDR step + /// for a float output ([`crate::RenderOpts::scene_output`]). + pub fn output_pipe(&self) -> color::Pipe { + if self.opts.scene_output { self.pipe.scene_output() } else { self.pipe } + } + /// Nesting depth (0 = the top-level comp). pub fn depth(&self) -> usize { self.depth diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index d27526553..54d6d82e0 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -117,6 +117,20 @@ impl Pipe { Conversion::new(ws, self.linear, self.out, false) } + /// The pipe of a float output ([`crate::RenderOpts::scene_output`]): the same, without + /// the HDR step (Clip), so [`Pipe::output`] converts to the output space as it is. + pub fn scene_output(self) -> Pipe { + Pipe { hdr: HdrMode::Clip, ..self } + } + + /// The top-level frame of a [`Pipe::scene_output`] render (output-encoded, straight colour) + /// → linear light in the output space's primaries: the output space's curve, or the Working + /// Gamma's when unmanaged (sRGB at 2.2, a pure 2.4 power curve at 2.4). The identity for a + /// linear output (ACES). + pub fn output_linear(&self) -> Conversion { + Conversion::linearize(if self.space.is_some() { self.out } else { self.curve() }) + } + /// Output with HDR compand / tone mapping (standard-dynamic-range outputs only): working /// space → linear output primaries, the HDR curve, then the output's encoding. pub fn output_hdr(&self) -> Option<(Option, HdrMode, Conversion)> { diff --git a/crates/render/src/opts.rs b/crates/render/src/opts.rs index c6175bf39..ae8d3435f 100644 --- a/crates/render/src/opts.rs +++ b/crates/render/src/opts.rs @@ -28,6 +28,11 @@ pub struct RenderOpts { pub nested_switches: bool, /// Realtime Shadows in Draft (Settings ▸ 3D): draft renders (`draft`) still cast shadows. pub draft_shadows: bool, + /// A float output (OpenEXR): the top-level comp leaves the working space for the output + /// space without Project Settings ▸ HDR (compand / tone mapping are for standard-dynamic-range + /// displays and outputs), so over-range values reach the file as they are. The writer + /// decodes the frame with [`crate::color::Pipe::output_linear`]. + pub scene_output: bool, } impl RenderOpts { @@ -49,6 +54,7 @@ impl Default for RenderOpts { backend: Backend::Cpu, nested_switches: true, draft_shadows: true, + scene_output: false, proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, } } diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index a4927144f..7ecf2f536 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -473,14 +473,15 @@ impl<'a> Renderer<'a> { self.pipe.quantize(&mut canvas); } // The top-level comp leaves the working space for the (sRGB) display / output. + let out = self.output_pipe(); if self.depth == 0 - && let Some(c) = self.pipe.output() + && let Some(c) = out.output() { color::convert(&mut canvas, &c); self.pipe.quantize(&mut canvas); } if self.depth == 0 - && let Some((lin, mode, enc)) = self.pipe.output_hdr() + && let Some((lin, mode, enc)) = out.output_hdr() { color::output_hdr(&mut canvas, lin, mode, enc); self.pipe.quantize(&mut canvas); diff --git a/crates/render/src/tests/color.rs b/crates/render/src/tests/color.rs index 29e4d62c2..de3ad19c6 100644 --- a/crates/render/src/tests/color.rs +++ b/crates/render/src/tests/color.rs @@ -339,3 +339,49 @@ fn float_footage_is_encoded_once_with_the_working_gamma() { assert_eq!(render(2.4, Some(ColorSpace::Srgb), linear_light), srgb); } } + +/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR: its top-level frame +/// is the Clip frame in every mode, while other renders keep compand / tone mapping. +#[test] +fn scene_output_skips_the_hdr_step() { + use effectcraft_project::HdrMode; + for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { + let (mut p, cid, _) = setup(depth); + p.settings.working_space = Some(ColorSpace::Srgb); + p.settings.linearize = true; + add_solid(&mut p, cid, [4.0, 2.0, 0.5], BlendMode::Normal, 100.0); + let frame = |p: &Project, scene_output: bool| { + Renderer::new(p, &crate::NoFootage, RenderOpts { scene_output, ..Default::default() }).comp_frame(cid, Tick::ZERO) + }; + let clip = frame(&p, false); + assert_eq!(clip.data, frame(&p, true).data, "{depth:?} Clip"); + for mode in [HdrMode::Compand, HdrMode::ToneMap] { + p.settings.hdr = mode; + assert_eq!(frame(&p, true).data, clip.data, "{depth:?} {mode:?}"); + assert_ne!(frame(&p, false).data, clip.data, "{depth:?} {mode:?} keeps the HDR step"); + } + } +} + +/// `Pipe::output_linear` decodes with the output space's curve (the Working Gamma's when +/// unmanaged) and undoes `Pipe::output` exactly where the primaries match. +#[test] +fn output_linear_decodes_with_the_output_curve() { + use effectcraft_color::Conversion; + use effectcraft_project::ProjectSettings; + let pipe = |ws: Option, out: Option, gamma: f64| { + crate::color::Pipe::of(&ProjectSettings { working_space: ws, output_space: out, working_gamma: gamma, ..Default::default() }) + }; + assert_eq!(pipe(None, None, 2.2).output_linear(), Conversion::linearize(ColorSpace::Srgb)); + assert_eq!(pipe(None, Some(ColorSpace::DisplayP3), 2.4).output_linear(), Conversion::linearize(ColorSpace::Rec709), "unmanaged: the Working Gamma"); + assert_eq!(pipe(Some(ColorSpace::Srgb), None, 2.4).output_linear(), Conversion::linearize(ColorSpace::Srgb)); + assert_eq!(pipe(Some(ColorSpace::Srgb), Some(ColorSpace::Rec709), 2.2).output_linear(), Conversion::linearize(ColorSpace::Rec709)); + // Linear light in, linear light out: 4.0 stays 4.0 through output + output_linear. + let p = pipe(Some(ColorSpace::Rec709), Some(ColorSpace::Srgb), 2.2); + let p = crate::color::Pipe { linear: true, ..p }; + let enc = p.output().unwrap().apply([4.0, 1.0, 0.25]); + let lin = p.output_linear().apply(enc); + for (got, want) in lin.into_iter().zip([4.0, 1.0, 0.25]) { + assert!((got - want).abs() <= 1e-5 * want, "{lin:?}"); + } +} diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index f87db615c..241951c54 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -164,6 +164,7 @@ impl EffectcraftApp { backend: effectcraft_engine::render::Backend::Auto, nested_switches: self.session.prefs.general.switches_affect_nested_comps, draft_shadows: self.session.prefs.three_d.realtime_shadows, + scene_output: false, proxy: Default::default(), } } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index d04793f50..77584e2df 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -6,6 +6,14 @@ fn converting_a_frame_with_nan_alpha_keeps_the_viewer_alive() { assert_eq!(to_color_image(&image).pixels, [egui::Color32::TRANSPARENT]); } +/// A float output's frames (`RenderOpts::scene_output`, no HDR step) never share a RAM or disk +/// cache key with the HDR-mapped frames of the same options. +#[test] +fn scene_output_frames_have_keys_of_their_own() { + let opts = RenderOpts::default(); + assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { scene_output: true, ..opts })); +} + struct ProbeDisk { keys: Mutex>, probes: std::sync::atomic::AtomicUsize, diff --git a/docs/architecture.md b/docs/architecture.md index c4bebd9ed..226f93a14 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -407,7 +407,11 @@ inputs in 32 bpc. With a working space (sRGB, Rec. 709, Rec. 2020, Display P3; m from the published primaries), footage is converted from its colour profile (stream metadata or Interpret Footage, else sRGB) and the top-level comp is converted to the sRGB display. Linearize Working Space runs everything in linear light; Blend Colors Using 1.0 Gamma linearises only for -blending. Layer cache keys include these settings. +blending. Layer cache keys include these settings. Project Settings ▸ HDR (compand / tone +mapping) is the last top-level step, for standard-dynamic-range displays and outputs; OpenEXR +renders (sequences and ProEXR) skip it (`RenderOpts::scene_output`) and decode the frame with the +output space's own curve (`Pipe::output_linear`), so they hold linear light in the output space's +primaries. **Frame blending**: footage and precomp layers whose source time falls between source frames (rate conform, stretch, remap) blend the two neighbouring frames, by cross-fade (Frame Mix) or by From 406c9a9648d1e4d1eedc7b594d9d65efa9238a55 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 01:57:53 +0100 Subject: [PATCH 03/35] EXR export: raw alpha for Alpha-only files, skip_hdr_step name, more tests - Rename RenderOpts::scene_output to skip_hdr_step (Pipe::without_hdr_step): the consumer decodes with Pipe::output_linear or applies the step itself. - Alpha-only EXRs hold the alpha as it is in every output space (it is coverage, not encoded colour; it used to be sRGB-decoded). - Tests: a half-opacity layer over a coloured background (premultiplied and RGB-over-background paths, golden hashes taken before the change), Rec. 2100 PQ and HLG EXR outputs, raw alpha, and a visible skip of the GPU test without an adapter. - docs/architecture.md: the Working Gamma curve for unmanaged projects, and no HDR step for unmanaged projects or PQ / HLG outputs. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/frame_export.rs | 15 +-- crates/export/src/lib.rs | 8 +- crates/export/src/pipeline.rs | 2 +- crates/export/tests/it/exr_scene_linear.rs | 148 +++++++++++++++++++-- crates/gpu/src/tests.rs | 14 +- crates/render/src/accel.rs | 7 +- crates/render/src/color.rs | 8 +- crates/render/src/opts.rs | 12 +- crates/render/src/tests/color.rs | 8 +- crates/ui-egui/src/app/viewer_frames.rs | 2 +- crates/ui-egui/src/frames/tests.rs | 6 +- docs/architecture.md | 5 +- 12 files changed, 182 insertions(+), 53 deletions(-) diff --git a/crates/engine/src/cmd/frame_export.rs b/crates/engine/src/cmd/frame_export.rs index 24b6f0c3e..7347b84bb 100644 --- a/crates/engine/src/cmd/frame_export.rs +++ b/crates/engine/src/cmd/frame_export.rs @@ -27,15 +27,14 @@ struct Isolated { image: Image, } -/// `scene_output`: a float output (ProEXR), rendered without the HDR step -/// ([`RenderOpts::scene_output`]). -fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, scene_output: bool) -> Image { +/// `skip_hdr_step`: no HDR step, for a float output (ProEXR; [`RenderOpts::skip_hdr_step`]). +fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64, skip_hdr_step: bool) -> Image { let opts = RenderOpts { scale, backend: Backend::Auto, nested_switches: s.prefs.general.switches_affect_nested_comps, draft_shadows: s.prefs.three_d.realtime_shadows, - scene_output, + skip_hdr_step, ..Default::default() }; let mut r = Renderer::new(project, s.footage.as_ref(), opts); @@ -44,11 +43,11 @@ fn render_with(s: &Session, project: &Project, comp: ItemId, t: Tick, scale: f64 r.comp_frame(comp, t) } -/// The comp frame and each visible layer rendered alone (top layer first). `scene_output`: as +/// The comp frame and each visible layer rendered alone (top layer first). `skip_hdr_step`: as /// [`render_with`]. -fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> Result<(Image, Vec)> { +fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, skip_hdr_step: bool) -> Result<(Image, Vec)> { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; - let full = render_with(s, &s.project, cid, t, scale, scene_output); + let full = render_with(s, &s.project, cid, t, scale, skip_hdr_step); let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.has_video()); let ids: Vec = comp .layers @@ -76,7 +75,7 @@ fn isolate(s: &Session, cid: ItemId, t: Tick, scale: f64, scene_output: bool) -> } } let Some((name, blend, opacity)) = info else { continue }; - let image = render_with(s, &p, cid, t, scale, scene_output); + let image = render_with(s, &p, cid, t, scale, skip_hdr_step); out.push(Isolated { name, blend, opacity, image }); } Ok((full, out)) diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 2d0d38071..9a9e66d0f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -302,7 +302,7 @@ fn write_still(cx: &Cx, file: &mut W, img: &I OutputFormat::ExrSequence => { // Linear light, premultiplied (the EXR convention); RGB keeps alpha = 1 over the background. // Colour is floored at 0 but not capped at 1: a 32 bpc frame's over-range values survive (#339). - // The frame was rendered without the HDR step (`RenderOpts::scene_output`) and is + // The frame was rendered without the HDR step (`RenderOpts::skip_hdr_step`) and is // decoded with the output space's curve, in the output space's primaries. let dec = effectcraft_render::color::Pipe::of(&cx.project.settings).output_linear(); let lin3 = |c: [f32; 3]| dec.apply(c.map(|v| v.max(0.0))); @@ -311,10 +311,8 @@ fn write_still(cx: &Cx, file: &mut W, img: &I for x in 0..w { let p = if x < img.width && y < img.height { img.data[img.idx(x, y)] } else { [0.0; 4] }; let (c, a) = match channels { - Channels::Alpha => { - let a = p[3].clamp(0.0, 1.0); - (lin3([a; 3]), 1.0) - } + // Alpha is coverage, not encoded colour: written as it is. + Channels::Alpha => ([p[3].clamp(0.0, 1.0); 3], 1.0), Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); let c = if a > 0.0 { lin3([p[0] / a, p[1] / a, p[2] / a]).map(|v| v * a) } else { [0.0; 3] }; diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index 3742ae0a2..01e3f3faf 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -252,7 +252,7 @@ impl<'a> Cx<'a> { nested_switches: self.job.nested_switches, draft_shadows: true, // OpenEXR keeps over-range values: no HDR compand / tone mapping on its frames. - scene_output: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, + skip_hdr_step: self.job.output.format == effectcraft_project::render_queue::OutputFormat::ExrSequence, proxy: s.proxy_use, }; let mut r = Renderer::new(&self.project, self.job.footage, opts); diff --git a/crates/export/tests/it/exr_scene_linear.rs b/crates/export/tests/it/exr_scene_linear.rs index 99a419894..b4b8c027c 100644 --- a/crates/export/tests/it/exr_scene_linear.rs +++ b/crates/export/tests/it/exr_scene_linear.rs @@ -43,6 +43,34 @@ fn project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: boo (p, cid) } +/// A comp builder: bit depth, HDR mode, working space, Linearize Working Space, output space. +type Build = fn(BitDepth, HdrMode, Option, bool, Option) -> (Project, ItemId); + +/// The half-opacity layer of [`half_project`]: over range in red. +const HALF: [f32; 3] = [4.0, 0.6, 0.3]; + +/// A one-frame comp over a coloured background whose only layer, [`HALF`] at 50% opacity, +/// covers the left two thirds: semi-transparent and transparent pixels (the premultiplied and +/// RGB-over-background paths of the writers). +fn half_project(depth: BitDepth, hdr: HdrMode, ws: Option, linearize: bool, out: Option) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + p.settings.hdr = hdr; + p.settings.working_space = ws; + p.settings.linearize = linearize; + p.settings.output_space = out; + let mut comp = Comp::new(W, H, FrameRate::new(10, 1), Tick::from_seconds_f64(0.1)); + comp.background = [0.2, 0.3, 0.4]; + let w = W / 3 * 2; + let sid = p.add_item("Half", Label::Red, None, ItemKind::Solid(Solid { color: HALF, width: w, height: H, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Half", LayerSource::Solid { item: sid }, (w, H), None); + l.props.prop_mut("transform/position").expect("position").value = effectcraft_keyframe::Value::Vec3([w as f64 / 2.0, H as f64 / 2.0, 0.0]); + l.props.prop_mut("transform/opacity").expect("opacity").value = effectcraft_keyframe::Value::Scalar(50.0); + comp.layers.push(l); + let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + /// Export the comp's one frame as a `fmt` sequence; the frame file's path. fn render(p: &Project, cid: ItemId, fmt: OutputFormat, channels: Channels, dir: &Path, ext: &str) -> PathBuf { let mut om = OutputModule::for_format(fmt); @@ -80,11 +108,11 @@ const STILLS: [(OutputFormat, Channels, &str); 3] = [ /// The bytes of every 8-bit still for each bit depth × HDR mode, in a managed linear sRGB /// project with a Display P3 output. -fn still_hashes() -> Vec { +fn still_hashes(build: Build) -> Vec { let mut out = vec![]; for depth in DEPTHS { for mode in MODES { - let (p, cid) = project(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); + let (p, cid) = build(depth, mode, Some(ColorSpace::Srgb), true, Some(ColorSpace::DisplayP3)); for (fmt, ch, ext) in STILLS { let d = out_dir(&format!("golden-{depth:?}-{mode:?}-{ext}")); let bytes = std::fs::read(render(&p, cid, fmt, ch, &d, ext)).expect("read"); @@ -130,9 +158,41 @@ const STILL_GOLDEN: [u64; 27] = [ #[test] fn eight_bit_stills_are_unchanged() { - assert_eq!(still_hashes(), STILL_GOLDEN); + assert_eq!(still_hashes(project), STILL_GOLDEN); + assert_eq!(still_hashes(half_project), HALF_STILL_GOLDEN); } +/// [`still_hashes`] of [`half_project`] before: premultiplied and RGB-over-background stills. +const HALF_STILL_GOLDEN: [u64; 27] = [ + 0x3abd611ee63b7edd, + 0x51e7ec0426773cdb, + 0xbee34abe9f0a3d87, + 0x554c03dba915d290, + 0xe98490fc826a7adb, + 0x0a0e77927bf1a187, + 0x4f1c076634822d50, + 0xe790f474e05b3adb, + 0xe517c3136471a187, + 0x07cd5e73765539f9, + 0x68e51bbd8d442a64, + 0x844a5f23a9cc9a3d, + 0xc302b2b86f4cdd70, + 0xb7247795d204a2c5, + 0x3b18867537c69265, + 0x9a5f7679bb5a2399, + 0x868b729dcc804656, + 0x2e0bc64a3fb562f0, + 0x6dc9e1a1eac1220d, + 0x6e101cd849e78f32, + 0x7b7a2a776f73d70b, + 0x6b3244f808c65d4e, + 0xbfa1b35fada2c77d, + 0x8c3a31415ed47e33, + 0xac787507caf1c896, + 0x4a38297060de19ff, + 0x300d6e4222eaa5b5, +]; + /// The decoded samples of a still EXR frame, row-major RGBA. fn exr_samples(path: &Path) -> Vec { image::open(path).expect("exr").to_rgba32f().into_raw() @@ -143,12 +203,12 @@ fn bits_hash(v: &[f32]) -> u64 { } /// Clip-mode EXR samples with an sRGB output, for each colour setup × bit depth × channels. -fn exr_clip_hashes() -> Vec { +fn exr_clip_hashes(build: Build, channels: &[Channels]) -> Vec { let mut out = vec![]; for (ws, linearize) in [(None, false), (Some(ColorSpace::Srgb), false), (Some(ColorSpace::Srgb), true), (Some(ColorSpace::AcesCg), true)] { for depth in DEPTHS { - let (p, cid) = project(depth, HdrMode::Clip, ws, linearize, None); - for ch in [Channels::Rgba, Channels::Rgb, Channels::Alpha] { + let (p, cid) = build(depth, HdrMode::Clip, ws, linearize, None); + for &ch in channels { let d = out_dir(&format!("clip-{ws:?}-{linearize}-{depth:?}-{ch:?}")); out.push(bits_hash(&exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")))); let _ = std::fs::remove_dir_all(&d); @@ -200,7 +260,54 @@ const EXR_CLIP_GOLDEN: [u64; 36] = [ #[test] fn exr_clip_with_srgb_output_is_unchanged() { - assert_eq!(exr_clip_hashes(), EXR_CLIP_GOLDEN); + // The bands are opaque everywhere, so the Alpha-only files (alpha 1.0) are unchanged too. + assert_eq!(exr_clip_hashes(project, &[Channels::Rgba, Channels::Rgb, Channels::Alpha]), EXR_CLIP_GOLDEN); + assert_eq!(exr_clip_hashes(half_project, &[Channels::Rgba, Channels::Rgb]), HALF_EXR_CLIP_GOLDEN); +} + +/// [`exr_clip_hashes`] of [`half_project`] before (RGBA premultiplied, RGB over the background). +const HALF_EXR_CLIP_GOLDEN: [u64; 24] = [ + 0xe7f7d517df7baf25, + 0xc9d6dcb2671f9925, + 0x2fe5733c26d16325, + 0x8b267197d2645925, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0xe7f7d517df7baf25, + 0xc9d6dcb2671f9925, + 0x2fe5733c26d16325, + 0x8b267197d2645925, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0x8c261c1cd9674b25, + 0xace550c3a8e72925, + 0xb9bd0b7863b78325, + 0xdf8e948c5e8af125, + 0x6fe736332cb80f25, + 0x30d5f66395d92d25, + 0xe4aedfefacc80725, + 0x95ddfccddc885525, + 0x81fa3c39c1d09725, + 0x49cc0014f05a1d25, + 0x4ed1b417cc847725, + 0x2df6df5ce394d925, +]; + +/// An Alpha-only EXR holds the alpha (coverage) as it is in every output space: no transfer +/// curve is decoded from it (it used to be sRGB-decoded, 0.5 → 0.214). +#[test] +fn exr_alpha_only_is_raw_alpha() { + for out in [None, Some(ColorSpace::Rec709), Some(ColorSpace::Rec2100Pq), Some(ColorSpace::AcesCg)] { + for ws in [None, Some(ColorSpace::Srgb)] { + let (p, cid) = half_project(BitDepth::Bpc32, HdrMode::Clip, ws, false, out); + let d = out_dir(&format!("alpha-{ws:?}-{out:?}")); + let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, Channels::Alpha, &d, "exr")); + let _ = std::fs::remove_dir_all(&d); + let at = |x: u32| &v[((H / 2 * W + x) * 4) as usize..][..4]; + assert_eq!(at(2), [0.5, 0.5, 0.5, 1.0], "{ws:?} {out:?} covered"); + assert_eq!(at(W - 2), [0.0, 0.0, 0.0, 1.0], "{ws:?} {out:?} transparent"); + } + } } /// Compand and tone mapping leave EXR output alone: the file holds the Clip mode's samples, for @@ -215,17 +322,19 @@ fn exr_ignores_compand_and_tone_mapping() { ]; for (ws, linearize, out) in setups { for depth in DEPTHS { - for ch in [Channels::Rgba, Channels::Rgb] { + for (ch, (name, build)) in + [Channels::Rgba, Channels::Rgb].into_iter().flat_map(|c| [(c, ("bands", project as Build)), (c, ("half", half_project as Build))]) + { let samples = |mode: HdrMode| { - let (p, cid) = project(depth, mode, ws, linearize, out); - let d = out_dir(&format!("modes-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); + let (p, cid) = build(depth, mode, ws, linearize, out); + let d = out_dir(&format!("modes-{name}-{ws:?}-{linearize}-{out:?}-{depth:?}-{ch:?}-{mode:?}")); let v = exr_samples(&render(&p, cid, OutputFormat::ExrSequence, ch, &d, "exr")); let _ = std::fs::remove_dir_all(&d); v }; let clip = samples(HdrMode::Clip); for mode in [HdrMode::Compand, HdrMode::ToneMap] { - assert!(samples(mode) == clip, "{ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); + assert!(samples(mode) == clip, "{name} {ws:?} {linearize} {out:?} {depth:?} {ch:?} {mode:?}"); } } } @@ -275,6 +384,23 @@ fn exr_holds_scene_linear_values_in_output_primaries() { assert_eq!(exr_centre(&p, cid, "lin-exact")[..3], band, "4.0 is exactly 4.0"); } +/// Rec. 2100 PQ and HLG outputs (no HDR step there in any mode) are decoded with their own +/// curves: linear light in Rec. 2020 primaries, 1.0 = reference white (203 cd/m²). +#[test] +fn exr_decodes_pq_and_hlg_outputs() { + let [_, _, band] = BANDS; + let want = Conversion::new(ColorSpace::AcesCg, true, ColorSpace::Rec2020, true).expect("matrix").apply(band).map(|v| v.max(0.0)); + for out in [ColorSpace::Rec2100Pq, ColorSpace::Rec2100Hlg] { + for mode in MODES { + let (p, cid) = project(BitDepth::Bpc32, mode, Some(ColorSpace::AcesCg), true, Some(out)); + let got = exr_centre(&p, cid, &format!("hdr-out-{out:?}-{mode:?}")); + for i in 0..3 { + assert!((got[i] - want[i]).abs() <= 1e-3 * want[i].max(1.0), "{out:?} {mode:?} channel {i}: got {got:?}, want {want:?}"); + } + } + } +} + /// The EXR is decoded with the output space's curve, not always sRGB's: a Rec. 709 output (pure /// 2.4 power curve) and an unmanaged project at Working Gamma 2.4 give back the linear values. #[test] diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 4c09faa29..0d27da63d 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -1316,11 +1316,16 @@ fn owned_devices_favour_memory_usage() { assert_eq!(d.required_limits.max_texture_dimension_2d, wgpu::Limits::default().max_texture_dimension_2d); } -/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR on the GPU as on the +/// `RenderOpts::skip_hdr_step` (float outputs) skips Project Settings ▸ HDR on the GPU as on the /// CPU: both compositors agree, and the frame is the Clip frame with its over-range values. #[test] -fn scene_output_skips_the_hdr_step_on_the_gpu() { +fn skipping_the_hdr_step_on_the_gpu_matches_clip() { use effectcraft_project::HdrMode; + let Some(g) = gpu() else { + // Visible skip (EFFECTCRAFT_REQUIRE_GPU=1 makes a missing adapter fail instead). + eprintln!("skipping_the_hdr_step_on_the_gpu_matches_clip: skipped: no GPU adapter"); + return; + }; for mode in [HdrMode::Compand, HdrMode::ToneMap] { let mut s = blend_scene(BitDepth::Bpc32, BlendMode::Normal); let mut bright = s.solid([3.0, 1.5, 0.5], 30, 20); @@ -1329,9 +1334,8 @@ fn scene_output_skips_the_hdr_step_on_the_gpu() { s.p.settings.working_space = Some(ColorSpace::Srgb); s.p.settings.linearize = true; s.p.settings.hdr = mode; - let scene = RenderOpts { scene_output: true, ..opts() }; - check(&format!("scene output {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); - let Some(g) = gpu() else { continue }; + let scene = RenderOpts { skip_hdr_step: true, ..opts() }; + check(&format!("no HDR step {mode:?}"), compare_at(&s, scene, Tick::ZERO), 0.0); let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..scene }); r.accel = Some(g); let img = g.render(&r, s.cid, Tick::ZERO).expect("the GPU renders the frame"); diff --git a/crates/render/src/accel.rs b/crates/render/src/accel.rs index 613853e69..52cd2be4a 100644 --- a/crates/render/src/accel.rs +++ b/crates/render/src/accel.rs @@ -174,10 +174,11 @@ impl<'a> Renderer<'a> { self.pipe } - /// The colour pipeline of the top-level output steps: the project's, without the HDR step - /// for a float output ([`crate::RenderOpts::scene_output`]). + /// The colour pipeline of the top-level output steps: the project's, with no HDR step under + /// [`crate::RenderOpts::skip_hdr_step`] (the consumer decodes with + /// [`color::Pipe::output_linear`] or applies the step itself). pub fn output_pipe(&self) -> color::Pipe { - if self.opts.scene_output { self.pipe.scene_output() } else { self.pipe } + if self.opts.skip_hdr_step { self.pipe.without_hdr_step() } else { self.pipe } } /// Nesting depth (0 = the top-level comp). diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index 54d6d82e0..16760c95a 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -117,13 +117,13 @@ impl Pipe { Conversion::new(ws, self.linear, self.out, false) } - /// The pipe of a float output ([`crate::RenderOpts::scene_output`]): the same, without - /// the HDR step (Clip), so [`Pipe::output`] converts to the output space as it is. - pub fn scene_output(self) -> Pipe { + /// The pipe without the HDR step (Clip), as [`crate::RenderOpts::skip_hdr_step`] renders + /// with: [`Pipe::output`] converts to the output space as it is. + pub fn without_hdr_step(self) -> Pipe { Pipe { hdr: HdrMode::Clip, ..self } } - /// The top-level frame of a [`Pipe::scene_output`] render (output-encoded, straight colour) + /// The top-level frame of a [`Pipe::without_hdr_step`] render (output-encoded, straight colour) /// → linear light in the output space's primaries: the output space's curve, or the Working /// Gamma's when unmanaged (sRGB at 2.2, a pure 2.4 power curve at 2.4). The identity for a /// linear output (ACES). diff --git a/crates/render/src/opts.rs b/crates/render/src/opts.rs index ae8d3435f..352f85983 100644 --- a/crates/render/src/opts.rs +++ b/crates/render/src/opts.rs @@ -28,11 +28,11 @@ pub struct RenderOpts { pub nested_switches: bool, /// Realtime Shadows in Draft (Settings ▸ 3D): draft renders (`draft`) still cast shadows. pub draft_shadows: bool, - /// A float output (OpenEXR): the top-level comp leaves the working space for the output - /// space without Project Settings ▸ HDR (compand / tone mapping are for standard-dynamic-range - /// displays and outputs), so over-range values reach the file as they are. The writer - /// decodes the frame with [`crate::color::Pipe::output_linear`]. - pub scene_output: bool, + /// No HDR step: the top-level comp leaves the working space for the output space without + /// Project Settings ▸ HDR (compand / tone mapping), so over-range values stay. The consumer + /// decodes the frame with [`crate::color::Pipe::output_linear`] (float outputs: OpenEXR) + /// or applies the step itself. Part of the viewer's frame keys (`opts_hash`). + pub skip_hdr_step: bool, } impl RenderOpts { @@ -54,7 +54,7 @@ impl Default for RenderOpts { backend: Backend::Cpu, nested_switches: true, draft_shadows: true, - scene_output: false, + skip_hdr_step: false, proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, } } diff --git a/crates/render/src/tests/color.rs b/crates/render/src/tests/color.rs index de3ad19c6..4e5cff574 100644 --- a/crates/render/src/tests/color.rs +++ b/crates/render/src/tests/color.rs @@ -340,18 +340,18 @@ fn float_footage_is_encoded_once_with_the_working_gamma() { } } -/// A float output (`RenderOpts::scene_output`) skips Project Settings ▸ HDR: its top-level frame +/// `RenderOpts::skip_hdr_step` (float outputs) skips Project Settings ▸ HDR: its top-level frame /// is the Clip frame in every mode, while other renders keep compand / tone mapping. #[test] -fn scene_output_skips_the_hdr_step() { +fn skip_hdr_step_frames_are_the_clip_frames() { use effectcraft_project::HdrMode; for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { let (mut p, cid, _) = setup(depth); p.settings.working_space = Some(ColorSpace::Srgb); p.settings.linearize = true; add_solid(&mut p, cid, [4.0, 2.0, 0.5], BlendMode::Normal, 100.0); - let frame = |p: &Project, scene_output: bool| { - Renderer::new(p, &crate::NoFootage, RenderOpts { scene_output, ..Default::default() }).comp_frame(cid, Tick::ZERO) + let frame = |p: &Project, skip_hdr_step: bool| { + Renderer::new(p, &crate::NoFootage, RenderOpts { skip_hdr_step, ..Default::default() }).comp_frame(cid, Tick::ZERO) }; let clip = frame(&p, false); assert_eq!(clip.data, frame(&p, true).data, "{depth:?} Clip"); diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index 241951c54..7277a7cb5 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -164,7 +164,7 @@ impl EffectcraftApp { backend: effectcraft_engine::render::Backend::Auto, nested_switches: self.session.prefs.general.switches_affect_nested_comps, draft_shadows: self.session.prefs.three_d.realtime_shadows, - scene_output: false, + skip_hdr_step: false, proxy: Default::default(), } } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index 77584e2df..3716db953 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -6,12 +6,12 @@ fn converting_a_frame_with_nan_alpha_keeps_the_viewer_alive() { assert_eq!(to_color_image(&image).pixels, [egui::Color32::TRANSPARENT]); } -/// A float output's frames (`RenderOpts::scene_output`, no HDR step) never share a RAM or disk +/// Frames without the HDR step (`RenderOpts::skip_hdr_step`) never share a RAM or disk /// cache key with the HDR-mapped frames of the same options. #[test] -fn scene_output_frames_have_keys_of_their_own() { +fn frames_without_the_hdr_step_have_keys_of_their_own() { let opts = RenderOpts::default(); - assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { scene_output: true, ..opts })); + assert_ne!(opts_hash(&opts), opts_hash(&RenderOpts { skip_hdr_step: true, ..opts })); } struct ProbeDisk { diff --git a/docs/architecture.md b/docs/architecture.md index 226f93a14..48f3ad442 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -409,9 +409,10 @@ Interpret Footage, else sRGB) and the top-level comp is converted to the sRGB di Working Space runs everything in linear light; Blend Colors Using 1.0 Gamma linearises only for blending. Layer cache keys include these settings. Project Settings ▸ HDR (compand / tone mapping) is the last top-level step, for standard-dynamic-range displays and outputs; OpenEXR -renders (sequences and ProEXR) skip it (`RenderOpts::scene_output`) and decode the frame with the +renders (sequences and ProEXR) skip it (`RenderOpts::skip_hdr_step`) and decode the frame with the output space's own curve (`Pipe::output_linear`), so they hold linear light in the output space's -primaries. +primaries; unmanaged projects decode with the Working Gamma's curve (sRGB at 2.2, a pure 2.4 power +curve at 2.4). Unmanaged projects and Rec. 2100 PQ / HLG outputs never get the HDR step. **Frame blending**: footage and precomp layers whose source time falls between source frames (rate conform, stretch, remap) blend the two neighbouring frames, by cross-fade (Frame Mix) or by From 8add8ac7b401634fc2af0c6d2cde9ce96d1d3f08 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 00:51:59 +0100 Subject: [PATCH 04/35] #474: decode a multi-layer EXR once per frame, and only its picture when nothing else reads it - The channel cache decodes a file once however many callers ask for it at the same time (the others wait for that decode), and is bounded by the bytes of its planes (1 GiB, least recently used out first, a file counting as used when its decode finishes, the newest file always kept) instead of four files of any size. - Once an effect has read a footage item's channels, its pictures are made of the cached planes, also for files with an unnamed RGB layer (bit for bit what the image crate shows, which still decodes the files the planes can't stand in for); footage alone decodes only its picture's channels (a multi-layer file's were all decoded and cached for every frame). - Channels are decoded straight into their display-window planes, and the picture is converted from the planes without two interleaved copies. Measured on a 30-frame 2560x1440 sequence of 32 channels (DWAA), one frame at a time: footage alone 0.185 -> 0.073 s per frame and 4.4 -> 1.8 GB peak; with an unnamed RGB layer plus Cryptomatte 0.244 -> 0.159 s; multi-part plus EXtractoR 0.194 -> 0.127 s. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/convert.rs | 52 +- crates/media/src/exr_channels.rs | 816 +++++++++++++++++++++++++------ crates/media/src/pool.rs | 30 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 5 + docs/gaps.md | 2 +- 7 files changed, 742 insertions(+), 167 deletions(-) diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 9e88ab4dd..a2b24b3ae 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -376,30 +376,20 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image { use image::DynamicImage as D; let (w, h) = (img.width(), img.height()); + if let D::ImageRgb32F(_) | D::ImageRgba32F(_) = img { + let px = img.to_rgba32f(); + let raw = px.as_raw(); + return float_to_image(w, h, op, |i| { + let s = &raw[i * 4..i * 4 + 4]; + [s[0], s[1], s[2], s[3]] + }); + } let mut out = Image::new(w, h); let wu = w as usize; if wu == 0 || h == 0 { return out; } match img { - D::ImageRgb32F(_) | D::ImageRgba32F(_) => { - let px = img.to_rgba32f(); - let raw = px.as_raw(); - let premul = op.mode == AlphaMode::Premultiplied; - out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { - for (x, o) in row.iter_mut().enumerate() { - let s = &raw[(y * wu + x) * 4..(y * wu + x) * 4 + 4]; - let a = s[3]; - let c = if premul { - let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; - [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] - } else { - [op.float(s[0]), op.float(s[1]), op.float(s[2])] - }; - *o = op.apply(c, a.clamp(0.0, 1.0)); - } - }); - } D::ImageLuma16(_) | D::ImageLumaA16(_) | D::ImageRgb16(_) | D::ImageRgba16(_) => { let px = img.to_rgba16(); let raw = px.as_raw(); @@ -425,6 +415,32 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Float RGBA (linear light, alpha straight or premultiplied as the footage says), `px(i)` for +/// pixel `i` in rows, to an [`Image`]: how [`dynamic_to_image`] converts float stills, also used +/// for OpenEXR pictures made from decoded channel planes without an interleaved copy (#474). +pub(crate) fn float_to_image(w: u32, h: u32, op: AlphaOp, px: impl Fn(usize) -> [f32; 4] + Sync) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + let premul = op.mode == AlphaMode::Premultiplied; + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px(y * wu + x); + let a = s[3]; + let c = if premul { + let inv = if a > 0.0 { 1.0 / a } else { 0.0 }; + [op.float(s[0] * inv) * a, op.float(s[1] * inv) * a, op.float(s[2] * inv) * a] + } else { + [op.float(s[0]), op.float(s[1]), op.float(s[2])] + }; + *o = op.apply(c, a.clamp(0.0, 1.0)); + } + }); + out +} + /// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { let mut out = Image::new(w, h); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index d372ec849..747882337 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -6,12 +6,25 @@ //! 0, or the background depth for depth channels). Cryptomatte manifests come from the //! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published //! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). +//! +//! Decoded files are kept in a small cache ([`CACHE_BUDGET`]) that the picture of an EXR +//! footage item shares with EXtractoR / Cryptomatte once an effect has read its channels +//! (#474); a file is decoded once however many callers ask for it at the same time. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; -use effectcraft_raster::AuxChannels; use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, split_channel}; +use effectcraft_raster::{AuxChannels, Image}; +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::attribute::SampleType; +use exr::meta::header::Header; + +use crate::convert::{AlphaOp, float_to_image}; /// The largest display window decoded (16384²; vector footage and Continuously Rasterize /// stop at the same size). @@ -33,6 +46,11 @@ fn channel_name(part: Option<&str>, channel: &str) -> String { } } +/// The part name channels are prefixed with (`None`: unnamed). +fn part_name(header: &Header) -> Option { + header.own_attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()) +} + /// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; @@ -53,23 +71,209 @@ pub fn unsupported_compression(bytes: &[u8]) -> Option<&'static str> { .then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression") } -/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. -pub fn read_exr_channels(bytes: &[u8]) -> Option { - use exr::prelude::*; - let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; - let dw = img.attributes.display_window; - let (w, h) = (dw.size.0, dw.size.1); - if w == 0 || h == 0 { - return None; +/// Which channels a decode keeps, by plane name, and the display window they are placed in +/// (the first part's, as the `exr` crate reports the image's). +struct Planes<'k> { + keep: &'k dyn Fn(&str) -> bool, + display: exr::meta::attribute::IntegerBounds, +} + +/// One part's kept channels while it is decoded: each line of samples goes straight into its +/// channel's display-window plane (no copy of the part's samples in their own type first). +struct PlanesReader { + /// Per channel of the part: its plane name, sample type and plane (`None`: not kept). + planes: Vec)>>, + /// The data window's position in the display window, and the data window's and display + /// window's sizes. + offset: (i64, i64), + layer: (usize, usize), + display: (usize, usize), + f16s: Vec, + u32s: Vec, + f32s: Vec, + /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. + rgb: Option<([usize; 3], Option)>, + /// The part has a channel that doesn't sample every pixel. + subsampled: bool, + /// The part's display window differs from the first part's. + own_display: bool, +} + +/// A decoded part: its kept planes (in channel order) and what [`PlanesReader`] found. +struct PartPlanes { + planes: Vec<(String, Vec)>, + /// Positions in `planes` of the part's R, G, B (and A) channels, when it has them and keeps + /// them all. + rgb: Option<([usize; 3], Option)>, + has_rgb: bool, + exact: bool, +} + +impl<'s> ReadChannels<'s> for Planes<'_> { + type Reader = PlanesReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + if header.deep { + return Err(exr::error::Error::NotSupported("deep data".into())); + } + let dw = self.display; + let display = (dw.size.0, dw.size.1); + // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a + // multi-gigabyte allocation per channel (#2141). + let n = display.0.checked_mul(display.1).filter(|n| *n > 0 && *n <= MAX_DISPLAY_PIXELS).ok_or(exr::error::Error::Invalid("display window".into()))?; + let prefix = part_name(header); + let pos = header.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); + let planes = header + .channels + .list + .iter() + .map(|ch| { + let name = channel_name(prefix.as_deref(), &ch.name.to_string()); + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + (self.keep)(&name).then(|| (name, ch.sample_type, vec![fill; n])) + }) + .collect(); + let find = |c: &str| header.channels.find_index_of_channel(&exr::meta::attribute::Text::from(c)); + let rgb = match (find("R"), find("G"), find("B")) { + (Some(r), Some(g), Some(b)) => Some(([r, g, b], find("A"))), + _ => None, + }; + Ok(PlanesReader { + planes, + offset, + layer: (header.layer_size.0, header.layer_size.1), + display, + f16s: vec![], + u32s: vec![], + f32s: vec![], + rgb, + subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), + own_display: header.shared_attributes.display_window != dw, + }) } - let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); +} + +impl ChannelsReader for PlanesReader { + type Channels = PartPlanes; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + let ((lw, lh), (w, h)) = (self.layer, self.display); + for line in block.lines(&header.channels) { + let Some(Some((_, kind, plane))) = self.planes.get_mut(line.location.channel) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` + // from display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + self.offset.1).ok().filter(|ty| *ty < h) else { continue }; + let tx0 = x0 as i64 + self.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + let start = ty.checked_mul(w).and_then(|row| row.checked_add(tx)); + let Some(dst) = start.filter(|_| take > 0).and_then(|s| plane.get_mut(s..s.checked_add(take)?)) else { continue }; + match kind { + SampleType::F16 => { + self.f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut self.f16s)?; + for (d, s) in dst.iter_mut().zip(self.f16s.iter().skip(skip)) { + *d = s.to_f32(); + } + } + SampleType::F32 => { + self.f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut self.f32s)?; + for (d, s) in dst.iter_mut().zip(self.f32s.iter().skip(skip)) { + *d = *s; + } + } + SampleType::U32 => { + self.u32s.resize(count, 0); + line.read_samples_into_slice(&mut self.u32s)?; + for (d, s) in dst.iter_mut().zip(self.u32s.iter().skip(skip)) { + *d = *s as f32; + } + } + } + } + Ok(()) + } + + fn into_channels(self) -> PartPlanes { + // Positions of the kept channels among the kept planes. + let mut at = vec![None; self.planes.len()]; + let mut planes = Vec::with_capacity(self.planes.len()); + for (i, p) in self.planes.into_iter().enumerate() { + if let Some((name, _, data)) = p { + if let Some(slot) = at.get_mut(i) { + *slot = Some(planes.len()); + } + planes.push((name, data)); + } + } + let kept = |i: usize| at.get(i).copied().flatten(); + let has_rgb = self.rgb.is_some(); + let rgb = self.rgb.and_then(|([r, g, b], a)| { + let a = match a { + Some(a) => Some(kept(a)?), + None => None, + }; + Some(([kept(r)?, kept(g)?, kept(b)?], a)) + }); + PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + } +} + +/// How the `image` crate's OpenEXR decoder would show a file: the first part with exact `R`, `G` +/// and `B` channels (alpha `A` when it has one, else opaque), placed in the display window with +/// 0 outside the data window. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Rgb { + /// No part has R, G and B: the decoder refuses the file (a multi-layer file, shown with + /// [`layered_image`]'s rule). + None, + /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. + Planes([usize; 3], Option), + /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another + /// display window): decode with the `image` crate. + Unsupported, +} + +/// A decoded file: its channels and how its picture is made of them. +pub(crate) struct Decoded { + pub(crate) aux: Arc, + rgb: Rgb, +} + +impl Decoded { + /// Bytes held (the planes; names and manifests are small). + fn bytes(&self) -> usize { + self.aux.channels.iter().map(|(n, p)| n.len() + std::mem::size_of_val(p.as_slice())).sum() + } +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it +/// is not a readable EXR. +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let display = meta.headers.first()?.shared_attributes.display_window; + let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); + let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); - let mut collect_meta = |other: &HashMap| { + let mut collect_meta = |other: &HashMap| { for (k, v) in other { let key = k.to_string(); let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; let Some((id, field)) = rest.split_once('/') else { continue }; - let AttributeValue::Text(t) = v else { continue }; + let exr::meta::attribute::AttributeValue::Text(t) = v else { continue }; let e = crypto.entry(id.to_string()).or_default(); match field { "name" => e.0 = Some(t.to_string()), @@ -79,24 +283,22 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } }; collect_meta(&img.attributes.other); - // Each channel is a display-window plane: a 60000² window over a 1×1 data window was a - // multi-gigabyte allocation per channel (#2141). - let n = w.checked_mul(h).filter(|n| *n <= MAX_DISPLAY_PIXELS)?; - // The decoded image is consumed channel by channel, so each channel's samples are freed (or - // moved) as its plane is made instead of the whole file being held twice (#474). + let mut rgb = Rgb::None; for layer in img.layer_data { collect_meta(&layer.attributes.other); - let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); - let size = (layer.size.0, layer.size.1); - let pos = layer.attributes.layer_position; - let offset = (i64::from(pos.0) - i64::from(dw.position.0), i64::from(pos.1) - i64::from(dw.position.1)); - for ch in layer.channel_data.list { - let name = channel_name(prefix.as_deref(), &ch.name.to_string()); - let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; - if let Some(plane) = place(ch.sample_data, size, offset, (w, h), n, fill) { - aux.channels.push((name, plane)); - } + let part = layer.channel_data; + if rgb == Rgb::None && part.has_rgb { + let base = aux.channels.len(); + rgb = match part.rgb { + Some(([r, g, b], a)) if part.exact => { + let picked = [Some(r), Some(g), Some(b), a]; + let depth = picked.iter().flatten().any(|i| part.planes.get(*i).is_some_and(|(n, _)| is_depth(n))); + if depth { Rgb::Unsupported } else { Rgb::Planes([base + r, base + g, base + b], a.map(|a| base + a)) } + } + _ => Rgb::Unsupported, + }; } + aux.channels.extend(part.planes); } for (_, (name, manifest)) in crypto { if let (Some(n), Some(m)) = (name, manifest) { @@ -104,46 +306,13 @@ pub fn read_exr_channels(bytes: &[u8]) -> Option { } } aux.manifests.sort(); - Some(aux) + Some(Decoded { aux: Arc::new(aux), rgb }) } -/// One channel's samples (`size`, at `offset` in the `display` window of `n` pixels) as a -/// display-window plane, `fill` outside its data window. A channel covering exactly the display -/// window is converted in one pass, and an f32 one is moved without a copy. `None` when the -/// channel has fewer samples than its size. -fn place(samples: exr::image::FlatSamples, size: (usize, usize), offset: (i64, i64), display: (usize, usize), n: usize, fill: f32) -> Option> { - use exr::image::FlatSamples; - let (lw, lh) = size; - if samples.len() < lw.checked_mul(lh)? { - return None; - } - if size == display && offset == (0, 0) { - return Some(match samples { - FlatSamples::F32(mut v) => { - v.truncate(n); - v - } - FlatSamples::F16(v) => v.iter().take(n).map(|s| s.to_f32()).collect(), - FlatSamples::U32(v) => v.iter().take(n).map(|s| *s as f32).collect(), - }); - } - let value = |i: usize| match &samples { - FlatSamples::F32(v) => v.get(i).copied(), - FlatSamples::F16(v) => v.get(i).map(|s| s.to_f32()), - FlatSamples::U32(v) => v.get(i).map(|s| *s as f32), - }; - let (w, h) = display; - let mut plane = vec![fill; n]; - for y in 0..lh { - let Some(ty) = usize::try_from(y as i64 + offset.1).ok().filter(|ty| *ty < h) else { continue }; - for x in 0..lw { - let Some(tx) = usize::try_from(x as i64 + offset.0).ok().filter(|tx| *tx < w) else { continue }; - if let (Some(v), Some(p)) = (value(y * lw + x), plane.get_mut(ty * w + tx)) { - *p = v; - } - } - } - Some(plane) +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + let aux = decode(bytes, &|_| true)?.aux; + Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } /// Names of the beauty pass, most preferred first, matched against the last part of a layer's @@ -186,6 +355,36 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { .copied() } +/// The planes a multi-layer file's picture is made of (see [`layered_image`]): red, green, blue +/// (one channel thrice for grey) and alpha, by name. Only the names of `aux` are read. +fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { + let layers = aux.layers(); + let colour = picture_layer(aux, &layers)?; + let names = aux.layer_rgba(colour); + // A layer without R, G and B channels shows its first channel as grey. + let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); + let pick = |k: usize| names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first).to_string(); + let a = names.get(3).filter(|n| !n.is_empty() && aux.get(n).is_some()).cloned(); + Some(([pick(0), pick(1), pick(2)], a)) +} + +/// The channels of the picture of a multi-layer file and nothing else (names first from the +/// headers, then only those channels decoded; in a multi-part file the other parts' pixels are +/// not even decompressed), with the names of its planes. +fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; + let mut names = AuxChannels::new(0, 0, 1.0); + for header in meta.headers.iter() { + let prefix = part_name(header); + names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); + } + let (rgb, a) = picture_names(&names)?; + // Planes are looked up by name ignoring case, so every channel that could answer is kept. + let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); + let decoded = decode(bytes, &keep)?; + Some((decoded, rgb, a)) +} + /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's /// `ViewLayer.Combined.R`, the compositor's `Image.R`, Nuke's `beauty.R`, …), which the image /// decoder rejects. After Effects shows a file's main RGBA channels and leaves the other layers @@ -193,26 +392,13 @@ fn picture_layer<'a>(aux: &AuxChannels, layers: &[&'a str]) -> Option<&'a str> { /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - picture_of(&read_exr_channels(bytes)?) -} - -/// [`layered_image`] of the EXR at `path` from the channel cache, so the picture and EXtractoR / -/// Cryptomatte share one decode of the file (#474). -pub(crate) fn cached_layered_image(path: &str, bytes: Arc<[u8]>) -> Option { - let aux = cached(path, || Some(bytes))?; - picture_of(&aux) -} - -/// The picture of decoded channels (see [`layered_image`]). -fn picture_of(aux: &AuxChannels) -> Option { - let layers = aux.layers(); - let colour = picture_layer(aux, &layers)?; - let names = aux.layer_rgba(colour); - // A layer without R, G and B channels shows its first channel as grey. - let first = aux.names().into_iter().find(|n| split_channel(n).0 == colour).unwrap_or_default(); - let plane = |k: usize| -> Option<&[f32]> { aux.get(names.get(k).map(String::as_str).filter(|n| !n.is_empty()).unwrap_or(first)) }; - let (r, g, b) = (plane(0)?, plane(1)?, plane(2)?); - let a = names.get(3).filter(|n| !n.is_empty()).and_then(|n| aux.get(n)); + let (d, rgb, a) = picture_only(bytes)?; + let aux = &d.aux; + let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; let n = (aux.width as usize).checked_mul(aux.height as usize)?; let mut data = Vec::with_capacity(n.checked_mul(4)?); for i in 0..n { @@ -227,40 +413,214 @@ fn picture_of(aux: &AuxChannels) -> Option { } } -type Cache = Mutex>)>>; +/// The picture made of planes `rgb` and `a` (opaque without one) of `aux`, as +/// [`crate::convert::dynamic_to_image`] makes it of the same values. `None` when a plane is +/// missing or short. +fn planes_picture(aux: &AuxChannels, rgb: [&[f32]; 3], a: Option<&[f32]>, op: AlphaOp) -> Option { + let n = (aux.width as usize).checked_mul(aux.height as usize)?; + if rgb.iter().chain(&a).any(|p| p.len() < n) { + return None; + } + let [r, g, b] = rgb; + let at = |p: &[f32], i: usize| p.get(i).copied().unwrap_or(0.0); + Some(float_to_image(aux.width, aux.height, op, |i| [at(r, i), at(g, i), at(b, i), a.map_or(1.0, |a| at(a, i))])) +} -fn cache() -> &'static Cache { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| Mutex::new(Vec::new())) +/// The picture of a decoded file by [`layered_image`]'s rule. +fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { + let (rgb, a) = picture_names(aux)?; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: +/// from the channel cache when `shared` (an effect reads the footage's channels, so they are +/// decoded anyway), else of the picture's channels alone. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { + if shared { + let d = cache().get(path, || Some(bytes))?; + return layered_picture_of(&d.aux, op); + } + let (d, rgb, a) = picture_only(&bytes)?; + let aux = &d.aux; + let a = match &a { + Some(n) => Some(aux.get(n)?), + None => None, + }; + planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) +} + +/// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read +/// this footage's channels, so the picture shares their decode): what the `image` crate's +/// decoder shows for a file it reads, [`layered_image`]'s rule for one it refuses. `None` when +/// the planes can't stand in for the `image` crate's picture (decode with it), or the file is +/// not a readable EXR. +pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { + let d = cache().get(path, || Some(bytes))?; + let aux = &d.aux; + match d.rgb { + Rgb::None => layered_picture_of(aux, op), + Rgb::Planes([r, g, b], a) => { + let plane = |i: usize| aux.channels.get(i).map(|(_, p)| p.as_slice()); + let a = match a { + Some(a) => Some(plane(a)?), + None => None, + }; + planes_picture(aux, [plane(r)?, plane(g)?, plane(b)?], a, op) + } + Rgb::Unsupported => None, + } +} + +/// How many bytes of decoded channels the cache keeps, like the frame cache's default +/// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane +/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects +/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak +/// memory about 1 GB lower than 2 GiB, at the same speed.) +#[cfg(not(target_arch = "wasm32"))] +pub const CACHE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const CACHE_BUDGET: usize = 512 << 20; + +/// At most this many files are cached (unreadable ones hold no bytes). +const CACHE_FILES: usize = 64; + +/// A file's decode: done once, by the first caller; the others wait for it. +type Slot = Arc>>>; + +struct Entry { + path: String, + slot: Slot, + /// Bytes of the finished decode (0 while decoding). + bytes: usize, + /// When it was last asked for (a counter). + used: u64, +} + +/// Decoded EXR files by path, bounded by bytes, least recently used out first. +pub(crate) struct ChannelCache { + entries: Mutex<(Vec, u64)>, + budget: usize, + /// Files decoded (for tests). + decodes: AtomicU64, +} + +impl ChannelCache { + pub(crate) const fn new(budget: usize) -> Self { + Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + } + + /// The decoded file at `path`, read with `read` by the first caller; callers asking for the + /// same file meanwhile wait for that decode instead of decoding it again. + pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + let slot = { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + match entries.iter_mut().find(|e| e.path == path) { + Some(e) => { + e.used = *clock; + e.slot.clone() + } + None => { + let slot = Slot::default(); + entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + slot + } + } + }; + let mut decoded_here = false; + let v = slot + .get_or_init(|| { + decoded_here = true; + self.decodes.fetch_add(1, Ordering::Relaxed); + read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + }) + .clone(); + if decoded_here { + let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); + let (entries, clock) = &mut *guard; + *clock += 1; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for + // before another but decoded after it is not the older one, or concurrent frames + // would evict each other's files before their effects read them. + if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { + e.bytes = v.as_ref().map_or(0, |d| d.bytes()); + e.used = *clock; + } + Self::evict(entries, self.budget, &slot); + } + v + } + + /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { + while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { + let oldest = entries + .iter() + .enumerate() + .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) + .min_by_key(|(_, e)| e.used) + .map(|(i, _)| i); + let Some(i) = oldest else { break }; + entries.remove(i); + } + } + + pub(crate) fn forget(&self, path: &str) { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + } + + #[cfg(test)] + fn decodes(&self) -> u64 { + self.decodes.load(Ordering::Relaxed) + } + + #[cfg(test)] + fn cached(&self) -> Vec { + self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + } +} + +fn cache() -> &'static ChannelCache { + static CACHE: ChannelCache = ChannelCache::new(CACHE_BUDGET); + &CACHE } /// Drop the cached channels of `path` (the file changed on disk). pub(crate) fn forget(path: &str) { - if let Ok(mut c) = cache().lock() { - c.retain(|(p, _)| p != path); - } + cache().forget(path); } -/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +/// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - let cache = cache(); - if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { - return v; - } - let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); - if let Ok(mut c) = cache.lock() { - if c.len() >= 4 { - c.remove(0); - } - c.push((path.to_string(), v.clone())); - } - v + cache().get(path, read).map(|d| d.aux.clone()) } #[cfg(test)] mod tests { use super::*; + /// An EXR file of `layers` (with their display window `display`). + fn file(layers: Vec>>, display: exr::prelude::IntegerBounds) -> Vec { + use exr::prelude::*; + let image = Image::from_layers(ImageAttributes::new(display), layers); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() + } + + /// A single-part file of float channels named `names`, `w`×`h`, every sample `v`. + fn flat(names: &[&str], (w, h): (usize, usize), v: f32) -> Vec { + use exr::prelude::*; + let list: Vec> = names.iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![v; w * h]))).collect(); + let layer = Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + file(vec![layer], IntegerBounds::from_dimensions((w, h))) + } + #[test] fn reads_layered_channels_and_cryptomatte_metadata() { use exr::prelude::*; @@ -294,53 +654,229 @@ mod tests { assert!(read_exr_channels(b"not an exr").is_none()); } - /// #474: a full-window f32 channel (Cryptomatte ranks, …) becomes its plane without a copy, - /// so a decode no longer holds the file's samples twice; other channels are placed in the - /// display window as before. + /// #474: each line of samples goes straight into its display-window plane: a data window + /// inside the display window is placed with the fill around it (the background depth for + /// depth), one partly outside is clipped, never out of bounds; half and uint samples are + /// converted as the file stores them. #[test] - fn channels_become_planes_without_an_extra_copy() { - use exr::image::FlatSamples; - let v = vec![0.25f32; 6]; - let ptr = v.as_ptr(); - let plane = place(FlatSamples::F32(v), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(plane.as_ptr(), ptr, "moved, not copied"); - assert_eq!(plane, [0.25; 6]); - let half = |x: f32| exr::prelude::f16::from_f32(x); - let full = place(FlatSamples::F16(vec![half(0.5); 6]), (3, 2), (0, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(full, [0.5; 6]); - // A 2×1 data window at (1, 1) of a 3×2 display window; the rest is the fill. - let part = place(FlatSamples::F16(vec![half(1.0), half(2.0)]), (2, 1), (1, 1), (3, 2), 6, 9.0).unwrap(); - assert_eq!(part, [9.0, 9.0, 9.0, 9.0, 1.0, 2.0]); - // Partly outside the display window: clipped, never out of bounds. - let off = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, -1), (3, 2), 6, 0.0).unwrap(); - assert_eq!(off, [0.0; 6]); - let edge = place(FlatSamples::U32(vec![7, 8]), (2, 1), (2, 0), (3, 2), 6, 0.0).unwrap(); - assert_eq!(edge, [0.0, 0.0, 7.0, 0.0, 0.0, 0.0]); - assert!(place(FlatSamples::F32(vec![1.0; 5]), (3, 2), (0, 0), (3, 2), 6, 0.0).is_none(), "too few samples"); + fn channels_are_placed_in_the_display_window() { + use exr::prelude::*; + let half = |x: f32| f16::from_f32(x); + let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = LayerAttributes::default(); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::from_dimensions((3, 2)); + // A 2×1 data window at (1, 1). + let inside = file( + vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + display, + ); + let aux = read_exr_channels(&inside).unwrap(); + assert_eq!(aux.get("G").unwrap(), [0.0, 0.0, 0.0, 0.0, 1.0, 2.0]); + let bg = BACKGROUND_DEPTH; + assert_eq!(aux.get("Z").unwrap(), [bg, bg, bg, bg, 7.0, 8.0]); + // A 2×2 data window at (2, -1): only its bottom-left sample lands at (2, 0). + let off = file(vec![part((2, -1), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0, 2.0, 3.0, 4.0]))])], display); + assert_eq!(read_exr_channels(&off).unwrap().get("G").unwrap(), [0.0, 0.0, 3.0, 0.0, 0.0, 0.0]); + // Entirely outside: all fill. + let away = file(vec![part((-5, 0), (2, 2), vec![AnyChannel::new("G", FlatSamples::F32(vec![1.0; 4]))])], display); + assert_eq!(read_exr_channels(&away).unwrap().get("G").unwrap(), [0.0; 6]); + } + + /// #474: concurrent requests for a file that isn't cached decode it once; the others wait + /// for that decode and share its planes. + #[test] + fn concurrent_requests_decode_a_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let reads = AtomicU64::new(0); + let barrier = std::sync::Barrier::new(8); + let got: Vec> = std::thread::scope(|s| { + let jobs: Vec<_> = (0..8) + .map(|_| { + s.spawn(|| { + barrier.wait(); + cache + .get("/474-concurrent.exr", || { + reads.fetch_add(1, Ordering::Relaxed); + // Long enough for the others to arrive while this one decodes. + std::thread::sleep(std::time::Duration::from_millis(50)); + Some(bytes.clone()) + }) + .unwrap() + }) + }) + .collect(); + jobs.into_iter().map(|j| j.join().unwrap()).collect() + }); + assert_eq!(cache.decodes(), 1, "one decode"); + assert_eq!(reads.load(Ordering::Relaxed), 1, "the file is read once"); + assert!(got.iter().all(|d| Arc::ptr_eq(d, &got[0])), "every caller has the same planes"); + assert_eq!(got[0].aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + + /// #474: the cache is bounded by the bytes of its planes, least recently used out first; the + /// newest file stays even when it alone is over the budget, and unreadable files are cached + /// (as such) without counting bytes. + #[test] + fn cache_evicts_least_recently_used_by_bytes() { + // 4 float planes of 32×32: 16 KiB of samples (plus a few bytes of names) per file. + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + assert_eq!(get("/a.exr").bytes(), one); + get("/b.exr"); + get("/a.exr"); // a is now more recent than b + get("/c.exr"); + assert_eq!(cache.cached(), ["/a.exr", "/c.exr"], "b was least recently used"); + assert_eq!(cache.decodes(), 3); + get("/a.exr"); + assert_eq!(cache.decodes(), 3, "a hit"); + // A file over the budget on its own is still kept while it is the newest. + let small = ChannelCache::new(one / 2); + assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert_eq!(small.cached(), ["/big.exr"]); + small.get("/big2.exr", || Some(bytes.clone())); + assert_eq!(small.cached(), ["/big2.exr"]); + // An unreadable file is remembered (not read again) without bytes, and forgotten on request. + assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + cache.forget("/broken.exr"); + assert!(!cache.cached().contains(&"/broken.exr".to_string())); + // At most CACHE_FILES files, however small. + for i in 0..CACHE_FILES + 10 { + cache.get(&format!("/none{i}.exr"), || None); + } + assert!(cache.cached().len() <= CACHE_FILES); + } + + /// #474: a file counts as used when its decode finishes: one asked for first but decoded + /// after another is the newer of the two, so concurrent frames don't evict each other's + /// files before their effects read them. + #[test] + fn a_file_is_used_as_of_its_decode() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); + let one = 4 * 32 * 32 * 4 + 4; + let cache = ChannelCache::new(2 * one); + cache.get("/first.exr", || { + // Another frame's file is asked for and decoded while this one decodes. + cache.get("/second.exr", || Some(bytes.clone())); + Some(bytes.clone()) + }); + cache.get("/third.exr", || Some(bytes.clone())); + assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); + } + + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when + /// that decode finishes. + #[test] + fn a_file_forgotten_while_it_decodes_is_not_cached() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let d = cache.get("/changed.exr", || { + cache.forget("/changed.exr"); + Some(bytes.clone()) + }); + assert!(d.is_some()); + assert!(cache.cached().is_empty()); } /// #295: a multi-layer EXR whose colour is only in named layers (`diffuse.R`, as Blender and /// Nuke write them) failed to import ("does not contain non-deep rgb channels"); it now /// shows its colour layer and keeps every channel for EXtractoR. - /// #474: the picture and EXtractoR / Cryptomatte share one decode of a multi-layer file: - /// after the frame is decoded, the aux channels come from the cache without reading the file. + /// #474: once an effect has read a footage item's channels, the picture of each later frame + /// shares one decode with them: the frame's decode is cached and the channels come from it + /// without reading the file again. #[test] fn picture_and_aux_channels_share_one_decode() { use exr::prelude::*; let (w, h) = (2usize, 2usize); - let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); - let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", 0.25)].into()); - let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); - let mut bytes = std::io::Cursor::new(Vec::new()); - image.write().to_buffered(&mut bytes).unwrap(); - let b: Arc<[u8]> = bytes.into_inner().into(); - let path = "/474-shared-decode.exr"; + let frame = |v: f32| -> Arc<[u8]> { + let ch = |n: &str, v: f32| AnyChannel::new(n, FlatSamples::F32(vec![v; w * h])); + let channels = AnyChannels::sort(vec![ch("Image.R", 0.5), ch("Image.G", 0.5), ch("Image.B", 0.5), ch("GlossDir.R", v)].into()); + let image = Image::from_layer(Layer::new((w, h), LayerAttributes::default(), Encoding::FAST_LOSSLESS, channels)); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + bytes.into_inner().into() + }; + let paths = ["/474-shared-decode_0001.exr", "/474-shared-decode_0002.exr"]; let pool = crate::MediaPool::new(); - pool.add_bytes(path, b.clone()); - let f = crate::probe_bytes(path, b).unwrap(); + pool.add_bytes(paths[0], frame(0.25)); + pool.add_bytes(paths[1], frame(0.75)); + let mut f = crate::probe_bytes(paths[0], frame(0.25)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.iter().map(|p| p.to_string()).collect(); + let t1 = f.frame_rate.tick_of(1); + use effectcraft_render::FootageSource; + // The first frame: the picture, then an effect reads the channels. pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap(); - let aux = cached(path, || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f, effectcraft_time::Tick::ZERO).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + // The next frame's picture decodes the file into the cache, so its channels are not read. + let px = pool.frame_at(&f, t1).unwrap().get(0, 0); + let aux = cached(paths[1], || panic!("the frame's decode is reused, the file is not read again")).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.75)); + assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); + } + + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the + /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 + /// around it), in every alpha interpretation and float curve; a file it can't stand in for + /// (a subsampled channel) is left to the `image` crate. + #[test] + fn picture_from_planes_is_the_image_crates() { + use effectcraft_project::{AlphaMode, Footage}; + use effectcraft_render::FloatCurve; + use exr::prelude::*; + let half = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F16((0..len).map(|i| f16::from_f32(v + i as f32 * 0.37 - 1.0)).collect())); + let float = |n: &str, v: f32, len: usize| AnyChannel::new(n, FlatSamples::F32((0..len).map(|i| ((i * 7) % 11) as f32 * v).collect())); + let part = |name: Option<&str>, pos: (i32, i32), size: (usize, usize), list: Vec>| { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(pos.0, pos.1); + Layer::new(size, attrs, Encoding::SMALL_LOSSLESS, AnyChannels::sort(list.into())) + }; + let display = IntegerBounds::new((1, -1), (5, 4)); + let n = 20; + let files = [ + file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), + file( + vec![ + part(Some("diffuse"), (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n)]), + part(Some("beauty"), (2, 0), (2, 2), vec![half("R", 0.4, 4), half("G", 0.5, 4), half("B", 0.6, 4), half("A", 0.5, 4)]), + ], + display, + ), + ]; + let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + for (k, bytes) in files.iter().enumerate() { + let path = format!("/474-planes-{k}.exr"); + let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); + let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); + for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + for curve in [FloatCurve::Srgb, FloatCurve::Linear, FloatCurve::Gamma24] { + let op = AlphaOp::of(&Footage { alpha, premul_color: [0.2, 0.4, 0.6], ..Default::default() }, curve); + let want = crate::convert::dynamic_to_image(&img, op); + let got = cached_picture(&path, bytes.as_slice().into(), op).unwrap(); + assert_eq!((got.width, got.height), (want.width, want.height)); + assert_eq!(bits(&got), bits(&want), "file {k} {alpha:?} {curve:?}"); + } + } + } + // A subsampled channel in the picture's part: the `image` crate decodes it. + let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); + if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { + c.sampling = Vec2(2, 2); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { + let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); + assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); + } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 655ca4ed8..9f1f2aa23 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,6 +245,9 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, + /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// Channel effects): its pictures come from the same cached decode (#474). + aux_footage: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -315,6 +318,7 @@ impl MediaPool { conforming: Mutex::default(), conform_failed: Mutex::default(), problems: Mutex::default(), + aux_footage: Mutex::default(), }), } } @@ -856,13 +860,19 @@ impl Inner { { return Err(MediaError::Decode(format!("{path}: {why}"))); } - let loaded = crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)); - let img = match loaded { - Ok(img) => img, + // An effect reads this footage's channels: the picture is made of the same decode (#474). + let shared = lock(&self.aux_footage).contains(&footage.path); + if shared + && path.to_ascii_lowercase().ends_with(".exr") + && let Some(img) = crate::exr_channels::cached_picture(path, bytes.clone(), op) + { + return Ok(img); + } + match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { + Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::cached_layered_image(path, bytes).ok_or_else(|| MediaError::Decode(format!("{path}: {e}")))?, - }; - Ok(dynamic_to_image(&img, op)) + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + } } fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { @@ -962,6 +972,14 @@ impl FootageSource for MediaPool { return None; } let path = loc.key.path.clone(); + { + let mut wanted = lock(&self.inner.aux_footage); + // (a bound: names of footage items, never cleared otherwise) + if wanted.len() >= 4096 { + wanted.clear(); + } + wanted.insert(footage.path.clone()); + } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/docs/effects-parity.md b/docs/effects-parity.md index bd81a09df..ce4debd01 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR decode shared (#474 open: f16 storage and cache sizing remain) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6eeef1719..741582d9e 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); EXRs without an unnamed RGB layer decode twice (#474) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index f7f1df21b..0ebbeeecc 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,6 +120,11 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. +Footage on its own decodes only the channels of its picture. Once an effect reads an item's +channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for +the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); +several frames asking for the same file at once wait for one decode. + ## Replacing and reloading Importing a layered file as a composition refreshes the decoded sources used by its layers, diff --git a/docs/gaps.md b/docs/gaps.md index 016aeff22..46a96983a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (decode shared, faffad3a; f16 storage and cache sizing open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | From 2939b3769cb98274af5c7c83a2e302656fba820e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:18:30 +0100 Subject: [PATCH 05/35] #474: format the OpenEXR golden test Formatting only; no hash changes. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 34 ++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index 498944882..f94b3cac0 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -95,7 +95,14 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let origin = ((0, 0), full); let cat = |a: &[(&'static str, Kind)], b: &[(&'static str, Kind)]| -> Vec<(&'static str, Kind)> { a.iter().chain(b).copied().collect() }; let layered: Vec<(&str, Kind)> = cat( - &[("Image.R", Kind::Half), ("Image.G", Kind::Half), ("Image.B", Kind::Half), ("Image.A", Kind::Half), ("depth.Z", Kind::Float), ("ObjectID", Kind::Uint)], + &[ + ("Image.R", Kind::Half), + ("Image.G", Kind::Half), + ("Image.B", Kind::Half), + ("Image.A", Kind::Half), + ("depth.Z", Kind::Float), + ("ObjectID", Kind::Uint), + ], &CRYPTO, ); let unnamed_plus: Vec<(&str, Kind)> = @@ -103,7 +110,11 @@ fn files() -> Vec<(&'static str, Vec, bool)> { let rgb_float = [("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("depth.Z", Kind::Float)]; vec![ ("rgba_half_zip16", write(vec![part(None, &RGBA_HALF, full, (0, 0), Compression::ZIP16)], origin), true), - ("rgb_float_piz", write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), true), + ( + "rgb_float_piz", + write(vec![part(None, &[("R", Kind::Float), ("G", Kind::Float), ("B", Kind::Float), ("Z", Kind::Float)], full, (0, 0), Compression::PIZ)], origin), + true, + ), ("layered_rle", write(vec![part(None, &layered, full, (0, 0), Compression::RLE)], origin), true), ("unnamed_rgb_plus_layers_zip1", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::ZIP1)], origin), true), ( @@ -112,7 +123,13 @@ fn files() -> Vec<(&'static str, Vec, bool)> { vec![ part(Some("Image"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), part(Some("depth"), &[("Z", Kind::Float)], full, (0, 0), Compression::ZIP16), - part(Some("CryptoObject00"), &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], full, (0, 0), Compression::ZIP16), + part( + Some("CryptoObject00"), + &[("r", Kind::Float), ("g", Kind::Float), ("b", Kind::Float), ("a", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + ), ], origin, ), @@ -138,7 +155,12 @@ fn files() -> Vec<(&'static str, Vec, bool)> { part(Some("ViewLayer.Depth"), &[("ViewLayer.Depth.Z", Kind::Float)], full, (0, 0), Compression::RLE), part( Some("ViewLayer.Combined"), - &[("ViewLayer.Combined.R", Kind::Half), ("ViewLayer.Combined.G", Kind::Half), ("ViewLayer.Combined.B", Kind::Half), ("ViewLayer.Combined.A", Kind::Half)], + &[ + ("ViewLayer.Combined.R", Kind::Half), + ("ViewLayer.Combined.G", Kind::Half), + ("ViewLayer.Combined.B", Kind::Half), + ("ViewLayer.Combined.A", Kind::Half), + ], full, (0, 0), Compression::RLE, @@ -194,7 +216,9 @@ fn aux_hash(pool: &MediaPool, f: &Footage) -> u64 { /// The interpretations a picture is decoded in: (label, footage, curve, size, pinned). fn variants(f: &Footage) -> Vec<(String, Footage, FloatCurve, Option<(u32, u32)>, bool)> { let mut out = vec![]; - for (mode, label) in [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] { + for (mode, label) in + [(None, "probed"), (Some(AlphaMode::Straight), "straight"), (Some(AlphaMode::Premultiplied), "premul"), (Some(AlphaMode::Ignore), "ignore")] + { let mut g = f.clone(); if let Some(m) = mode { g.alpha = m; From d4816719474e4b1d5fa3ae6272f781333b421b57 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:20:15 +0100 Subject: [PATCH 06/35] #474: golden OpenEXR files for tiles, mip levels, line order, uncompressed and uint RGB Adds tiled (ZIP16, RLE), tiled mip-mapped (ZIP16, PIZ), decreasing line order, uncompressed, a uint R/G/B picture and a multi-part file whose first R/G/B part is depth-named (the planes can't stand in for the image crate's picture there). The new hashes were taken with main's crates/media/src at a646bdcc (extracted with git archive) and the same test, before the decode rework. Co-Authored-By: Claude Opus 5.5 --- crates/media/tests/it/exr_golden.rs | 156 +++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index f94b3cac0..d99de635a 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -70,9 +70,36 @@ fn with_manifest(mut attrs: LayerAttributes) -> LayerAttributes { /// One part: `list` channels over a data window of `size` at `pos`. fn part(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), compression: Compression) -> Layer> { + part_with(name, list, size, pos, encoding(compression)) +} + +/// [`part`] with its own block layout (tiles) and line order. +fn part_with(name: Option<&str>, list: &[(&str, Kind)], size: (usize, usize), pos: (i32, i32), encoding: Encoding) -> Layer> { let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); attrs.layer_position = Vec2(pos.0, pos.1); - Layer::new(size, with_manifest(attrs), encoding(compression), channels(list, size)) + Layer::new(size, with_manifest(attrs), encoding, channels(list, size)) +} + +/// A tiled single-part file of `list` with mip levels down to 1×1 (each level its own values); +/// the picture and the channels are its largest level. +fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { + let levels = |name: &str, kind: Kind, c: usize| { + let (mut data, mut size) = (vec![], (W, H)); + loop { + data.push(channel(name, kind, size, c + data.len()).sample_data); + if size == (1, 1) { + break; + } + size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); + } + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + }; + let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); + let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; + let layer = Layer::new((W, H), with_manifest(LayerAttributes::default()), encoding, AnyChannels::sort(list.into())); + let mut bytes = std::io::Cursor::new(Vec::new()); + Image::from_layer(layer).write().non_parallel().to_buffered(&mut bytes).unwrap(); + bytes.into_inner() } /// Encode `layers` with the display window `display` (position, size). @@ -176,6 +203,63 @@ fn files() -> Vec<(&'static str, Vec, bool)> { ("layered_window_larger_piz", write(vec![part(None, &layered, (45, 30), (-4, -3), Compression::PIZ)], origin), true), ("unnamed_rgb_plus_layers_dwaa", write(vec![part(None, &unnamed_plus, full, (0, 0), Compression::DWAA(None))], origin), false), ("layered_dwaa", write(vec![part(None, &layered, full, (0, 0), Compression::DWAA(None))], origin), false), + ( + "unnamed_rgb_plus_layers_tiled_zip16", + write( + vec![part_with(None, &unnamed_plus, (30, 20), (3, 2), Encoding { blocks: Blocks::Tiles(Vec2(16, 8)), ..encoding(Compression::ZIP16) })], + origin, + ), + true, + ), + ( + "layered_tiled_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { blocks: Blocks::Tiles(Vec2(8, 8)), ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_mipmapped_zip16", mipmapped(&unnamed_plus, Compression::ZIP16), true), + ("layered_mipmapped_piz", mipmapped(&layered, Compression::PIZ), true), + ( + "unnamed_rgb_plus_layers_decreasing_zip1", + write( + vec![part_with(None, &unnamed_plus, (20, 10), (5, 7), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::ZIP1) })], + origin, + ), + true, + ), + ( + "layered_decreasing_rle", + write(vec![part_with(None, &layered, full, (0, 0), Encoding { line_order: LineOrder::Decreasing, ..encoding(Compression::RLE) })], origin), + true, + ), + ("unnamed_rgb_plus_layers_uncompressed", write(vec![part(None, &unnamed_plus, (20, 10), (-3, 4), Compression::Uncompressed)], origin), true), + ("layered_uncompressed", write(vec![part(None, &layered, full, (0, 0), Compression::Uncompressed)], origin), true), + ( + "uint_rgb_zip16", + write( + vec![part( + None, + &[("R", Kind::Uint), ("G", Kind::Uint), ("B", Kind::Uint), ("A", Kind::Half), ("depth.Z", Kind::Float)], + full, + (0, 0), + Compression::ZIP16, + )], + origin, + ), + true, + ), + ( + // The image crate shows the part "depth" (the first with R, G and B), whose planes are + // filled with the background depth outside its data window: decoded by the image crate. + "multipart_depth_named_rgb_zip16", + write( + vec![ + part(Some("depth"), &[("R", Kind::Half), ("G", Kind::Half), ("B", Kind::Half)], (20, 10), (5, 7), Compression::ZIP16), + part(Some("beauty"), &RGBA_HALF, full, (0, 0), Compression::ZIP16), + ], + origin, + ), + true, + ), ] } @@ -331,6 +415,76 @@ const PINNED: &[(&str, &str, u64)] = &[ ("layered_window_larger_piz", "ignore/linear", 0xaa01706accce2cc4), ("layered_window_larger_piz", "preserve_rgb", 0xb2cbb40b10bc56b4), ("layered_window_larger_piz", "aux", 0xb7d91c8eea56a772), + ("unnamed_rgb_plus_layers_tiled_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_tiled_zip16", "probed/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "straight/linear", 0xaa5f42cb6a31c78a), + ("unnamed_rgb_plus_layers_tiled_zip16", "premul/linear", 0x7612f20634fe957e), + ("unnamed_rgb_plus_layers_tiled_zip16", "ignore/linear", 0xbbf236635dcaac90), + ("unnamed_rgb_plus_layers_tiled_zip16", "preserve_rgb", 0x15b555a243d79613), + ("unnamed_rgb_plus_layers_tiled_zip16", "aux", 0x3d1439eed2202e41), + ("layered_tiled_rle", "probe", 0x0a32e074a28c1f66), + ("layered_tiled_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_tiled_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_tiled_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_tiled_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_tiled_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "probed/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "straight/linear", 0xf028e1367a2fbb30), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "premul/linear", 0xa84600283dce98f6), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "ignore/linear", 0xa90a57f9a67f7468), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "preserve_rgb", 0xc8e0050f5efe75b7), + ("unnamed_rgb_plus_layers_mipmapped_zip16", "aux", 0x27cc9b53d0862db1), + ("layered_mipmapped_piz", "probe", 0x0a32e074a28c1f66), + ("layered_mipmapped_piz", "probed/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "straight/linear", 0xf028e1367a2fbb30), + ("layered_mipmapped_piz", "premul/linear", 0xa84600283dce98f6), + ("layered_mipmapped_piz", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_mipmapped_piz", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_mipmapped_piz", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_decreasing_zip1", "probed/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "straight/linear", 0x2c73f9acaa936420), + ("unnamed_rgb_plus_layers_decreasing_zip1", "premul/linear", 0x19eb3b1a88d108a7), + ("unnamed_rgb_plus_layers_decreasing_zip1", "ignore/linear", 0x3b86057d18a16bf0), + ("unnamed_rgb_plus_layers_decreasing_zip1", "preserve_rgb", 0x5b999014e4a64bb2), + ("unnamed_rgb_plus_layers_decreasing_zip1", "aux", 0x8f78d3233450acf3), + ("layered_decreasing_rle", "probe", 0x0a32e074a28c1f66), + ("layered_decreasing_rle", "probed/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "straight/linear", 0xf028e1367a2fbb30), + ("layered_decreasing_rle", "premul/linear", 0xa84600283dce98f6), + ("layered_decreasing_rle", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_decreasing_rle", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_decreasing_rle", "aux", 0xa6b9bd9f02c192ad), + ("unnamed_rgb_plus_layers_uncompressed", "probe", 0x0a32e074a28c1f66), + ("unnamed_rgb_plus_layers_uncompressed", "probed/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "straight/linear", 0xfc73fe89457ce029), + ("unnamed_rgb_plus_layers_uncompressed", "premul/linear", 0xa27668bf31d9426b), + ("unnamed_rgb_plus_layers_uncompressed", "ignore/linear", 0xb3276cc5126776d4), + ("unnamed_rgb_plus_layers_uncompressed", "preserve_rgb", 0xdee1105fa6e6822a), + ("unnamed_rgb_plus_layers_uncompressed", "aux", 0x854a5e84e7aaa5c2), + ("layered_uncompressed", "probe", 0x0a32e074a28c1f66), + ("layered_uncompressed", "probed/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "straight/linear", 0xf028e1367a2fbb30), + ("layered_uncompressed", "premul/linear", 0xa84600283dce98f6), + ("layered_uncompressed", "ignore/linear", 0xa90a57f9a67f7468), + ("layered_uncompressed", "preserve_rgb", 0xc8e0050f5efe75b7), + ("layered_uncompressed", "aux", 0xa6b9bd9f02c192ad), + ("uint_rgb_zip16", "probe", 0x0a32e074a28c1f66), + ("uint_rgb_zip16", "probed/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "straight/linear", 0xf9fe11ab8ed97cc8), + ("uint_rgb_zip16", "premul/linear", 0xed89310ef0cdb9fd), + ("uint_rgb_zip16", "ignore/linear", 0xf78cd263c0d74f12), + ("uint_rgb_zip16", "preserve_rgb", 0x2a19d67b07cf4359), + ("uint_rgb_zip16", "aux", 0x03e42411f8521856), + ("multipart_depth_named_rgb_zip16", "probe", 0x6a38346c4c5662d5), + ("multipart_depth_named_rgb_zip16", "probed/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "straight/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "premul/linear", 0x869374e9b0384cea), + ("multipart_depth_named_rgb_zip16", "ignore/linear", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "preserve_rgb", 0x3b86057d18a16bf0), + ("multipart_depth_named_rgb_zip16", "aux", 0x0fa56884e83b4417), ]; #[test] From eedd64906c5859b9118bc3b0264646c3719bb699 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 02:25:07 +0100 Subject: [PATCH 07/35] #474: keep files in use in the channel cache, cap plane memory, fall back on a missing cache entry Review follow-ups: - Files in use are never dropped from the channel cache: planes an effect holds, the files of the last 4 pictures made of their planes until their effects ask for the channels, and decodes in progress, which now count the planes they will allocate (from the header) against the budget. With 4 frames at once over a budget of one file, each file decodes once (test). - A decode that panics removes its entry instead of leaving it decoding. - One decode allocates at most 8 GiB of planes (512 MiB on 32-bit and wasm); a header asking for more is refused with the reason. - A shared picture whose file has no cached channels (a failed read, a broken part it doesn't use) is decoded from its own channels. - Comment why waiting on a decode may block a rayon worker; drop the subsampling check and test (the exr crate refuses subsampled files). - Docs: the cache is a soft bound; a single-part file still decompresses every channel; the first frame and the image crate's route decode twice. Co-Authored-By: Claude Opus 5.5 --- crates/media/src/exr_channels.rs | 389 +++++++++++++++++++++------- crates/media/src/pool.rs | 3 +- crates/media/tests/it/exr_golden.rs | 2 +- docs/effects-parity.md | 2 +- docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- docs/gaps.md | 2 +- 7 files changed, 309 insertions(+), 101 deletions(-) diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 747882337..8a8f5edcc 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -93,8 +93,6 @@ struct PlanesReader { f32s: Vec, /// The part's channel indices of exact `R`, `G`, `B` and `A`, when it has R, G and B. rgb: Option<([usize; 3], Option)>, - /// The part has a channel that doesn't sample every pixel. - subsampled: bool, /// The part's display window differs from the first part's. own_display: bool, } @@ -148,7 +146,6 @@ impl<'s> ReadChannels<'s> for Planes<'_> { u32s: vec![], f32s: vec![], rgb, - subsampled: header.channels.list.iter().any(|c| c.sampling != exr::math::Vec2(1, 1)), own_display: header.shared_attributes.display_window != dw, }) } @@ -226,7 +223,7 @@ impl ChannelsReader for PlanesReader { }; Some(([kept(r)?, kept(g)?, kept(b)?], a)) }); - PartPlanes { planes, rgb, has_rgb, exact: !self.subsampled && !self.own_display } + PartPlanes { planes, rgb, has_rgb, exact: !self.own_display } } } @@ -240,8 +237,8 @@ enum Rgb { None, /// The channels (indices in [`AuxChannels::channels`]) hold exactly what it would show. Planes([usize; 3], Option), - /// The planes can't stand in for it (subsampled channels, a depth-filled plane, another - /// display window): decode with the `image` crate. + /// The planes can't stand in for it (a depth-named channel, whose plane is filled with the + /// background depth, or another display window): decode with the `image` crate. Unsupported, } @@ -258,13 +255,52 @@ impl Decoded { } } -/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). `None` when it -/// is not a readable EXR. -fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; - let display = meta.headers.first()?.shared_attributes.display_window; +/// The most bytes of planes one decode allocates (every kept channel is a display-window plane +/// of f32, allocated before any pixels are read): a header can ask for far more than the file +/// holds. +#[cfg(target_pointer_width = "64")] +const MAX_DECODE_BYTES: usize = 8 << 30; +#[cfg(not(target_pointer_width = "64"))] +const MAX_DECODE_BYTES: usize = 512 << 20; + +/// Why a file's channels weren't decoded. +#[derive(Debug, PartialEq)] +enum Refused { + /// Not a readable OpenEXR file. + Unreadable, + /// Its kept channels need more memory than [`MAX_DECODE_BYTES`] (the reason, to show). + TooLarge(String), +} + +/// Bytes the planes of the channels `keep` names would take (`None`: not a readable header). +fn planes_bytes(meta: &exr::meta::MetaData, keep: &dyn Fn(&str) -> bool) -> Option { + let dw = meta.headers.first()?.shared_attributes.display_window; + let n = dw.size.0.checked_mul(dw.size.1)?; + let mut kept = 0usize; + for header in meta.headers.iter() { + let prefix = part_name(header); + kept = kept.saturating_add(header.channels.list.iter().filter(|ch| keep(&channel_name(prefix.as_deref(), &ch.name.to_string()))).count()); + } + Some(n.saturating_mul(kept).saturating_mul(std::mem::size_of::())) +} + +/// Decode the channels of an EXR file's bytes that `keep` names (by plane name). +fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Result { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; + let display = meta.headers.first().ok_or(Refused::Unreadable)?.shared_attributes.display_window; + let need = planes_bytes(&meta, keep).ok_or(Refused::Unreadable)?; + if need > MAX_DECODE_BYTES { + let gb = |b: usize| b as f64 / f64::from(1u32 << 30); + return Err(Refused::TooLarge(format!( + "its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use", + display.size.0, + display.size.1, + gb(need), + gb(MAX_DECODE_BYTES) + ))); + } let read = ReadImage::new(ReadAllLayers { read_channels: Planes { keep, display } }, |_| {}); - let img = read.from_buffered(std::io::Cursor::new(bytes)).ok()?; + let img = read.from_buffered(std::io::Cursor::new(bytes)).map_err(|_| Refused::Unreadable)?; let dw = img.attributes.display_window; let mut aux = AuxChannels::new(dw.size.0 as u32, dw.size.1 as u32, 1.0); let mut crypto: HashMap, Option)> = HashMap::new(); @@ -306,12 +342,12 @@ fn decode(bytes: &[u8], keep: &dyn Fn(&str) -> bool) -> Option { } } aux.manifests.sort(); - Some(Decoded { aux: Arc::new(aux), rgb }) + Ok(Decoded { aux: Arc::new(aux), rgb }) } /// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. pub fn read_exr_channels(bytes: &[u8]) -> Option { - let aux = decode(bytes, &|_| true)?.aux; + let aux = decode(bytes, &|_| true).ok()?.aux; Some(Arc::try_unwrap(aux).unwrap_or_else(|a| (*a).clone())) } @@ -371,18 +407,18 @@ fn picture_names(aux: &AuxChannels) -> Option<([String; 3], Option)> { /// The channels of the picture of a multi-layer file and nothing else (names first from the /// headers, then only those channels decoded; in a multi-part file the other parts' pixels are /// not even decompressed), with the names of its planes. -fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> { - let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).ok()?; +fn picture_only(bytes: &[u8]) -> Result<(Decoded, [String; 3], Option), Refused> { + let meta = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(bytes), false).map_err(|_| Refused::Unreadable)?; let mut names = AuxChannels::new(0, 0, 1.0); for header in meta.headers.iter() { let prefix = part_name(header); names.channels.extend(header.channels.list.iter().map(|ch| (channel_name(prefix.as_deref(), &ch.name.to_string()), vec![]))); } - let (rgb, a) = picture_names(&names)?; + let (rgb, a) = picture_names(&names).ok_or(Refused::Unreadable)?; // Planes are looked up by name ignoring case, so every channel that could answer is kept. let keep = |n: &str| rgb.iter().chain(&a).any(|k| k.eq_ignore_ascii_case(n)); let decoded = decode(bytes, &keep)?; - Some((decoded, rgb, a)) + Ok((decoded, rgb, a)) } /// The picture of a multi-layer OpenEXR file without an unnamed RGB layer (Blender's @@ -392,7 +428,7 @@ fn picture_only(bytes: &[u8]) -> Option<(Decoded, [String; 3], Option)> /// that is not a data pass (never Cryptomatte, depth or normals, #412), else the first channel /// as grey. Linear, straight RGBA as the file stores it. `None` when it is not a readable EXR. pub fn layered_image(bytes: &[u8]) -> Option { - let (d, rgb, a) = picture_only(bytes)?; + let (d, rgb, a) = picture_only(bytes).ok()?; let aux = &d.aux; let [r, g, b] = [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?]; let a = match &a { @@ -438,19 +474,24 @@ fn layered_picture_of(aux: &AuxChannels, op: AlphaOp) -> Option { /// The picture of the multi-layer EXR at `path` ([`layered_image`]'s rule) as an [`Image`]: /// from the channel cache when `shared` (an effect reads the footage's channels, so they are -/// decoded anyway), else of the picture's channels alone. -pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Option { - if shared { - let d = cache().get(path, || Some(bytes))?; - return layered_picture_of(&d.aux, op); +/// decoded anyway), else of the picture's channels alone (also when the cache has no channels +/// for the file: a read that failed, or a broken part the picture doesn't use). `Err(Some(why))` +/// when the file is too large to decode, `Err(None)` when it isn't a readable multi-layer EXR. +pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: bool) -> Result> { + if shared && let Some(d) = cache().get(path, Use::Picture, || Some(bytes.clone())) { + return layered_picture_of(&d.aux, op).ok_or(None); } - let (d, rgb, a) = picture_only(&bytes)?; + let (d, rgb, a) = picture_only(&bytes).map_err(|r| match r { + Refused::TooLarge(why) => Some(why), + Refused::Unreadable => None, + })?; let aux = &d.aux; + let plane = |n: &str| aux.get(n).ok_or(None); let a = match &a { - Some(n) => Some(aux.get(n)?), + Some(n) => Some(plane(n)?), None => None, }; - planes_picture(aux, [aux.get(&rgb[0])?, aux.get(&rgb[1])?, aux.get(&rgb[2])?], a, op) + planes_picture(aux, [plane(&rgb[0])?, plane(&rgb[1])?, plane(&rgb[2])?], a, op).ok_or(None) } /// The picture of the EXR at `path` from the channel cache (#474: EXtractoR / Cryptomatte read @@ -459,7 +500,7 @@ pub(crate) fn layered_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp, shared: /// the planes can't stand in for the `image` crate's picture (decode with it), or the file is /// not a readable EXR. pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Option { - let d = cache().get(path, || Some(bytes))?; + let d = cache().get(path, Use::Picture, || Some(bytes))?; let aux = &d.aux; match d.rgb { Rgb::None => layered_picture_of(aux, op), @@ -477,9 +518,11 @@ pub(crate) fn cached_picture(path: &str, bytes: Arc<[u8]>, op: AlphaOp) -> Optio /// How many bytes of decoded channels the cache keeps, like the frame cache's default /// ([`crate::DEFAULT_BUDGET`]): two 2560×1440 files of 32 channels (470 MB each, as every plane -/// is f32). The newest file is kept whatever its size, so a frame's picture and its effects -/// share its decode. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak -/// memory about 1 GB lower than 2 GiB, at the same speed.) +/// is f32). A soft bound: the newest file is kept whatever its size (so a frame's picture and its +/// effects share its decode), and files still in use are never dropped (planes an effect holds, +/// [`HELD_FILES`] pictures waiting for their effects, decodes in progress), so memory can go +/// above it while they are. (Measured on such a sequence decoded one frame at a time: 1 GiB held +/// peak memory about 1 GB lower than 2 GiB, at the same speed.) #[cfg(not(target_arch = "wasm32"))] pub const CACHE_BUDGET: usize = 1 << 30; #[cfg(target_arch = "wasm32")] @@ -488,90 +531,178 @@ pub const CACHE_BUDGET: usize = 512 << 20; /// At most this many files are cached (unreadable ones hold no bytes). const CACHE_FILES: usize = 64; +/// At most this many files whose picture was made of their planes are kept for their effects +/// (the renderer asks for a frame's picture, then its channels): about as many frames as are +/// rendered at once. The oldest stops being kept when another comes. +const HELD_FILES: usize = 4; + /// A file's decode: done once, by the first caller; the others wait for it. type Slot = Arc>>>; struct Entry { path: String, slot: Slot, - /// Bytes of the finished decode (0 while decoding). + /// Bytes of the finished decode, or the planes it will allocate while it decodes. bytes: usize, - /// When it was last asked for (a counter). + /// When it was last asked for or finished decoding (a counter). used: u64, } +impl Entry { + /// Dropping it would free nothing or lose a decode in progress: an effect (or a picture + /// waiting for its effects) still holds its planes, or it is decoding. + fn in_use(&self) -> bool { + match self.slot.get() { + None => true, + Some(None) => false, + Some(Some(d)) => Arc::strong_count(d) > 1 || Arc::strong_count(&d.aux) > 1, + } + } +} + +struct State { + entries: Vec, + clock: u64, + /// Files whose picture was made of their planes and whose channels haven't been asked for + /// since, oldest first (at most [`HELD_FILES`]). + held: std::collections::VecDeque<(String, Arc)>, +} + +/// Who asks for a file's channels. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum Use { + /// The picture of a footage item whose effects read its channels next: kept for them. + Picture, + /// The channels themselves (EXtractoR, Cryptomatte, the 3D Channel effects). + Channels, +} + /// Decoded EXR files by path, bounded by bytes, least recently used out first. pub(crate) struct ChannelCache { - entries: Mutex<(Vec, u64)>, + state: Mutex, budget: usize, /// Files decoded (for tests). decodes: AtomicU64, } +/// Removes a decode's entry if the decode panics, so the file doesn't stay "decoding" (never +/// dropped, its estimated bytes counted) until someone asks for it again. +struct Abandon<'a> { + cache: &'a ChannelCache, + slot: &'a Slot, +} + +impl Drop for Abandon<'_> { + fn drop(&mut self) { + if std::thread::panicking() { + self.cache.lock().entries.retain(|e| !Arc::ptr_eq(&e.slot, self.slot)); + } + } +} + impl ChannelCache { pub(crate) const fn new(budget: usize) -> Self { - Self { entries: Mutex::new((Vec::new(), 0)), budget, decodes: AtomicU64::new(0) } + Self { state: Mutex::new(State { entries: Vec::new(), clock: 0, held: std::collections::VecDeque::new() }), budget, decodes: AtomicU64::new(0) } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Set the bytes of the entry of `slot` (if it's still cached), mark it used now and drop + /// least recently used files until the rest fit the budget. + fn account(&self, slot: &Slot, bytes: usize) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + // (gone when the file was forgotten meanwhile) Used as of now: a file asked for before + // another but decoded after it is not the older one. + if let Some(e) = state.entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, slot)) { + e.bytes = bytes; + e.used = clock; + } + Self::evict(&mut state.entries, self.budget, slot); } /// The decoded file at `path`, read with `read` by the first caller; callers asking for the /// same file meanwhile wait for that decode instead of decoding it again. - pub(crate) fn get(&self, path: &str, read: impl FnOnce() -> Option>) -> Option> { + pub(crate) fn get(&self, path: &str, by: Use, read: impl FnOnce() -> Option>) -> Option> { let slot = { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - match entries.iter_mut().find(|e| e.path == path) { + let mut state = self.lock(); + state.clock += 1; + let clock = state.clock; + match state.entries.iter_mut().find(|e| e.path == path) { Some(e) => { - e.used = *clock; + e.used = clock; e.slot.clone() } None => { let slot = Slot::default(); - entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: *clock }); + state.entries.push(Entry { path: path.to_string(), slot: slot.clone(), bytes: 0, used: clock }); slot } } }; let mut decoded_here = false; + // Waiting here can block a rayon worker (which `Inner::frame_at` avoids for frames): that + // is safe because the decode never waits on rayon's global pool (the exr crate + // decompresses on a pool of its own and this thread only receives the blocks), so the + // decoding thread can't be waiting for this one. let v = slot .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); - read().and_then(|b| decode(&b, &|_| true)).map(Arc::new) + let _abandon = Abandon { cache: self, slot: &slot }; + let bytes = read()?; + // The planes it will allocate count against the budget while it decodes. + if let Ok(meta) = exr::meta::MetaData::read_from_buffered(std::io::Cursor::new(&bytes[..]), false) + && let Some(need) = planes_bytes(&meta, &|_| true) + { + self.account(&slot, need.min(MAX_DECODE_BYTES)); + } + match decode(&bytes, &|_| true) { + Ok(d) => Some(Arc::new(d)), + Err(Refused::TooLarge(why)) => { + log::warn!("media: {path}: {why}"); + None + } + Err(Refused::Unreadable) => None, + } }) .clone(); if decoded_here { - let mut guard = self.entries.lock().unwrap_or_else(PoisonError::into_inner); - let (entries, clock) = &mut *guard; - *clock += 1; - // (gone when the file was forgotten meanwhile) Used as of now: a file asked for - // before another but decoded after it is not the older one, or concurrent frames - // would evict each other's files before their effects read them. - if let Some(e) = entries.iter_mut().find(|e| Arc::ptr_eq(&e.slot, &slot)) { - e.bytes = v.as_ref().map_or(0, |d| d.bytes()); - e.used = *clock; - } - Self::evict(entries, self.budget, &slot); + self.account(&slot, v.as_ref().map_or(0, |d| d.bytes())); } + let mut state = self.lock(); + match (by, &v) { + (Use::Picture, Some(d)) => { + if !state.held.iter().any(|(p, _)| p == path) { + state.held.push_back((path.to_string(), d.clone())); + if state.held.len() > HELD_FILES { + state.held.pop_front(); + } + } + } + (Use::Channels, _) => state.held.retain(|(p, _)| p != path), + (Use::Picture, None) => {} + } + drop(state); v } - /// Drop least recently used finished files until the rest fit the budget, keeping `newest`. + /// Drop least recently used files not in use until the rest fit the budget, keeping `newest`. fn evict(entries: &mut Vec, budget: usize, newest: &Slot) { while entries.iter().map(|e| e.bytes).sum::() > budget || entries.len() > CACHE_FILES { - let oldest = entries - .iter() - .enumerate() - .filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && e.slot.get().is_some()) - .min_by_key(|(_, e)| e.used) - .map(|(i, _)| i); + let oldest = entries.iter().enumerate().filter(|(_, e)| !Arc::ptr_eq(&e.slot, newest) && !e.in_use()).min_by_key(|(_, e)| e.used).map(|(i, _)| i); let Some(i) = oldest else { break }; entries.remove(i); } } pub(crate) fn forget(&self, path: &str) { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.retain(|e| e.path != path); + let mut state = self.lock(); + state.entries.retain(|e| e.path != path); + state.held.retain(|(p, _)| p != path); } #[cfg(test)] @@ -581,7 +712,7 @@ impl ChannelCache { #[cfg(test)] fn cached(&self) -> Vec { - self.entries.lock().unwrap_or_else(PoisonError::into_inner).0.iter().map(|e| e.path.clone()).collect() + self.lock().entries.iter().map(|e| e.path.clone()).collect() } } @@ -597,7 +728,7 @@ pub(crate) fn forget(path: &str) { /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { - cache().get(path, read).map(|d| d.aux.clone()) + cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } #[cfg(test)] @@ -663,14 +794,17 @@ mod tests { use exr::prelude::*; let half = |x: f32| f16::from_f32(x); let part = |pos: (i32, i32), size: (usize, usize), list: Vec>| { - let mut attrs = LayerAttributes::default(); - attrs.layer_position = Vec2(pos.0, pos.1); + let attrs = LayerAttributes { layer_position: Vec2(pos.0, pos.1), ..Default::default() }; Layer::new(size, attrs, Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())) }; let display = IntegerBounds::from_dimensions((3, 2)); // A 2×1 data window at (1, 1). let inside = file( - vec![part((1, 1), (2, 1), vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))])], + vec![part( + (1, 1), + (2, 1), + vec![AnyChannel::new("G", FlatSamples::F16(vec![half(1.0), half(2.0)])), AnyChannel::new("Z", FlatSamples::U32(vec![7, 8]))], + )], display, ); let aux = read_exr_channels(&inside).unwrap(); @@ -699,7 +833,7 @@ mod tests { s.spawn(|| { barrier.wait(); cache - .get("/474-concurrent.exr", || { + .get("/474-concurrent.exr", Use::Channels, || { reads.fetch_add(1, Ordering::Relaxed); // Long enough for the others to arrive while this one decodes. std::thread::sleep(std::time::Duration::from_millis(50)); @@ -726,7 +860,7 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - let get = |p: &str| cache.get(p, || Some(bytes.clone())).unwrap(); + let get = |p: &str| cache.get(p, Use::Channels, || Some(bytes.clone())).unwrap(); assert_eq!(get("/a.exr").bytes(), one); get("/b.exr"); get("/a.exr"); // a is now more recent than b @@ -737,18 +871,18 @@ mod tests { assert_eq!(cache.decodes(), 3, "a hit"); // A file over the budget on its own is still kept while it is the newest. let small = ChannelCache::new(one / 2); - assert!(small.get("/big.exr", || Some(bytes.clone())).is_some()); + assert!(small.get("/big.exr", Use::Channels, || Some(bytes.clone())).is_some()); assert_eq!(small.cached(), ["/big.exr"]); - small.get("/big2.exr", || Some(bytes.clone())); + small.get("/big2.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(small.cached(), ["/big2.exr"]); // An unreadable file is remembered (not read again) without bytes, and forgotten on request. - assert!(cache.get("/broken.exr", || Some(Arc::from(&b"not an exr"[..]))).is_none()); - assert!(cache.get("/broken.exr", || panic!("not read again")).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || Some(Arc::from(&b"not an exr"[..]))).is_none()); + assert!(cache.get("/broken.exr", Use::Channels, || panic!("not read again")).is_none()); cache.forget("/broken.exr"); assert!(!cache.cached().contains(&"/broken.exr".to_string())); // At most CACHE_FILES files, however small. for i in 0..CACHE_FILES + 10 { - cache.get(&format!("/none{i}.exr"), || None); + cache.get(&format!("/none{i}.exr"), Use::Channels, || None); } assert!(cache.cached().len() <= CACHE_FILES); } @@ -761,22 +895,101 @@ mod tests { let bytes: Arc<[u8]> = flat(&["R", "G", "B", "A"], (32, 32), 0.25).into(); let one = 4 * 32 * 32 * 4 + 4; let cache = ChannelCache::new(2 * one); - cache.get("/first.exr", || { + cache.get("/first.exr", Use::Channels, || { // Another frame's file is asked for and decoded while this one decodes. - cache.get("/second.exr", || Some(bytes.clone())); + cache.get("/second.exr", Use::Channels, || Some(bytes.clone())); Some(bytes.clone()) }); - cache.get("/third.exr", || Some(bytes.clone())); + cache.get("/third.exr", Use::Channels, || Some(bytes.clone())); assert_eq!(cache.cached(), ["/first.exr", "/third.exr"], "the file decoded first went first"); } + /// #474: frames rendered at once (each asks for its picture, then its effects for the + /// channels while other frames decode) decode each file once, though the budget holds only + /// one: a picture's file is kept for its effects, and files in use are never dropped. + #[test] + fn concurrent_frames_decode_each_file_once() { + let bytes: Arc<[u8]> = flat(&["R", "G", "B", "Z"], (64, 64), 0.5).into(); + let cache = ChannelCache::new(4 * 64 * 64 * 4 + 4); + let (next, reads, frames) = (std::sync::atomic::AtomicUsize::new(0), AtomicU64::new(0), 24); + let read = || { + reads.fetch_add(1, Ordering::Relaxed); + Some(bytes.clone()) + }; + std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= frames { + break; + } + let path = format!("/474-frames/{i:04}.exr"); + drop(cache.get(&path, Use::Picture, read).unwrap()); + std::thread::sleep(std::time::Duration::from_millis(3)); + let aux = cache.get(&path, Use::Channels, read).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(2)); + assert_eq!(aux.aux.get("Z").and_then(|z| z.first()).copied(), Some(0.5)); + } + }); + } + }); + assert_eq!(cache.decodes(), frames as u64, "one decode per frame"); + assert_eq!(reads.load(Ordering::Relaxed), frames as u64); + } + + /// #474: a decode that panics doesn't leave its file "decoding" in the cache (never dropped, + /// its planes counted): the entry goes, and the next request decodes the file. + #[test] + fn a_decode_that_panics_is_not_left_in_the_cache() { + let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); + let cache = ChannelCache::new(CACHE_BUDGET); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| cache.get("/474-panics.exr", Use::Channels, || panic!("the read failed")))); + assert!(r.is_err()); + assert!(cache.cached().is_empty()); + assert!(cache.get("/474-panics.exr", Use::Channels, || Some(bytes.clone())).is_some()); + assert_eq!(cache.cached(), ["/474-panics.exr"]); + } + + /// #474: the picture of a multi-layer file whose effects read its channels comes from the + /// cache, and from the picture's own channels when the cache has none for the file (a read + /// that failed, or a broken part the picture doesn't use). + #[test] + fn a_shared_picture_without_cached_channels_is_still_decoded() { + use effectcraft_project::Footage; + use effectcraft_render::FloatCurve; + let bytes: Arc<[u8]> = flat(&["Image.R", "Image.G", "Image.B", "depth.Z"], (4, 2), 0.5).into(); + let path = "/474-shared-fallback.exr"; + assert!(cache().get(path, Use::Channels, || None).is_none(), "a read that failed"); + let op = AlphaOp::of(&Footage::default(), FloatCurve::Linear); + let img = layered_picture(path, bytes, op, true).unwrap(); + assert_eq!(img.get(1, 1), [0.5, 0.5, 0.5, 1.0]); + } + + /// #474: a header asking for more plane memory than one decode may allocate (16384² display + /// window, 40 channels: 43 GB) is refused with a reason before anything is allocated. + #[test] + fn a_header_asking_for_too_much_memory_is_refused() { + use exr::prelude::*; + let list: Vec> = (0..40).map(|i| AnyChannel::new(format!("pass{i:02}.Y").as_str(), FlatSamples::F32(vec![0.5]))).collect(); + let layer = Layer::new((1, 1), LayerAttributes::default(), Encoding::FAST_LOSSLESS, AnyChannels::sort(list.into())); + let bytes = file(vec![layer], IntegerBounds::new((0, 0), (16384, 16384))); + match decode(&bytes, &|_| true) { + Err(Refused::TooLarge(why)) => assert!(why.contains("16384×16384") && why.contains("GB"), "{why}"), + Err(Refused::Unreadable) => panic!("readable"), + Ok(_) => panic!("refused"), + } + assert!(read_exr_channels(&bytes).is_none()); + assert!(cached("/474-too-large.exr", || Some(bytes.as_slice().into())).is_none()); + } + /// #474: a file forgotten (changed on disk) while it decodes isn't cached under its path when /// that decode finishes. #[test] fn a_file_forgotten_while_it_decodes_is_not_cached() { let bytes: Arc<[u8]> = flat(&["R"], (8, 8), 1.0).into(); let cache = ChannelCache::new(CACHE_BUDGET); - let d = cache.get("/changed.exr", || { + let d = cache.get("/changed.exr", Use::Channels, || { cache.forget("/changed.exr"); Some(bytes.clone()) }); @@ -824,8 +1037,7 @@ mod tests { /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 - /// around it), in every alpha interpretation and float curve; a file it can't stand in for - /// (a subsampled channel) is left to the `image` crate. + /// around it), in every alpha interpretation and float curve. #[test] fn picture_from_planes_is_the_image_crates() { use effectcraft_project::{AlphaMode, Footage}; @@ -841,7 +1053,10 @@ mod tests { let display = IntegerBounds::new((1, -1), (5, 4)); let n = 20; let files = [ - file(vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], display), + file( + vec![part(None, (1, -1), (5, 4), vec![half("R", 0.1, n), half("G", 0.2, n), half("B", 0.3, n), float("A", 0.1, n), float("Z", 2.0, n)])], + display, + ), file(vec![part(None, (0, 0), (3, 2), vec![float("R", 0.4, 6), float("G", 0.5, 6), float("B", 0.6, 6), half("Image.R", 0.1, 6)])], display), file( vec![ @@ -854,7 +1069,7 @@ mod tests { let bits = |i: &effectcraft_raster::Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); for (k, bytes) in files.iter().enumerate() { let path = format!("/474-planes-{k}.exr"); - let d = cache().get(&path, || Some(bytes.as_slice().into())).unwrap(); + let d = cache().get(&path, Use::Channels, || Some(bytes.as_slice().into())).unwrap(); assert!(matches!(d.rgb, Rgb::Planes(..)), "file {k}: {:?}", d.rgb); let img = crate::probe::load_oriented(bytes, Some(image::ImageFormat::OpenExr)).unwrap(); for alpha in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { @@ -867,16 +1082,6 @@ mod tests { } } } - // A subsampled channel in the picture's part: the `image` crate decodes it. - let mut sub = part(None, (0, 0), (4, 2), vec![half("R", 0.1, 8), half("G", 0.2, 8), half("B", 0.3, 8), half("C", 0.3, 2)]); - if let Some(c) = sub.channel_data.list.iter_mut().find(|c| c.name == *"C") { - c.sampling = Vec2(2, 2); - } - let mut bytes = std::io::Cursor::new(Vec::new()); - if Image::from_layer(sub).write().to_buffered(&mut bytes).is_ok() { - let d = cache().get("/474-subsampled.exr", || Some(bytes.into_inner().into())).map(|d| d.rgb); - assert!(d.is_none_or(|r| r == Rgb::Unsupported), "{d:?}"); - } } /// #2141: one pixel of data in a 60000² display window is refused, not allocated per channel. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 9f1f2aa23..910f8238f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -871,7 +871,8 @@ impl Inner { match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. - Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared).ok_or_else(|| MediaError::Decode(format!("{path}: {e}"))), + Err(e) => crate::exr_channels::layered_picture(path, bytes, op, shared) + .map_err(|why| MediaError::Decode(format!("{path}: {}", why.unwrap_or_else(|| e.to_string())))), } } diff --git a/crates/media/tests/it/exr_golden.rs b/crates/media/tests/it/exr_golden.rs index d99de635a..f760ed7c2 100644 --- a/crates/media/tests/it/exr_golden.rs +++ b/crates/media/tests/it/exr_golden.rs @@ -92,7 +92,7 @@ fn mipmapped(list: &[(&str, Kind)], compression: Compression) -> Vec { } size = ((size.0 / 2).max(1), (size.1 / 2).max(1)); } - AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data.into() }) + AnyChannel::new(name, Levels::Mip { rounding_mode: exr::math::RoundingMode::Down, level_data: data }) }; let list: Vec>> = list.iter().enumerate().map(|(c, (n, k))| levels(n, *k, c)).collect(); let encoding = Encoding { compression, blocks: Blocks::Tiles(Vec2(16, 16)), line_order: LineOrder::Increasing }; diff --git a/docs/effects-parity.md b/docs/effects-parity.md index ce4debd01..d80a874bd 100644 --- a/docs/effects-parity.md +++ b/docs/effects-parity.md @@ -26,7 +26,7 @@ corpus compares renders, each effect is assumed to differ somewhere, and users k | Category | Effects | On the GPU | Known differences or bugs | |---|---:|---:|---| -| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache bounded at 1 GB (#474 open: f16 storage remains) | +| 3D Channel | 8 | 8 | Cryptomatte selection can't be set in the UI and the GPU path is slow (#473; quoted names with commas fixed); multi-layer EXR: one decode per frame for the picture and the channels, cache softly bounded at 1 GB (#474; holding channels as f16 is still to do) | | Audio | 13 | 0 | CPU by design (applied in the mixdown) | | Blur & Sharpen | 16 | 15 | Radial Blur keeps the layer edges as After Effects 2026 (713 now pixel-identical); Gaussian Blur interaction and output to be measured first ([fidelity/ae-2026-acceptance.md](fidelity/ae-2026-acceptance.md)) | | Channel | 13 | 13 | — | diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 741582d9e..648eadcee 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, footage alone only its picture's channels (#474; decoded channels are held as f32) | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 0ebbeeecc..50bd4e7d1 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -120,10 +120,12 @@ Nuke's `beauty`) show the beauty pass: the layer named Combined, Beauty, Image, Color, else the first colour layer that isn't a data pass. Cryptomatte, depth, mist, normals, vectors, positions and IDs are never the picture. EXtractoR reads every other layer and channel. -Footage on its own decodes only the channels of its picture. Once an effect reads an item's -channels (EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for -the picture and the effects together, and kept for a while (up to 1 GB of decoded channels); -several frames asking for the same file at once wait for one decode. +Footage on its own keeps only the channels of its picture (a single-part file still +decompresses all of its pixels to read them). Once an effect reads an item's channels +(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the +picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB +in the browser), a soft bound, as the newest file and files still in use stay whatever their +size. Several frames asking for the same file at once wait for one decode. ## Replacing and reloading diff --git a/docs/gaps.md b/docs/gaps.md index 46a96983a..75e49a1d6 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -65,7 +65,7 @@ are grouped by area; the issue tracker has the rest. | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | -| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage open). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | +| 9 | **Effect behaviour bugs** | #580, #463 Curl Noise, #369 Motion Tile output size, #943 Turbulent Displace Evolution, #473 Cryptomatte selection (quoted names with commas fixed, 3b7f1ace; UI and GPU time open), #474 multi-layer EXR (one decode per frame for picture and channels, byte-bounded cache; f16 storage still to do). Closed: #526, #514, #533–#536 | Each one breaks a look someone built | 10–20 h | G1/G2 · [effects-parity.md](effects-parity.md) | | 10 | **Colour pipeline differences** | #340 ProRes 4444 tagged bt709 but sRGB-encoded (open); ACES 2.0 / OCIO 2.5 configs (26.5) missing; now closed: #494 32-bpc blend modes, #495 and #750 Working Gamma 2.4, #546 / #547 viewer colour | Wrong colour in delivery is a client-visible failure | 15–30 h | G1 · [file-format-parity.md](file-format-parity.md) | | 11 | **Text rendering and layout** | #453 CJK text in comps: the Han fallback now reaches the installed Chinese faces; open: #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | From 5b31b96cde1187eff124da080fb0755dfac2813d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 03:18:16 +0100 Subject: [PATCH 08/35] perf(media): tell the media pool which EXR footage effects read (#474) The pool learned that an effect reads a footage item's channels only at the first aux() call, so the first frame was decoded twice (picture, then in full), and the hint was never cleared after the effect was removed. The session now gives the footage source the whole set of files read by enabled 3D Channel effects (FootageSource::set_aux_footage) with every project change (Session::bump: edits, undo, redo, history jumps, rollbacks, project open), sending it only when it changed. The pool keeps that set and, as a safety net, what aux() learns; a new set drops what was learned. Refs #474 Co-Authored-By: Claude Opus 5.5 --- crates/automation/src/confine.rs | 3 + crates/effects/src/channel3d.rs | 19 +++ crates/effects/src/channel3d/tests.rs | 24 ++++ crates/effects/src/lib.rs | 2 +- crates/engine/src/lib.rs | 4 + crates/engine/src/session/edit.rs | 3 + crates/engine/src/session/render.rs | 39 ++++++ crates/engine/src/tests.rs | 1 + crates/engine/src/tests/aux_footage.rs | 175 +++++++++++++++++++++++++ crates/media/src/exr_channels.rs | 91 +++++++++++++ crates/media/src/pool.rs | 42 +++++- crates/render/src/footage.rs | 9 ++ docs/file-format-parity.md | 2 +- docs/footage.md | 10 +- 14 files changed, 412 insertions(+), 12 deletions(-) create mode 100644 crates/engine/src/tests/aux_footage.rs diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 5f0a7a51a..7cfe11ac2 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -349,6 +349,9 @@ impl FootageSource for ConfinedFootage { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inside(footage).then(|| self.inner.aux(item, footage, t)).flatten() } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inside(footage).then(|| self.inner.model(item, footage)).flatten() } diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs index dc65c3255..59128eb7e 100644 --- a/crates/effects/src/channel3d.rs +++ b/crates/effects/src/channel3d.rs @@ -30,6 +30,25 @@ fn spec(id: &'static str, name: &'static str, params: Vec, ren EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } } +/// The effects that read their layer's auxiliary channels ([`crate::EffectHost::aux`]): every +/// 3D Channel effect but 3D Layer Instance. +const READS_AUX: [&str; 8] = [ + "ec.3d.channelextract", + "ec.3d.cryptomatte", + "ec.3d.depthmatte", + "ec.3d.depthoffield", + "ec.3d.extractor", + "ec.3d.fog3d", + "ec.3d.idmatte", + "ec.3d.identifier", +]; + +/// Does effect `id` read its layer's auxiliary channels ([`crate::EffectHost::aux`])? The media +/// layer then decodes the footage's picture together with its channels (#474). +pub fn reads_aux(id: &str) -> bool { + READS_AUX.contains(&id) +} + fn aux(ctx: &EffectCtx) -> Option> { ctx.env.host?.aux() } diff --git a/crates/effects/src/channel3d/tests.rs b/crates/effects/src/channel3d/tests.rs index e69c2a6aa..9d9342a95 100644 --- a/crates/effects/src/channel3d/tests.rs +++ b/crates/effects/src/channel3d/tests.rs @@ -399,3 +399,27 @@ fn instance3d_replaces_the_pixels_with_the_referenced_layer() { let out = run_fx("ec.3d.instance3d", &[], solid(), 0.0, EffectEnv::default()); assert_eq!(out.img.data, solid().data); } + +/// #474: [`reads_aux`] names exactly the 3D Channel effects that ask the host for channels. +#[test] +fn reads_aux_names_the_effects_that_read_channels() { + struct Counting(AuxHost, std::sync::atomic::AtomicUsize); + impl EffectHost for Counting { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + self.1.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.0.aux() + } + } + for s in specs() { + let host = Counting(scene(), Default::default()); + run_fx(s.id, &[], img(), 0.0, EffectEnv { host: Some(&host), ..Default::default() }); + assert_eq!(host.1.into_inner() > 0, reads_aux(s.id), "{}", s.id); + } + assert!(READS_AUX.iter().all(|id| specs().iter().any(|s| s.id == *id)), "every listed effect exists"); +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c865d4f96..c574509f3 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -115,7 +115,7 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time, // effectcraft-gpu fx_tone. pub use ocio::color_stabilizer_maps; // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes). -pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, selection_hashes}; +pub use channel3d::{CRYPTO_LAYERS, CryptoAux, aux_point, crypto_aux_image, crypto_aux_image_with, id_color, reads_aux, selection_hashes}; pub use ocio::{ColorOp, Straight, Tf, color_program}; pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan}; pub use sim2::{ diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0cc9a7189..54616fdbf 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -244,6 +244,9 @@ pub struct Session { pub footage_stamps: std::collections::HashMap, /// The disk cache's footage salt and the footage it was taken for ([`Session::refresh_disk_salt`]). pub disk_salt: Option<(u64, u64)>, + /// The footage files effects read the channels of, as last given to the footage source + /// ([`Session::refresh_aux_footage`]); `None` = not given yet. + pub aux_footage: Option>, /// How many script-issued commands are running ([`Session::execute_for_script`]): while one /// is, settings that guard what scripts may do can't change (see /// [`prefs::Scripting::same_permissions`]). @@ -350,6 +353,7 @@ impl Default for Session { stored_paths: Default::default(), footage_stamps: Default::default(), disk_salt: None, + aux_footage: None, script_calls: ScriptCalls::default(), } } diff --git a/crates/engine/src/session/edit.rs b/crates/engine/src/session/edit.rs index c2590abc3..d63ec5ead 100644 --- a/crates/engine/src/session/edit.rs +++ b/crates/engine/src/session/edit.rs @@ -98,6 +98,7 @@ impl Session { self.saved_revision = self.revision; } self.refresh_disk_salt(false); + self.refresh_aux_footage(); self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -252,6 +253,8 @@ impl Session { self.project = Arc::new(p); // Its files may have changed since they were last stamped. self.disk_salt = None; + // The set of footage effects read the channels of is sent again, even when it is empty. + self.aux_footage = None; self.history = History::default(); self.state = EditorState { snapping: true, view_layout: 1, ..Default::default() }; self.path = path; diff --git a/crates/engine/src/session/render.rs b/crates/engine/src/session/render.rs index 86127a510..24f6552dd 100644 --- a/crates/engine/src/session/render.rs +++ b/crates/engine/src/session/render.rs @@ -137,4 +137,43 @@ impl Session { self.layer_cache.clear(); self.drop_crypto_probe(); } + + /// Tell the footage source which files effects read the channels of ([`aux_footage`]), so + /// it can decode their pictures and channels at once from the first frame (#474). Called with + /// every project change ([`Session::bump`]); the set is only sent when it changed, or after + /// [`Session::replace_project`] cleared what was sent. + pub(crate) fn refresh_aux_footage(&mut self) { + let paths = aux_footage(&self.project); + if self.aux_footage.as_ref() != Some(&paths) { + self.footage.set_aux_footage(&paths); + self.aux_footage = Some(paths); + } + } +} + +/// The files (each footage item's [`effectcraft_project::Footage::path`], sorted, once each) of +/// the footage layers that have an enabled effect reading their channels +/// ([`effectcraft_effects::reads_aux`]), with the layer's Effects switch on. Several items can +/// use one file: it stays in while any of them is read. +pub(crate) fn aux_footage(project: &effectcraft_project::Project) -> Vec { + use effectcraft_project::{GroupKind, ItemKind, LayerSource}; + let mut paths: Vec = project + .comps() + .flat_map(|(_, c)| c.layers.iter()) + .filter(|l| l.switches.effects) + .filter(|l| { + l.effects() + .is_some_and(|fx| fx.groups().any(|g| g.enabled && matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_effects::reads_aux(effect)))) + }) + .filter_map(|l| match &l.source { + LayerSource::Footage { item } => match &project.item(*item)?.kind { + ItemKind::Footage(f) if !f.path.is_empty() => Some(f.path.clone()), + _ => None, + }, + _ => None, + }) + .collect(); + paths.sort_unstable(); + paths.dedup(); + paths } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 013d2f24c..85d9353d3 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,6 +1,7 @@ mod add_item_options; mod anim_presets; mod anim_tools; +mod aux_footage; mod blender; mod camera_track; mod codec_options; diff --git a/crates/engine/src/tests/aux_footage.rs b/crates/engine/src/tests/aux_footage.rs new file mode 100644 index 000000000..5a74d3b77 --- /dev/null +++ b/crates/engine/src/tests/aux_footage.rs @@ -0,0 +1,175 @@ +//! The footage files effects read the channels of (#474): the session gives the footage source +//! the whole set with every project change ([`Session::bump`]), so a multi-layer OpenEXR read by +//! EXtractoR or Cryptomatte decodes its picture and channels at once from the first frame, and +//! goes back to its picture alone once no effect reads it. + +use std::sync::{Arc, Mutex}; + +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, Project}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::Session; +use crate::render::{FootageSource, RenderOpts}; + +/// Records every set it is given, and the set it had when it was first asked for a frame. +#[derive(Default)] +struct Recorder { + sets: Mutex>>, + at_first_frame: Mutex>>>, +} + +impl Recorder { + fn last(&self) -> Option> { + self.sets.lock().unwrap().last().cloned() + } + fn calls(&self) -> usize { + self.sets.lock().unwrap().len() + } +} + +impl FootageSource for Recorder { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + self.at_first_frame.lock().unwrap().get_or_insert_with(|| self.last()); + Some(Arc::new(Image::new(f.width, f.height))) + } + fn set_aux_footage(&self, paths: &[String]) { + self.sets.lock().unwrap().push(paths.to_vec()); + } +} + +struct Probe; +impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + Ok(Footage { + path: path.into(), + kind: FootageKind::Still, + width: 8, + height: 4, + has_video: true, + frame_rate: FrameRate::from_f64(24.0), + duration: Tick::from_seconds_f64(2.0), + ..Default::default() + }) + } +} + +fn session(rec: &Arc) -> Session { + let mut s = Session { footage: rec.clone(), importer: Some(Arc::new(Probe)), ..Default::default() }; + s.execute("comp.new", json!({"name": "C", "width": 8, "height": 4, "frameRate": 24, "duration": 2})).unwrap(); + s +} + +fn import(s: &mut Session, path: &str) -> serde_json::Value { + s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].clone() +} + +/// Add a layer of `item` to the active comp; its layer id. +fn add_layer(s: &mut Session, item: &serde_json::Value) -> u64 { + s.execute("layer.addItem", json!({"item": item})).unwrap(); + s.state.selected_layers[0].0 +} + +fn paths(v: &[&str]) -> Option> { + Some(v.iter().map(|p| p.to_string()).collect()) +} + +/// Adding, disabling, removing an effect and undoing or redoing each change the set; edits that +/// don't change it don't send it again. +#[test] +fn effect_edits_undo_and_redo_update_the_set() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let item = import(&mut s, "/renders/a.exr"); + let l = add_layer(&mut s, &item); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "no effect reads it yet"); + s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap(); + assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "a blur doesn't read channels"); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&[]), "undoing the add"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "redoing it"); + let calls = rec.calls(); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "Gaussian Blur", "value": false})).unwrap(); + assert_eq!(rec.calls(), calls, "the set didn't change: it isn't sent again"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": false})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "a disabled effect reads nothing"); + s.execute_checked("effect.toggle", json!({"layer": l, "effect": "EXtractoR", "value": true})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&[]), "removed"); + assert!(s.undo()); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "undoing the removal"); + assert!(s.redo()); + assert_eq!(rec.last(), paths(&[]), "redoing it"); + assert!(s.undo()); + s.execute_checked("effect.duplicate", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + s.execute_checked("effect.remove", json!({"layer": l, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the duplicate still reads it"); +} + +/// Opening a project gives the set before the first frame is rendered; opening another sends +/// it again even when it is empty. +#[test] +fn opening_a_project_gives_the_set_before_the_first_frame() { + let made = Arc::new(Recorder::default()); + let mut s = session(&made); + let item = import(&mut s, "/renders/crypto.exr"); + let l = add_layer(&mut s, &item); + s.execute_checked("effect.apply", json!({"effect": "Cryptomatte", "layers": [l]})).unwrap(); + let project: Project = (*s.project).clone(); + let comp = s.state.active_comp.unwrap(); + + let rec = Arc::new(Recorder::default()); + let mut s = Session { footage: rec.clone(), ..Default::default() }; + s.replace_project(project, None); + assert_eq!(rec.last(), paths(&["/renders/crypto.exr"])); + s.render(comp, Tick::ZERO, RenderOpts::default()); + assert_eq!(*rec.at_first_frame.lock().unwrap(), Some(paths(&["/renders/crypto.exr"])), "given before the first frame"); + s.replace_project(Project::default(), None); + assert_eq!(rec.last(), paths(&[])); + let calls = rec.calls(); + s.replace_project(Project::default(), None); + assert_eq!(rec.calls(), calls + 1, "each opened project gets its set"); +} + +/// Two footage items of one file: the file stays in while an effect reads either of them. +#[test] +fn two_items_of_one_file() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/shared.exr"); + let f = match &s.project.item(ItemId(a.as_u64().unwrap())).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + k => panic!("{k:?}"), + }; + let b = s.edit("Add", None, |p, _| Ok(p.add_item("shared copy", crate::color::Label::None, None, ItemKind::Footage(f)))).unwrap(); + let la = add_layer(&mut s, &a); + let lb = add_layer(&mut s, &json!(b.0)); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [la]})).unwrap(); + s.execute_checked("effect.apply", json!({"effect": "ID Matte", "layers": [lb]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "once"); + s.execute_checked("effect.remove", json!({"layer": la, "effect": "EXtractoR"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/shared.exr"]), "the other item still reads it"); + s.execute_checked("effect.remove", json!({"layer": lb, "effect": "ID Matte"})).unwrap(); + assert_eq!(rec.last(), paths(&[])); +} + +/// Replacing a layer's source or an item's file moves the set to the new file. +#[test] +fn source_and_footage_replacement_follow() { + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let a = import(&mut s, "/renders/a.exr"); + let b = import(&mut s, "/renders/b.exr"); + let l = add_layer(&mut s, &a); + s.execute_checked("effect.apply", json!({"effect": "Depth Matte", "layers": [l]})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/a.exr"])); + s.execute_checked("layer.replaceSource", json!({"layers": [l], "item": b})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/b.exr"]), "the layer's new source"); + s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap(); + assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file"); +} diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 8a8f5edcc..3b5ee6084 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -652,6 +652,8 @@ impl ChannelCache { .get_or_init(|| { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); + #[cfg(test)] + count_decode(); let _abandon = Abandon { cache: self, slot: &slot }; let bytes = read()?; // The planes it will allocate count against the budget while it decodes. @@ -726,11 +728,37 @@ pub(crate) fn forget(path: &str) { cache().forget(path); } +/// Stop keeping the decodes of the last pictures for their effects ([`HELD_FILES`]): which +/// footage effects read changed, so they may wait for effects that are gone. Their entries stay +/// cached as any other, least recently used out first. +pub(crate) fn release_held() { + cache().lock().held.clear(); +} + /// Read (and cache, see [`CACHE_BUDGET`]) the channels of the EXR at `path` via `read`. pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { cache().get(path, Use::Channels, read).map(|d| d.aux.clone()) } +#[cfg(test)] +thread_local! { + /// Decodes of a still's pixels on this thread: the channel cache's and the `image` crate's + /// ([`count_decode`]). + static DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Count a decode of a file's pixels on this thread (tests). +#[cfg(test)] +pub(crate) fn count_decode() { + DECODES.with(|d| d.set(d.get() + 1)); +} + +/// The decodes [`count_decode`] counted on this thread since the last call. +#[cfg(test)] +pub(crate) fn take_decodes() -> u64 { + DECODES.with(|d| d.replace(0)) +} + #[cfg(test)] mod tests { use super::*; @@ -1035,6 +1063,69 @@ mod tests { assert_eq!(px, pool.frame_at(&f, effectcraft_time::Tick::ZERO).unwrap().get(0, 0), "the same picture"); } + /// An EXR file with an unnamed RGB layer and a named `GlossDir.R` of `v` (2×2). + fn rgb_and_gloss(v: f32) -> Arc<[u8]> { + flat(&["R", "G", "B", "GlossDir.R"], (2, 2), v).into() + } + + /// #474: footage the project says effects read the channels of + /// ([`FootageSource::set_aux_footage`]) costs one decode from its first frame on: a single + /// still with EXtractoR decodes once. Without it the first frame decodes twice (the picture, + /// then the channels once `aux()` asks). + #[test] + fn given_aux_footage_decodes_once_from_the_first_frame() { + use effectcraft_render::FootageSource; + let (given, learned) = ("/474-given-still.exr", "/474-learned-still.exr"); + let pool = crate::MediaPool::new(); + let mut f = std::collections::HashMap::new(); + for p in [given, learned] { + pool.add_bytes(p, rgb_and_gloss(0.25)); + f.insert(p, crate::probe_bytes(p, rgb_and_gloss(0.25)).unwrap()); + } + pool.set_aux_footage(&[given.to_string()]); + take_decodes(); + let t = effectcraft_time::Tick::ZERO; + pool.frame_at(&f[given], t).unwrap(); + let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap(); + assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); + assert_eq!(take_decodes(), 1, "the picture and the channels share one decode"); + // Learned only at the first `aux()`: the picture was decoded on its own before. + pool.frame_at(&f[learned], t).unwrap(); + pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap(); + assert_eq!(take_decodes(), 2); + } + + /// #474: once the project no longer names footage (its effect removed), later frames decode + /// only their picture again; what `aux()` learned is dropped with each new set too. + #[test] + fn a_new_aux_footage_set_returns_to_picture_only() { + use effectcraft_render::FootageSource; + let paths: Vec = (1..=4).map(|i| format!("/474-unset_{i:04}.exr")).collect(); + let pool = crate::MediaPool::new(); + for p in &paths { + pool.add_bytes(p, rgb_and_gloss(0.5)); + } + let mut f = crate::probe_bytes(&paths[0], rgb_and_gloss(0.5)).unwrap(); + f.kind = effectcraft_project::FootageKind::Sequence; + f.sequence = paths.clone(); + let at = |i: i64| f.frame_rate.tick_of(i); + let in_cache = |p: &str| cache().cached().iter().any(|c| c == p); + pool.set_aux_footage(std::slice::from_ref(&f.path)); + pool.frame_at(&f, at(0)).unwrap(); + assert!(in_cache(&paths[0]), "given: the picture comes from the channel decode"); + pool.set_aux_footage(&[]); + assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it"); + pool.frame_at(&f, at(1)).unwrap(); + assert!(!in_cache(&paths[1]), "no longer given: the picture alone"); + // Learned by `aux()`, then a new set without it: picture only again. + pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap(); + pool.frame_at(&f, at(2)).unwrap(); + assert!(in_cache(&paths[2]), "learned"); + pool.set_aux_footage(&[]); + pool.frame_at(&f, at(3)).unwrap(); + assert!(!in_cache(&paths[3]), "a new set drops what was learned"); + } + /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the /// `image` crate's, bit for bit (first such part, alpha or opaque, data window placed with 0 /// around it), in every alpha interpretation and float curve. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 910f8238f..e9157a5ca 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -245,9 +245,25 @@ struct Inner { /// Movies that failed to open or decode (path to a readable reason; see /// [`FootageSource::problem`]). The first failure of a file is logged, later ones are not. problems: Mutex, String>>, - /// OpenEXR footage (by path) whose channels an effect has read (EXtractoR, Cryptomatte, the 3D + /// OpenEXR footage (by path) whose channels effects read (EXtractoR, Cryptomatte, the 3D /// Channel effects): its pictures come from the same cached decode (#474). - aux_footage: Mutex>, + aux_footage: Mutex, +} + +/// Footage whose channels effects read ([`Inner::aux_footage`]), by [`Footage::path`]. +#[derive(Default)] +struct AuxFootage { + /// What the project says ([`FootageSource::set_aux_footage`]). + given: HashSet, + /// Read through [`FootageSource::aux`] but not given (a reader the project doesn't name, or + /// a source that never gives a set). Cleared with each new set. + learned: HashSet, +} + +impl AuxFootage { + fn contains(&self, path: &str) -> bool { + self.given.contains(path) || self.learned.contains(path) + } } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -868,6 +884,8 @@ impl Inner { { return Ok(img); } + #[cfg(test)] + crate::exr_channels::count_decode(); match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. @@ -964,6 +982,16 @@ impl FootageSource for MediaPool { crate::layered::rasterize_vector(&footage.path, &bytes, footage.layer.as_ref(), footage.page, scale).map(Arc::new) } + fn set_aux_footage(&self, paths: &[String]) { + let given: HashSet = paths.iter().cloned().collect(); + let mut wanted = lock(&self.inner.aux_footage); + if given != wanted.given || !wanted.learned.is_empty() { + crate::exr_channels::release_held(); + } + wanted.given = given; + wanted.learned.clear(); + } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { if footage.missing || !footage.has_video { return None; @@ -975,11 +1003,13 @@ impl FootageSource for MediaPool { let path = loc.key.path.clone(); { let mut wanted = lock(&self.inner.aux_footage); - // (a bound: names of footage items, never cleared otherwise) - if wanted.len() >= 4096 { - wanted.clear(); + if !wanted.contains(&footage.path) { + // (a bound: names of footage items, cleared otherwise only by a new set) + if wanted.learned.len() >= 4096 { + wanted.learned.clear(); + } + wanted.learned.insert(footage.path.clone()); } - wanted.insert(footage.path.clone()); } crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) } diff --git a/crates/render/src/footage.rs b/crates/render/src/footage.rs index db1561ad6..8db3530d3 100644 --- a/crates/render/src/footage.rs +++ b/crates/render/src/footage.rs @@ -48,6 +48,12 @@ pub trait FootageSource: Send + Sync { fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { None } + /// The footage files (by [`Footage::path`]) whose channels effects of the project read + /// through [`FootageSource::aux`] (EXtractoR, Cryptomatte and the other 3D Channel effects), + /// replacing the set given before. A source that decodes a file's picture and its channels + /// separately can decode both at once for these from the first frame on. Sources that don't + /// ignore it. + fn set_aux_footage(&self, _paths: &[String]) {} /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { None @@ -140,6 +146,9 @@ impl FootageSource for CancellableFootage<'_> { fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inner.aux(item, footage, t) } + fn set_aux_footage(&self, paths: &[String]) { + self.inner.set_aux_footage(paths) + } fn model(&self, item: ItemId, footage: &Footage) -> Option> { self.inner.model(item, footage) } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 648eadcee..f56a57319 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte, except the first frame an effect reads and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | diff --git a/docs/footage.md b/docs/footage.md index 50bd4e7d1..2c88b15e8 100644 --- a/docs/footage.md +++ b/docs/footage.md @@ -122,10 +122,12 @@ vectors, positions and IDs are never the picture. EXtractoR reads every other la Footage on its own keeps only the channels of its picture (a single-part file still decompresses all of its pixels to read them). Once an effect reads an item's channels -(EXtractoR, Cryptomatte, the 3D Channel effects), each later frame is decoded once for the -picture and the effects together, and kept for a while: about 1 GB of decoded channels (512 MB -in the browser), a soft bound, as the newest file and files still in use stay whatever their -size. Several frames asking for the same file at once wait for one decode. +(EXtractoR, Cryptomatte, the 3D Channel effects), each frame, the first included, is decoded +once for the picture and the effects together, and kept for a while: about 1 GB of decoded +channels (512 MB in the browser), a soft bound, as the newest file and files still in use +stay whatever their size. Several frames asking for the same file at once wait for one +decode. Once no enabled effect reads the item any more, its frames go back to the picture's +channels alone. ## Replacing and reloading From 0d2153e25990cc56e41acb565d18620852174763 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 03:27:25 +0100 Subject: [PATCH 09/35] test(media): review follow-ups for the aux footage set (#474) - A watch-folder render gives the host session's set back after its own project replaced it on the shared footage source. - Guard gpu fx_depth::IDS against reads_aux drifting apart. - Count picture and channel decodes per thread instead of reading the shared channel cache, so parallel tests can't make the checks flaky. - The engine test fails when no set was ever sent. - Docs: Media Replacement items are learned at their first frame. Refs #474 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/watch_folder.rs | 3 ++ crates/engine/src/tests/aux_footage.rs | 29 +++++++++++++-- crates/gpu/src/tests/fx_depth.rs | 7 ++++ crates/media/src/exr_channels.rs | 49 +++++++++++++++----------- crates/media/src/pool.rs | 2 +- docs/file-format-parity.md | 2 +- 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/cmd/watch_folder.rs b/crates/engine/src/cmd/watch_folder.rs index fd41071ed..6e480a9e6 100644 --- a/crates/engine/src/cmd/watch_folder.rs +++ b/crates/engine/src/cmd/watch_folder.rs @@ -88,6 +88,9 @@ fn render_project(host: &Session, path: &Path) -> Value { let failed = s.project.render_queue.iter().any(|i| matches!(i.status, RenderStatus::Failed(_))); Ok(json!({"state": if failed { "failed" } else { "done" }, "items": items})) })(); + // The project shared `host`'s footage source and gave it its own set of footage effects read + // the channels of (#474): give back the host's. + host.footage.set_aux_footage(host.aux_footage.as_deref().unwrap_or(&[])); let mut v = match result { Ok(v) => v, Err(e) => json!({"state": "failed", "error": e, "items": []}), diff --git a/crates/engine/src/tests/aux_footage.rs b/crates/engine/src/tests/aux_footage.rs index 5a74d3b77..01c3c1c96 100644 --- a/crates/engine/src/tests/aux_footage.rs +++ b/crates/engine/src/tests/aux_footage.rs @@ -83,9 +83,9 @@ fn effect_edits_undo_and_redo_update_the_set() { let mut s = session(&rec); let item = import(&mut s, "/renders/a.exr"); let l = add_layer(&mut s, &item); - assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "no effect reads it yet"); + assert_eq!(rec.last(), paths(&[]), "no effect reads it yet"); s.execute_checked("effect.apply", json!({"effect": "Gaussian Blur", "layers": [l]})).unwrap(); - assert_eq!(rec.last().unwrap_or_default(), Vec::::new(), "a blur doesn't read channels"); + assert_eq!(rec.last(), paths(&[]), "a blur doesn't read channels"); s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); assert_eq!(rec.last(), paths(&["/renders/a.exr"])); assert!(s.undo()); @@ -173,3 +173,28 @@ fn source_and_footage_replacement_follow() { s.execute_checked("file.replaceFootage", json!({"item": b, "path": "/renders/c.exr"})).unwrap(); assert_eq!(rec.last(), paths(&["/renders/c.exr"]), "the item's new file"); } + +/// A watch-folder render opens its project in a session sharing the host's footage source: the +/// host's set is given back afterwards. +#[test] +fn a_watch_folder_render_gives_the_host_set_back() { + let dir = std::env::temp_dir().join(format!("ec-aux-footage-watch-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut other = Session::default(); + other.execute("comp.new", json!({"name": "Idle", "width": 8, "height": 8, "duration": 1})).unwrap(); + other.execute("file.saveAs", json!({"path": dir.join("idle.ecproj").to_string_lossy()})).unwrap(); + + let rec = Arc::new(Recorder::default()); + let mut s = session(&rec); + let item = import(&mut s, "/renders/a.exr"); + let l = add_layer(&mut s, &item); + s.execute_checked("effect.apply", json!({"effect": "EXtractoR", "layers": [l]})).unwrap(); + let calls = rec.calls(); + let r = s.execute("file.watchFolder", json!({"folder": dir.to_string_lossy()})).unwrap(); + assert_eq!(r["rendered"].as_array().map(Vec::len), Some(1), "{r}"); + assert!(rec.calls() > calls + 1, "the watched project gave its own set"); + assert_eq!(rec.last(), paths(&["/renders/a.exr"]), "the host's set is back"); + s.execute("file.watchFolder", json!({"stop": true})).unwrap(); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/gpu/src/tests/fx_depth.rs b/crates/gpu/src/tests/fx_depth.rs index 8a6665655..ebc75919d 100644 --- a/crates/gpu/src/tests/fx_depth.rs +++ b/crates/gpu/src/tests/fx_depth.rs @@ -208,6 +208,13 @@ fn on() -> Value { Value::Bool(true) } +/// #474: every effect done here reads aux channels, so the session gives the media layer its +/// footage ([`effectcraft_effects::reads_aux`]). +#[test] +fn every_effect_here_reads_aux() { + assert!(crate::fx_depth::IDS.iter().all(|i| effectcraft_effects::reads_aux(i))); +} + #[test] fn channel_extract() { case("ec.3d.channelextract", &[("blackPoint", n(0.0)), ("whitePoint", n(1200.0))]); diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index 3b5ee6084..c3e7cfc3d 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -653,7 +653,7 @@ impl ChannelCache { decoded_here = true; self.decodes.fetch_add(1, Ordering::Relaxed); #[cfg(test)] - count_decode(); + count_decode(true); let _abandon = Abandon { cache: self, slot: &slot }; let bytes = read()?; // The planes it will allocate count against the budget while it decodes. @@ -742,21 +742,27 @@ pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Op #[cfg(test)] thread_local! { - /// Decodes of a still's pixels on this thread: the channel cache's and the `image` crate's - /// ([`count_decode`]). - static DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Decodes of a still's pixels on this thread ([`count_decode`]): the `image` crate's, then + /// the channel cache's. + static DECODES: std::cell::Cell<[u64; 2]> = const { std::cell::Cell::new([0; 2]) }; } -/// Count a decode of a file's pixels on this thread (tests). +/// Count a decode of a file's pixels on this thread (tests): by the channel cache (`channels`) +/// or by the `image` crate. #[cfg(test)] -pub(crate) fn count_decode() { - DECODES.with(|d| d.set(d.get() + 1)); +pub(crate) fn count_decode(channels: bool) { + DECODES.with(|d| { + let mut n = d.get(); + n[usize::from(channels)] += 1; + d.set(n); + }); } -/// The decodes [`count_decode`] counted on this thread since the last call. +/// The decodes [`count_decode`] counted on this thread since the last call: by the `image` crate +/// and by the channel cache. #[cfg(test)] -pub(crate) fn take_decodes() -> u64 { - DECODES.with(|d| d.replace(0)) +pub(crate) fn take_decodes() -> [u64; 2] { + DECODES.with(|d| d.replace([0; 2])) } #[cfg(test)] @@ -1088,11 +1094,11 @@ mod tests { pool.frame_at(&f[given], t).unwrap(); let aux = pool.aux(effectcraft_project::ItemId(1), &f[given], t).unwrap(); assert_eq!(aux.get("GlossDir.R").and_then(|p| p.first()).copied(), Some(0.25)); - assert_eq!(take_decodes(), 1, "the picture and the channels share one decode"); + assert_eq!(take_decodes(), [0, 1], "the picture and the channels share one decode"); // Learned only at the first `aux()`: the picture was decoded on its own before. pool.frame_at(&f[learned], t).unwrap(); pool.aux(effectcraft_project::ItemId(2), &f[learned], t).unwrap(); - assert_eq!(take_decodes(), 2); + assert_eq!(take_decodes(), [1, 1]); } /// #474: once the project no longer names footage (its effect removed), later frames decode @@ -1109,21 +1115,22 @@ mod tests { f.kind = effectcraft_project::FootageKind::Sequence; f.sequence = paths.clone(); let at = |i: i64| f.frame_rate.tick_of(i); - let in_cache = |p: &str| cache().cached().iter().any(|c| c == p); + // (counted on this thread, so other tests sharing the channel cache don't change them) + let frame = |i: i64| { + take_decodes(); + pool.frame_at(&f, at(i)).unwrap(); + take_decodes() + }; pool.set_aux_footage(std::slice::from_ref(&f.path)); - pool.frame_at(&f, at(0)).unwrap(); - assert!(in_cache(&paths[0]), "given: the picture comes from the channel decode"); + assert_eq!(frame(0), [0, 1], "given: the picture comes from the channel decode"); pool.set_aux_footage(&[]); assert!(!cache().lock().held.iter().any(|(p, _)| *p == paths[0]), "no longer kept once no effect reads it"); - pool.frame_at(&f, at(1)).unwrap(); - assert!(!in_cache(&paths[1]), "no longer given: the picture alone"); + assert_eq!(frame(1), [1, 0], "no longer given: the picture alone"); // Learned by `aux()`, then a new set without it: picture only again. pool.aux(effectcraft_project::ItemId(1), &f, at(1)).unwrap(); - pool.frame_at(&f, at(2)).unwrap(); - assert!(in_cache(&paths[2]), "learned"); + assert_eq!(frame(2), [0, 1], "learned"); pool.set_aux_footage(&[]); - pool.frame_at(&f, at(3)).unwrap(); - assert!(!in_cache(&paths[3]), "a new set drops what was learned"); + assert_eq!(frame(3), [1, 0], "a new set drops what was learned"); } /// #474: the picture made of the cached planes of a file with exact R, G, B channels is the diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index e9157a5ca..b0b2f2539 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -885,7 +885,7 @@ impl Inner { return Ok(img); } #[cfg(test)] - crate::exr_channels::count_decode(); + crate::exr_channels::count_decode(false); match crate::probe::load_oriented(&bytes, crate::still_format(path, &bytes)) { Ok(img) => Ok(dynamic_to_image(&img, op)), // A multi-layer OpenEXR file without an unnamed RGB layer: its colour layer. diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index f56a57319..ba8c1db7a 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -41,7 +41,7 @@ trips and renders, checked by tests, not against After Effects), **partial** (na | PNG | R/W | yes | yes (sequence) | good | | JPEG | R/W | yes | yes (sequence) | good | | TIFF | R/W | yes | yes (sequence) | good | -| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | +| OpenEXR (multi-layer, Cryptomatte) | R/W | yes | yes (32-bit sequence) | good; HTJ2K compression fails (#482); a multi-layer frame decodes once for its picture and EXtractoR / Cryptomatte from the first frame on, except Media Replacement items, which are learned at their first frame, and files whose picture the `image` crate still decodes (#474); footage alone keeps only its picture's channels, though a single-part file still decompresses every channel's pixels; decoded channels are held as f32 | | Photoshop PSD / PSB | R (footage, composition) / W (layers, sequence) | yes (layers, groups, masks, styles, smart objects) | Save Frame As ▸ Photoshop Layers; no Photoshop sequence output | good, from Adobe's published spec | | Illustrator `.ai`, PDF, EPS | R | yes (vector, Continuously Rasterize, Create Shapes; Composition and Composition - Retain Layer Sizes, #1536) | — | partial: no JPX / JBIG2 images, no Illustrator procset EPS | | SVG | R | yes | — | good; SVG markup on the clipboard pastes as a shape layer (26.3, #738); Illustrator's own clipboard data (PDF/AI) doesn't | From 6572514eb18dc4803f567bc2fd34f4b5c65c3565 Mon Sep 17 00:00:00 2001 From: Joseph <56056764+00xJS@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:55:02 -0700 Subject: [PATCH 10/35] GPU: size the memory cap from the card's video memory on Linux (fixes #1487) (#1706) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux, the discrete-GPU memory cap now comes from the card's video memory: min(3 GiB, ¾ of amdgpu's `mem_info_vram_total`), so a 2 GB RX 550 gets 1.5 GiB. The card is matched by PCI address, falling back to the smallest card with the same ids. Also folds in CodeRabbit follow-ups from #1856 (PatchMatch flags buffer limit preflight), #1490 (arbitrary-map index wrap), #1836 (round_away for |x| >= 2^23) and #3025 (saturating FrameCache byte accounting). Fixes #1487 Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> --- crates/gpu/src/fx_tone.rs | 20 ++++- crates/gpu/src/inpaint.rs | 35 ++++++++ crates/gpu/src/memcap.rs | 128 +++++++++++++++++++++++++---- crates/gpu/src/shaders/common.wgsl | 4 + crates/gpu/src/tests.rs | 28 +++++++ crates/ui-egui/src/frames.rs | 4 +- 6 files changed, 198 insertions(+), 21 deletions(-) diff --git a/crates/gpu/src/fx_tone.rs b/crates/gpu/src/fx_tone.rs index f6e7062e2..0274cdb0f 100644 --- a/crates/gpu/src/fx_tone.rs +++ b/crates/gpu/src/fx_tone.rs @@ -190,7 +190,9 @@ fn arbitrary_map_table(map: &[f32], phase: f64) -> Option> { } let at = |v: f32| { let x = (v as f64 * scale + offset).rem_euclid(n as f64); - let i = x.floor() as usize; + // `rem_euclid` can round a tiny negative phase up to `n`: wrap the index like the CPU + // lookup (`color3::arbitrary_map`) so both endpoints come from the start of the map. + let i = x.floor() as usize % n; let a = *map.get(i).unwrap_or(&0.0); let b = *map.get((i + 1).min(n - 1)).unwrap_or(&a); (a + (b - a) * (x - x.floor()) as f32, (b - a) * scale as f32) @@ -435,3 +437,19 @@ fn point(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { } run(e, &p, b, data) } + +#[cfg(test)] +mod map_tests { + use super::arbitrary_map_table; + + /// A phase just below zero rounds `rem_euclid` up to `n` at input 0: the table must hold the + /// map's first entry and the slope towards its second, as the CPU lookup does. + #[test] + fn arbitrary_map_wraps_a_phase_rounded_to_the_end() { + let map = [0.25f32, 0.75, 1.0]; + let table = arbitrary_map_table(&map, -1e-300).unwrap(); + assert_eq!(table[0], 0.0); + assert_eq!(table[1], 0.25, "value at input 0 is map[0]"); + assert_eq!(table[2], 0.5 * 2.0, "slope from map[0] to map[1], times n - 1"); + } +} diff --git a/crates/gpu/src/inpaint.rs b/crates/gpu/src/inpaint.rs index 25cbc31b9..8adbc3069 100644 --- a/crates/gpu/src/inpaint.rs +++ b/crates/gpu/src/inpaint.rs @@ -577,6 +577,12 @@ pub(crate) fn patch_image(g: &GpuContext, img: &Image, hole: &[bool], init: Opti /// The plan of one level: its flags (`bit 0` hole, `bit 1` a target centre) and the dispatch /// bounds of its targets, holes and field. fn plan_level(e: &mut Enc, hole: &[bool], w: u32, h: u32, patch: usize) -> Option { + // Texture and buffer limits are independent: a level that fits as a texture can still need a + // flags buffer over the device's storage binding limit, and that validation error would + // retire the context. Decline the GPU path instead (the fill runs on the CPU). + if !flags_fit(&e.g.device.limits(), w, h) { + return None; + } let (iw, ih) = (w as usize, h as usize); let r = radius(patch, w, h); let near = dilate(hole, iw, ih, r as usize); @@ -585,6 +591,16 @@ fn plan_level(e: &mut Enc, hole: &[bool], w: u32, h: u32, patch: usize) -> Optio Some(Lv { w, h, r, near: near_bb, holes: bbox(hole, iw, ih)?, field: grow(near_bb, r + 1, w, h), flags: e.data(&flags) }) } +/// Whether a level's `w × h` f32 flags buffer (at least the 16 bytes [`Enc::data`] allocates) +/// can be created and bound as one storage buffer on a device with `limits`. +fn flags_fit(limits: &wgpu::Limits, w: u32, h: u32) -> bool { + u64::from(w) + .checked_mul(u64::from(h)) + .and_then(|n| n.checked_mul(4)) + .map(|b| b.max(16)) + .is_some_and(|b| b <= limits.max_buffer_size && b <= limits.max_storage_buffer_binding_size) +} + /// The PatchMatch run at one level: the random stream's seed and the EM round and iteration /// counts. struct PmRun { @@ -727,3 +743,22 @@ pub(crate) fn distance_histogram(g: &GpuContext, field: &Image, near: &[bool], w } out } + +#[cfg(test)] +mod limit_tests { + use super::flags_fit; + + /// A level whose flags buffer is over the storage binding limit is declined before it is + /// created (the default 128 MiB binding holds 33,554,432 flags: 8192 × 4096). + #[test] + fn flags_buffer_is_checked_against_the_device_limits() { + let l = wgpu::Limits::default(); + assert!(flags_fit(&l, 1, 1)); + assert!(flags_fit(&l, 8192, 4096)); + assert!(!flags_fit(&l, 8192, 4097)); + assert!(!flags_fit(&l, 16384, 16384)); + let small = wgpu::Limits { max_buffer_size: 1024, ..wgpu::Limits::default() }; + assert!(flags_fit(&small, 16, 16)); + assert!(!flags_fit(&small, 16, 17)); + } +} diff --git a/crates/gpu/src/memcap.rs b/crates/gpu/src/memcap.rs index 1c2ce6456..9c745322f 100644 --- a/crates/gpu/src/memcap.rs +++ b/crates/gpu/src/memcap.rs @@ -10,10 +10,13 @@ //! be reclaimed; a warning is logged once. //! //! wgpu has no safe query for the adapter's memory size (`max_buffer_size` is capped at `i32::MAX` -//! on Direct3D 12 and Linux/Mesa Vulkan), so the default cap is conservative per device type; +//! on Direct3D 12 and Linux/Mesa Vulkan), so the default cap is conservative per device type, +//! and lower on a discrete card whose memory the OS reports without `unsafe` (Linux's amdgpu +//! driver, in sysfs: a 2–4 GB card ran out of memory and took the system down, #1487); //! `EFFECTCRAFT_GPU_MEMCAP_MB` overrides it (0 = off). Backends without an allocator report //! (Metal, WebGPU, GL) are never gated. +use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; use std::time::Duration; @@ -44,11 +47,11 @@ pub(crate) struct MemCap { impl MemCap { pub(crate) fn new(info: &wgpu::AdapterInfo) -> MemCap { let env = std::env::var(MEMCAP_ENV).ok(); - Self::from_env(env.as_deref(), info.device_type) + Self::from_env(env.as_deref(), info.device_type, vram(info)) } - fn from_env(var: Option<&str>, device_type: wgpu::DeviceType) -> MemCap { - let cap = cap_from_env(var, device_type); + fn from_env(var: Option<&str>, device_type: wgpu::DeviceType, vram: Option) -> MemCap { + let cap = cap_from_env(var, device_type, vram); MemCap { cap, hysteresis: default_hysteresis(cap), @@ -145,12 +148,12 @@ impl MemCap { /// The cap from the `EFFECTCRAFT_GPU_MEMCAP_MB` value: whole MiB (0 turns the gate off), or the /// default for `device_type` when unset, or (with a warning) when not a number or too large. -fn cap_from_env(var: Option<&str>, device_type: wgpu::DeviceType) -> u64 { - let Some(v) = var else { return default_cap(device_type) }; +fn cap_from_env(var: Option<&str>, device_type: wgpu::DeviceType, vram: Option) -> u64 { + let Some(v) = var else { return default_cap(device_type, vram) }; match v.trim().parse::().ok().and_then(|mb| mb.checked_mul(MIB)) { Some(cap) => cap, None => { - let cap = default_cap(device_type); + let cap = default_cap(device_type, vram); log::warn!(target: "effectcraft_gpu", "{MEMCAP_ENV}={v:?} is not a whole number of MiB that fits in 64 bits: using {} MiB", cap / MIB); cap } @@ -159,15 +162,59 @@ fn cap_from_env(var: Option<&str>, device_type: wgpu::DeviceType) -> u64 { /// Discrete cards: 3 GiB, low enough to fire on 4–8 GB cards with headroom for the driver and /// desktop while leaving ~1 GiB above the default 1.5 GiB upload cache + 256 MB pool (larger -/// cards can raise it with `EFFECTCRAFT_GPU_MEMCAP_MB`). Integrated and software adapters share -/// system memory: 2 GiB. -fn default_cap(t: wgpu::DeviceType) -> u64 { +/// cards can raise it with `EFFECTCRAFT_GPU_MEMCAP_MB`), and at most three quarters of the card's +/// memory when it is known (`vram`), so a 2 GB card gets 1.5 GiB (#1487). Integrated and software +/// adapters share system memory: 2 GiB, whatever their carve-out. +fn default_cap(t: wgpu::DeviceType, vram: Option) -> u64 { match t { - wgpu::DeviceType::DiscreteGpu => 3 * GIB, + wgpu::DeviceType::DiscreteGpu => vram.map_or(3 * GIB, |v| (v / 4).saturating_mul(3).min(3 * GIB)), _ => 2 * GIB, } } +/// The adapter's dedicated memory in bytes, where the OS reports it without `unsafe`. +fn vram(info: &wgpu::AdapterInfo) -> Option { + if !cfg!(target_os = "linux") || info.device_type != wgpu::DeviceType::DiscreteGpu { + return None; + } + let bytes = vram_in_sysfs(Path::new("/sys/class/drm"), info.vendor, info.device, &info.device_pci_bus_id); + if let Some(b) = bytes { + log::info!(target: "effectcraft_gpu", "gpu {}: {} MiB of video memory (sysfs)", info.name, b >> 20); + } + bytes +} + +/// `mem_info_vram_total` (amdgpu) of the DRM card under `drm` whose PCI vendor and device ids are +/// `vendor` and `device`. Connector entries (`card1-DP-1`) are skipped. Variants of one model share +/// those ids (the RX 550 ships with 2 and 4 GB), so the card at PCI address `bus` (`uevent`'s +/// `PCI_SLOT_NAME`, wgpu's `device_pci_bus_id`) wins; without one, the smallest matching card. +fn vram_in_sysfs(drm: &Path, vendor: u32, device: u32, bus: &str) -> Option { + let id = |path: &Path| std::fs::read_to_string(path).ok().and_then(|s| u32::from_str_radix(s.trim().trim_start_matches("0x"), 16).ok()); + let slot = |dev: &Path| { + std::fs::read_to_string(dev.join("uevent")).ok()?.lines().find_map(|l| l.strip_prefix("PCI_SLOT_NAME=").map(|s| s.trim().to_ascii_lowercase())) + }; + let mut cards: Vec<_> = std::fs::read_dir(drm) + .ok()? + .flatten() + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with("card") && !n.contains('-'))) + .collect(); + cards.sort(); + let found: Vec<(Option, u64)> = cards + .iter() + .filter_map(|card| { + let dev = card.join("device"); + if id(&dev.join("vendor")) != Some(vendor) || id(&dev.join("device")) != Some(device) { + return None; + } + let bytes = std::fs::read_to_string(dev.join("mem_info_vram_total")).ok()?.trim().parse::().ok().filter(|&b| b > 0)?; + Some((slot(&dev), bytes)) + }) + .collect(); + let bus = bus.trim().to_ascii_lowercase(); + found.iter().find(|(s, _)| !bus.is_empty() && s.as_deref() == Some(bus.as_str())).map(|&(_, b)| b).or_else(|| found.iter().map(|&(_, b)| b).min()) +} + /// Minimum reclaimable bytes before a wait is worth draining an encoder (1/8 of the cap, /// clamped between 1 MiB and 256 MiB, or 1/4 of caps smaller than 4 MiB). fn default_hysteresis(cap: u64) -> u64 { @@ -180,20 +227,20 @@ mod tests { #[test] fn defaults_and_cadence() { - assert_eq!(default_cap(wgpu::DeviceType::DiscreteGpu), 3 * GIB); - assert_eq!(default_cap(wgpu::DeviceType::IntegratedGpu), 2 * GIB); - let m = MemCap::from_env(None, wgpu::DeviceType::IntegratedGpu); + assert_eq!(default_cap(wgpu::DeviceType::DiscreteGpu, None), 3 * GIB); + assert_eq!(default_cap(wgpu::DeviceType::IntegratedGpu, None), 2 * GIB); + let m = MemCap::from_env(None, wgpu::DeviceType::IntegratedGpu, None); assert_eq!(m.cap(), 2 * GIB); let due = (0..2 * EVERY).filter(|_| m.due()).count(); assert_eq!(due, 2); - let off = MemCap::from_env(Some("0"), wgpu::DeviceType::DiscreteGpu); + let off = MemCap::from_env(Some("0"), wgpu::DeviceType::DiscreteGpu, None); assert_eq!(off.cap(), 0); assert!((0..EVERY).all(|_| !off.due())); } #[test] fn override_is_whole_mib_or_the_default() { - let cap = |v| MemCap::from_env(Some(v), wgpu::DeviceType::DiscreteGpu).cap(); + let cap = |v| MemCap::from_env(Some(v), wgpu::DeviceType::DiscreteGpu, None).cap(); assert_eq!(cap("2048"), 2 * GIB); assert_eq!(cap(" 8 "), 8 * MIB); assert_eq!(cap("17592186044415"), 17592186044415 * MIB, "the largest value that fits"); @@ -204,7 +251,7 @@ mod tests { #[test] fn hysteresis_skips_non_reclaimable_and_unfreed_memory() { - let m = MemCap::from_env(Some("2048"), wgpu::DeviceType::IntegratedGpu); + let m = MemCap::from_env(Some("2048"), wgpu::DeviceType::IntegratedGpu, None); // Upload cache + pool + live frames (2.15 GiB) sit over the 2 GiB cap with only 50 MiB // reclaimable: waiting would free almost nothing, so the gate must stay quiet. let kept = 2150 * MIB; @@ -225,7 +272,7 @@ mod tests { #[test] fn kept_memory_over_the_cap_warns_once_and_never_waits() { - let m = MemCap::from_env(Some("8"), wgpu::DeviceType::IntegratedGpu); + let m = MemCap::from_env(Some("8"), wgpu::DeviceType::IntegratedGpu, None); // Cap 8 MiB: kept alone over the cap with reclaimable under the hysteresis → no wait, warn once. assert!(!m.should_wait(20 * MIB, 20 * MIB)); assert!(m.warned()); @@ -233,4 +280,49 @@ mod tests { assert!(!m.should_wait(20 * MIB, 19 * MIB + MIB / 2)); assert!(m.should_wait(20 * MIB, 4 * MIB), "16 MiB reclaimable is over the hysteresis"); } + + /// #1487: a 2–4 GB card gets three quarters of its memory instead of the 3 GiB default; a + /// larger card keeps the default, and an integrated GPU's carve-out doesn't lower its cap. + #[test] + fn known_video_memory_lowers_the_discrete_cap() { + let d = wgpu::DeviceType::DiscreteGpu; + assert_eq!(default_cap(d, Some(2 * GIB)), 3 * GIB / 2); + assert_eq!(default_cap(d, Some(4 * GIB)), 3 * GIB); + assert_eq!(default_cap(d, Some(16 * GIB)), 3 * GIB, "never above the default"); + assert_eq!(default_cap(wgpu::DeviceType::IntegratedGpu, Some(GIB / 2)), 2 * GIB); + assert_eq!(cap_from_env(None, d, Some(2 * GIB)), 3 * GIB / 2); + assert_eq!(cap_from_env(Some("lots"), d, Some(2 * GIB)), 3 * GIB / 2, "a bad override falls back to it"); + assert_eq!(cap_from_env(Some("4096"), d, Some(2 * GIB)), 4 * GIB, "the override still wins"); + } + + #[test] + fn video_memory_is_read_from_the_matching_drm_card() { + let dir = std::env::temp_dir().join(format!("effectcraft-memcap-{}", std::process::id())); + let card = |name: &str, vendor: &str, device: &str, vram: Option<&str>| { + let dev = dir.join(name).join("device"); + std::fs::create_dir_all(&dev).unwrap(); + std::fs::write(dev.join("vendor"), vendor).unwrap(); + std::fs::write(dev.join("device"), device).unwrap(); + if let Some(v) = vram { + std::fs::write(dev.join("mem_info_vram_total"), v).unwrap(); + } + }; + // An integrated Intel GPU (no amdgpu file), the RX 550 and a connector entry. + card("card0", "0x8086\n", "0x9bc5\n", None); + card("card1", "0x1002\n", "0x699f\n", Some("4294967296\n")); + std::fs::create_dir_all(dir.join("card1-DP-1")).unwrap(); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x699f, ""), Some(4 * GIB)); + assert_eq!(vram_in_sysfs(&dir, 0x8086, 0x9bc5, ""), None, "no mem_info_vram_total"); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x73bf, ""), None, "no such card"); + assert_eq!(vram_in_sysfs(&dir.join("missing"), 0x1002, 0x699f, ""), None); + // A 2 GB RX 550 next to the 4 GB one: same ids, told apart by PCI address. + card("card2", "0x1002\n", "0x699f\n", Some("2147483648\n")); + std::fs::write(dir.join("card1/device/uevent"), "DRIVER=amdgpu\nPCI_SLOT_NAME=0000:01:00.0\n").unwrap(); + std::fs::write(dir.join("card2/device/uevent"), "DRIVER=amdgpu\nPCI_SLOT_NAME=0000:03:00.0\n").unwrap(); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x699f, "0000:01:00.0"), Some(4 * GIB)); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x699f, "0000:03:00.0"), Some(2 * GIB)); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x699f, ""), Some(2 * GIB), "unknown address: the smaller card"); + assert_eq!(vram_in_sysfs(&dir, 0x1002, 0x699f, "0000:09:00.0"), Some(2 * GIB), "no such address: the smaller card"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/gpu/src/shaders/common.wgsl b/crates/gpu/src/shaders/common.wgsl index 1d8a701b6..5541a42c4 100644 --- a/crates/gpu/src/shaders/common.wgsl +++ b/crates/gpu/src/shaders/common.wgsl @@ -247,6 +247,10 @@ fn powz(x: f32, y: f32) -> f32 { // that feeds `x - t` into one fma (AMD does), skipping the product's rounding, so `5/6 · 3`, // which rounds to exactly 2.5 on the CPU, came out 0.49999994 above 2 and rounded down (#1605). fn round_away(x: f32) -> f32 { + // From 2^23 up every f32 is an integer, and `t + 0.5` would round to even (8388608 → 8388609). + if (abs(x) >= 8388608.0) { + return x; + } let t = trunc(x); if (x >= 0.0) { return select(t, t + 1.0, x >= t + 0.5); diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 96e49dd7d..35694e8a8 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -69,6 +69,10 @@ fn math_probe(@builtin(global_invocation_id) gid: vec3) { textureStore(out, p, vec4(q, fxs_posterize_bin(q, P.u[0].y), 0.0, 1.0)); return; } + if (P.u[0].x == 2u) { + textureStore(out, p, vec4(round_away(v), 0.0, 0.0, 1.0)); + return; + } textureStore(out, p, vec4(sin_p(v), cos_p(v), 0.0, 1.0)); } "#; @@ -127,6 +131,30 @@ fn alpha_division_and_posterization_match_f32_rounding() { } } +/// `round_away` is `f32::round`, also from 2^23 up where every value is already an integer +/// (integer IDs there must not merge: 8388608 and 8388609 both rounded to 8388609). +#[test] +fn round_away_matches_f32_round_for_large_values() { + let Some(g) = gpu() else { return }; + let mut values = vec![]; + for m in [1.0f32, -1.0] { + for v in [0.0f32, 0.5, 1.5, 2.5, 0.49999997, 4194303.5, 8388607.5, 8388608.0, 8388609.0, 8388610.0, 16777216.0, 3.0e9] { + values.push([m * v, 0.0, 0.0, 1.0]); + } + } + let img = Image { width: values.len() as u32, height: 1, data: values.clone() }; + let input = g.ctx.upload_image(&img).unwrap(); + let mut e = crate::context::Enc::new(&g.ctx); + let out = e.scratch(img.width, 1); + let mut params = crate::context::Params::default(); + params.u[0] = [2, 0, 0, 0]; + e.pixels("math_probe", ¶ms, &input, None, &out, None); + let result = e.download(&out).unwrap(); + for (v, got) in values.iter().zip(result.data) { + assert_eq!(v[0].round(), got[0], "round_away({})", v[0]); + } +} + /// Prove that the numerical fixes run on the device, without per-effect image /// downloads or CPU fallback. Compiler-limited DX12 devices retain their tested /// fallback contract; these kernels must compile on the Vulkan test adapters. diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 6a98abfe6..9aad2a81c 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -235,9 +235,9 @@ impl Cache { self.gpu_pressure.is_none_or(|last| last.elapsed() >= GPU_PRESSURE_COOLDOWN) } fn forget(&mut self, img: &FrameImage) { - self.bytes -= img.bytes(); + self.bytes = self.bytes.saturating_sub(img.bytes()); if img.is_gpu() { - self.gpu_bytes -= img.bytes(); + self.gpu_bytes = self.gpu_bytes.saturating_sub(img.bytes()); } } fn remove(&mut self, k: &FrameKey) { From 58ca9c1c1b163fdada41291847f3d0f7559339b2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 3 Nov 2026 20:23:08 -0500 Subject: [PATCH 11/35] OCIO: refuse a .csp LUT whose header claims an oversized table before reserving it (#2938) --- crates/effects/src/ocio/lut.rs | 29 ++++++++++++++++++++++------- crates/effects/src/ocio/tests.rs | 8 ++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/effects/src/ocio/lut.rs b/crates/effects/src/ocio/lut.rs index 8a6481eb6..de931a411 100644 --- a/crates/effects/src/ocio/lut.rs +++ b/crates/effects/src/ocio/lut.rs @@ -57,7 +57,7 @@ pub fn parse_spi1d(text: &str) -> Option { } } // A 1D table is interpolated by index, so cap it like the lattices (2²⁴ entries). - if !(2..=1 << 24).contains(&len) + if !(2..=MAX_1D).contains(&len) || !matches!(comps, 1 | 3) || values.len() != len.checked_mul(comps)? || from[1] <= from[0] @@ -79,7 +79,7 @@ pub fn parse_spi3d(text: &str) -> Option { lines.next()?; let size: Vec = lines.next()?.split_whitespace().map(|t| t.parse().ok()).collect::>()?; let n = *size.first()?; - if size.len() != 3 || size.iter().any(|s| *s != n) || !(2..=129).contains(&n) { + if size.len() != 3 || size.iter().any(|s| *s != n) || !(2..=MAX_LATTICE).contains(&n) { return None; } let mut d3 = vec![[0.0f32; 3]; n * n * n]; @@ -154,6 +154,11 @@ pub fn parse_3dl(text: &str) -> Option { Some(Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }) } +/// Largest lattice side the lattice readers accept (`.spi3d`, `.csp`). +const MAX_LATTICE: usize = 129; +/// Most entries of a 1D table (interpolated by index, so capped like the lattices: 2²⁴). +const MAX_1D: usize = 1 << 24; + /// Parse a cineSpace `.csp` LUT (1D or 3D, with its three pre-LUT shapers). pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); @@ -185,8 +190,15 @@ pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { if sz.len() != 3 || sz[0] != sz[1] || sz[1] != sz[2] { return None; } - let n = sz[0] as usize; - let mut d3 = Vec::with_capacity(n * n * n); + // The size comes from the file: an integer lattice size the other lattice readers accept + // (2..=129), checked before anything is allocated. Rows are read before they are stored, + // so a short file never reserves the table it claims. + let n = sz[0]; + if n.fract() != 0.0 || !(2.0..=MAX_LATTICE as f32).contains(&n) { + return None; + } + let n = n as usize; + let mut d3 = Vec::new(); for _ in 0..n * n * n { let v = nums(lines.next()?)?; if v.len() != 3 { @@ -194,10 +206,13 @@ pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { } d3.push([v[0], v[1], v[2]]); } - (n >= 2).then_some((Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + Some((Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) } else { let n: usize = lines.next()?.parse().ok()?; - let mut d1 = Vec::with_capacity(n); + if !(2..=MAX_1D).contains(&n) { + return None; + } + let mut d1 = Vec::new(); for _ in 0..n { let v = nums(lines.next()?)?; if v.len() != 3 { @@ -205,7 +220,7 @@ pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { } d1.push([v[0], v[1], v[2]]); } - (n >= 2).then_some((Lut { n1: n, d1, n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + Some((Lut { n1: n, d1, n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) } } diff --git a/crates/effects/src/ocio/tests.rs b/crates/effects/src/ocio/tests.rs index edbc6eff2..2df2320e6 100644 --- a/crates/effects/src/ocio/tests.rs +++ b/crates/effects/src/ocio/tests.rs @@ -179,6 +179,14 @@ fn lut_files_parse_and_interpolate() { assert!((o[0] - 0.25).abs() < 1e-6 && (o[1] - 0.4).abs() < 1e-6, "{o:?}"); let inv = x.apply(o, 2, true, ""); assert!((inv[0] - 0.5).abs() < 1e-4, "{inv:?}"); + // A .csp whose header claims a huge or fractional table is refused before anything is reserved. + let head = "CSPLUTV100\n{k}\n\n2\n0 1\n0 1\n2\n0 1\n0 1\n2\n0 1\n0 1\n\n"; + for (k, size) in [("3D", "100000 100000 100000"), ("3D", "130 130 130"), ("3D", "2.5 2.5 2.5"), ("1D", "4000000000000")] { + let text = head.replace("{k}", k) + size + "\n0 0 0\n"; + assert!(parse_file(&text, "csp").is_none(), "{k} {size}"); + } + // A short file that claims a valid size is refused once its rows run out. + assert!(parse_file(&(head.replace("{k}", "3D") + "129 129 129\n0 0 0\n"), "csp").is_none()); // CDL collection: pick by id. let ccc = r#"2 2 20 0 01 1 1 0.5 0.5 0.51"#; From bb78d55825adc879ae522c18ac57c4ce5690318d Mon Sep 17 00:00:00 2001 From: Valentin Klein Date: Wed, 4 Nov 2026 12:06:08 +0100 Subject: [PATCH 12/35] After Effects project import (.aep / .aepx); export follows (#2558) Add After Effects project import with bounded parsing, explicit loss reports, and regression coverage. Fetch the cleaned external test corpus at commit 97262276f625afdeeb61787e4cbb74b3f17af510 and verify each fixture against the pinned SHA-256 manifest. --- .gitignore | 4 + ATTRIBUTION.md | 2 + Cargo.lock | 16 + Cargo.toml | 1 + crates/aep/Cargo.toml | 27 + crates/aep/README.md | 89 ++ crates/aep/examples/aepdump.rs | 75 ++ crates/aep/examples/stubicc.rs | 359 ++++++ crates/aep/src/container.rs | 57 + crates/aep/src/container/aepx.rs | 640 ++++++++++ crates/aep/src/container/arbitrary.rs | 309 +++++ crates/aep/src/container/bytes.rs | 50 + crates/aep/src/container/camera.rs | 135 +++ crates/aep/src/container/chunk.rs | 213 ++++ crates/aep/src/container/effect_map.rs | 1038 +++++++++++++++++ crates/aep/src/container/effects.rs | 659 +++++++++++ crates/aep/src/container/error.rs | 57 + crates/aep/src/container/footage.rs | 332 ++++++ crates/aep/src/container/header.rs | 549 +++++++++ crates/aep/src/container/keys.rs | 361 ++++++ crates/aep/src/container/masks.rs | 258 ++++ crates/aep/src/container/matte.rs | 68 ++ crates/aep/src/container/read.rs | 142 +++ crates/aep/src/container/settings.rs | 224 ++++ crates/aep/src/container/shapes.rs | 273 +++++ crates/aep/src/container/style_map.rs | 368 ++++++ crates/aep/src/container/textdoc.rs | 1031 ++++++++++++++++ crates/aep/src/container/typed.rs | 539 +++++++++ crates/aep/src/container/values.rs | 263 +++++ crates/aep/src/container/write.rs | 86 ++ crates/aep/src/import.rs | 153 +++ crates/aep/src/import/build.rs | 906 ++++++++++++++ crates/aep/src/import/camera.rs | 114 ++ crates/aep/src/import/effects.rs | 552 +++++++++ crates/aep/src/import/markers.rs | 102 ++ crates/aep/src/import/masks.rs | 285 +++++ crates/aep/src/import/props.rs | 440 +++++++ crates/aep/src/import/report.rs | 241 ++++ crates/aep/src/import/settings.rs | 96 ++ crates/aep/src/import/shapes.rs | 492 ++++++++ crates/aep/src/import/styles.rs | 180 +++ crates/aep/src/import/text.rs | 453 +++++++ crates/aep/src/import/text_anim.rs | 202 ++++ crates/aep/src/import/time.rs | 112 ++ crates/aep/src/lib.rs | 24 + crates/aep/tests/fixtures/CASES.md | 390 +++++++ crates/aep/tests/fixtures/corpus.lock | 31 + crates/aep/tests/fixtures/synthetic.aep | Bin 0 -> 115806 bytes .../tests/fixtures/synthetic.aep.attribution | 7 + crates/aep/tests/fixtures/tools/3d.jsx | 44 + crates/aep/tests/fixtures/tools/README.md | 101 ++ crates/aep/tests/fixtures/tools/effects.jsx | 353 ++++++ .../aep/tests/fixtures/tools/fam-common.jsx | 37 + crates/aep/tests/fixtures/tools/fam-to-mcp.js | 78 ++ .../tests/fixtures/tools/footage-precomp.jsx | 44 + crates/aep/tests/fixtures/tools/keyframes.jsx | 107 ++ .../fixtures/tools/layers-comps-scrub-aepx.py | 25 + .../aep/tests/fixtures/tools/layers-comps.jsx | 173 +++ crates/aep/tests/fixtures/tools/markers.jsx | 21 + crates/aep/tests/fixtures/tools/masks.jsx | 75 ++ crates/aep/tests/fixtures/tools/mss-to-mcp.js | 63 + crates/aep/tests/fixtures/tools/patch-text.py | 72 ++ crates/aep/tests/fixtures/tools/scrub.py | 37 + crates/aep/tests/fixtures/tools/settings.jsx | 44 + crates/aep/tests/fixtures/tools/shapes.jsx | 96 ++ crates/aep/tests/fixtures/tools/styles.jsx | 84 ++ crates/aep/tests/fixtures/tools/text.jsx | 322 +++++ crates/aep/tests/fixtures/tools/to-mcp.js | 99 ++ crates/aep/tests/it/common/cases.rs | 381 ++++++ crates/aep/tests/it/common/mod.rs | 333 ++++++ crates/aep/tests/it/container.rs | 29 + crates/aep/tests/it/container/aepx.rs | 174 +++ crates/aep/tests/it/container/batch3_core.rs | 383 ++++++ .../aep/tests/it/container/batch3_objects.rs | 543 +++++++++ crates/aep/tests/it/container/batch5.rs | 152 +++ crates/aep/tests/it/container/cases_600.rs | 186 +++ crates/aep/tests/it/container/ease_round2.rs | 69 ++ crates/aep/tests/it/container/effects.rs | 292 +++++ .../aep/tests/it/container/effects_batch4.rs | 307 +++++ .../aep/tests/it/container/effects_batch5.rs | 151 +++ .../tests/it/container/effects_exposure.rs | 145 +++ crates/aep/tests/it/container/header.rs | 364 ++++++ crates/aep/tests/it/container/helpers.rs | 413 +++++++ crates/aep/tests/it/container/keyframes.rs | 230 ++++ .../aep/tests/it/container/layer_switches.rs | 225 ++++ crates/aep/tests/it/container/masks.rs | 203 ++++ crates/aep/tests/it/container/matte.rs | 82 ++ crates/aep/tests/it/container/odd_cases.rs | 266 +++++ crates/aep/tests/it/container/robustness.rs | 729 ++++++++++++ crates/aep/tests/it/container/roundtrip.rs | 130 +++ crates/aep/tests/it/container/shapes.rs | 128 ++ crates/aep/tests/it/container/text_fields.rs | 209 ++++ .../it/container/text_markers_footage.rs | 275 +++++ .../tests/it/container/text_style_slots.rs | 153 +++ crates/aep/tests/it/container/typed.rs | 268 +++++ crates/aep/tests/it/container/values.rs | 252 ++++ crates/aep/tests/it/import.rs | 27 + crates/aep/tests/it/import/aepx.rs | 95 ++ crates/aep/tests/it/import/blend_modes.rs | 87 ++ crates/aep/tests/it/import/cameras.rs | 92 ++ crates/aep/tests/it/import/cases.rs | 736 ++++++++++++ crates/aep/tests/it/import/combined.rs | 244 ++++ crates/aep/tests/it/import/ease_round2.rs | 80 ++ crates/aep/tests/it/import/effects.rs | 312 +++++ crates/aep/tests/it/import/effects_batch5.rs | 196 ++++ crates/aep/tests/it/import/effects_round2.rs | 139 +++ crates/aep/tests/it/import/effects_round3.rs | 323 +++++ crates/aep/tests/it/import/helpers.rs | 69 ++ crates/aep/tests/it/import/markers.rs | 96 ++ crates/aep/tests/it/import/masks.rs | 285 +++++ crates/aep/tests/it/import/mattes.rs | 105 ++ .../aep/tests/it/import/resaved_defaults.rs | 74 ++ crates/aep/tests/it/import/robustness.rs | 811 +++++++++++++ crates/aep/tests/it/import/shapes.rs | 203 ++++ crates/aep/tests/it/import/shapes_round2.rs | 114 ++ crates/aep/tests/it/import/styles.rs | 192 +++ crates/aep/tests/it/import/synthetic.rs | 297 +++++ crates/aep/tests/it/import/text.rs | 511 ++++++++ crates/aep/tests/it/import/text_anim.rs | 227 ++++ crates/aep/tests/it/import/timing.rs | 88 ++ crates/aep/tests/it/main.rs | 8 + crates/automation/src/confine.rs | 1 + crates/engine/Cargo.toml | 1 + crates/engine/src/cmd.rs | 2 + crates/engine/src/cmd/aep.rs | 101 ++ crates/engine/src/footage/doc_path.rs | 15 +- crates/engine/src/menus.rs | 1 + crates/engine/src/tests.rs | 1 + crates/engine/src/tests/aep_import.rs | 321 +++++ crates/project/src/item.rs | 5 + crates/project/src/lib.rs | 1 + crates/ui-egui/src/i18n.rs | 35 + crates/ui-egui/src/i18n/ui.rs | 58 + crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/src/menus/frontend.rs | 10 + crates/ui-egui/src/menus/invoke.rs | 5 + crates/ui-egui/src/panels.rs | 1 + crates/ui-egui/src/panels/aep_losses.rs | 189 +++ crates/ui-egui/src/panels/dialogs.rs | 2 + docs/aep-format.md | 883 ++++++++++++++ docs/aep-import.md | 393 +++++++ docs/agents.md | 2 + docs/architecture.md | 1 + docs/file-format-parity.md | 12 +- docs/testing.md | 16 +- xtask/Cargo.toml | 1 + xtask/src/aep_corpus.rs | 290 +++++ xtask/src/main.rs | 11 +- 148 files changed, 29712 insertions(+), 12 deletions(-) create mode 100644 crates/aep/Cargo.toml create mode 100644 crates/aep/README.md create mode 100644 crates/aep/examples/aepdump.rs create mode 100644 crates/aep/examples/stubicc.rs create mode 100644 crates/aep/src/container.rs create mode 100644 crates/aep/src/container/aepx.rs create mode 100644 crates/aep/src/container/arbitrary.rs create mode 100644 crates/aep/src/container/bytes.rs create mode 100644 crates/aep/src/container/camera.rs create mode 100644 crates/aep/src/container/chunk.rs create mode 100644 crates/aep/src/container/effect_map.rs create mode 100644 crates/aep/src/container/effects.rs create mode 100644 crates/aep/src/container/error.rs create mode 100644 crates/aep/src/container/footage.rs create mode 100644 crates/aep/src/container/header.rs create mode 100644 crates/aep/src/container/keys.rs create mode 100644 crates/aep/src/container/masks.rs create mode 100644 crates/aep/src/container/matte.rs create mode 100644 crates/aep/src/container/read.rs create mode 100644 crates/aep/src/container/settings.rs create mode 100644 crates/aep/src/container/shapes.rs create mode 100644 crates/aep/src/container/style_map.rs create mode 100644 crates/aep/src/container/textdoc.rs create mode 100644 crates/aep/src/container/typed.rs create mode 100644 crates/aep/src/container/values.rs create mode 100644 crates/aep/src/container/write.rs create mode 100644 crates/aep/src/import.rs create mode 100644 crates/aep/src/import/build.rs create mode 100644 crates/aep/src/import/camera.rs create mode 100644 crates/aep/src/import/effects.rs create mode 100644 crates/aep/src/import/markers.rs create mode 100644 crates/aep/src/import/masks.rs create mode 100644 crates/aep/src/import/props.rs create mode 100644 crates/aep/src/import/report.rs create mode 100644 crates/aep/src/import/settings.rs create mode 100644 crates/aep/src/import/shapes.rs create mode 100644 crates/aep/src/import/styles.rs create mode 100644 crates/aep/src/import/text.rs create mode 100644 crates/aep/src/import/text_anim.rs create mode 100644 crates/aep/src/import/time.rs create mode 100644 crates/aep/src/lib.rs create mode 100644 crates/aep/tests/fixtures/CASES.md create mode 100644 crates/aep/tests/fixtures/corpus.lock create mode 100644 crates/aep/tests/fixtures/synthetic.aep create mode 100644 crates/aep/tests/fixtures/synthetic.aep.attribution create mode 100644 crates/aep/tests/fixtures/tools/3d.jsx create mode 100644 crates/aep/tests/fixtures/tools/README.md create mode 100644 crates/aep/tests/fixtures/tools/effects.jsx create mode 100644 crates/aep/tests/fixtures/tools/fam-common.jsx create mode 100644 crates/aep/tests/fixtures/tools/fam-to-mcp.js create mode 100644 crates/aep/tests/fixtures/tools/footage-precomp.jsx create mode 100644 crates/aep/tests/fixtures/tools/keyframes.jsx create mode 100644 crates/aep/tests/fixtures/tools/layers-comps-scrub-aepx.py create mode 100644 crates/aep/tests/fixtures/tools/layers-comps.jsx create mode 100644 crates/aep/tests/fixtures/tools/markers.jsx create mode 100644 crates/aep/tests/fixtures/tools/masks.jsx create mode 100644 crates/aep/tests/fixtures/tools/mss-to-mcp.js create mode 100644 crates/aep/tests/fixtures/tools/patch-text.py create mode 100644 crates/aep/tests/fixtures/tools/scrub.py create mode 100644 crates/aep/tests/fixtures/tools/settings.jsx create mode 100644 crates/aep/tests/fixtures/tools/shapes.jsx create mode 100644 crates/aep/tests/fixtures/tools/styles.jsx create mode 100644 crates/aep/tests/fixtures/tools/text.jsx create mode 100644 crates/aep/tests/fixtures/tools/to-mcp.js create mode 100644 crates/aep/tests/it/common/cases.rs create mode 100644 crates/aep/tests/it/common/mod.rs create mode 100644 crates/aep/tests/it/container.rs create mode 100644 crates/aep/tests/it/container/aepx.rs create mode 100644 crates/aep/tests/it/container/batch3_core.rs create mode 100644 crates/aep/tests/it/container/batch3_objects.rs create mode 100644 crates/aep/tests/it/container/batch5.rs create mode 100644 crates/aep/tests/it/container/cases_600.rs create mode 100644 crates/aep/tests/it/container/ease_round2.rs create mode 100644 crates/aep/tests/it/container/effects.rs create mode 100644 crates/aep/tests/it/container/effects_batch4.rs create mode 100644 crates/aep/tests/it/container/effects_batch5.rs create mode 100644 crates/aep/tests/it/container/effects_exposure.rs create mode 100644 crates/aep/tests/it/container/header.rs create mode 100644 crates/aep/tests/it/container/helpers.rs create mode 100644 crates/aep/tests/it/container/keyframes.rs create mode 100644 crates/aep/tests/it/container/layer_switches.rs create mode 100644 crates/aep/tests/it/container/masks.rs create mode 100644 crates/aep/tests/it/container/matte.rs create mode 100644 crates/aep/tests/it/container/odd_cases.rs create mode 100644 crates/aep/tests/it/container/robustness.rs create mode 100644 crates/aep/tests/it/container/roundtrip.rs create mode 100644 crates/aep/tests/it/container/shapes.rs create mode 100644 crates/aep/tests/it/container/text_fields.rs create mode 100644 crates/aep/tests/it/container/text_markers_footage.rs create mode 100644 crates/aep/tests/it/container/text_style_slots.rs create mode 100644 crates/aep/tests/it/container/typed.rs create mode 100644 crates/aep/tests/it/container/values.rs create mode 100644 crates/aep/tests/it/import.rs create mode 100644 crates/aep/tests/it/import/aepx.rs create mode 100644 crates/aep/tests/it/import/blend_modes.rs create mode 100644 crates/aep/tests/it/import/cameras.rs create mode 100644 crates/aep/tests/it/import/cases.rs create mode 100644 crates/aep/tests/it/import/combined.rs create mode 100644 crates/aep/tests/it/import/ease_round2.rs create mode 100644 crates/aep/tests/it/import/effects.rs create mode 100644 crates/aep/tests/it/import/effects_batch5.rs create mode 100644 crates/aep/tests/it/import/effects_round2.rs create mode 100644 crates/aep/tests/it/import/effects_round3.rs create mode 100644 crates/aep/tests/it/import/helpers.rs create mode 100644 crates/aep/tests/it/import/markers.rs create mode 100644 crates/aep/tests/it/import/masks.rs create mode 100644 crates/aep/tests/it/import/mattes.rs create mode 100644 crates/aep/tests/it/import/resaved_defaults.rs create mode 100644 crates/aep/tests/it/import/robustness.rs create mode 100644 crates/aep/tests/it/import/shapes.rs create mode 100644 crates/aep/tests/it/import/shapes_round2.rs create mode 100644 crates/aep/tests/it/import/styles.rs create mode 100644 crates/aep/tests/it/import/synthetic.rs create mode 100644 crates/aep/tests/it/import/text.rs create mode 100644 crates/aep/tests/it/import/text_anim.rs create mode 100644 crates/aep/tests/it/import/timing.rs create mode 100644 crates/aep/tests/it/main.rs create mode 100644 crates/engine/src/cmd/aep.rs create mode 100644 crates/engine/src/tests/aep_import.rs create mode 100644 crates/ui-egui/src/panels/aep_losses.rs create mode 100644 docs/aep-format.md create mode 100644 docs/aep-import.md create mode 100644 xtask/src/aep_corpus.rs diff --git a/.gitignore b/.gitignore index 4a79323c8..ff6ad2397 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ target/ /log .DS_Store .claude/worktrees/ +__pycache__/ + +# After Effects test corpus fetched by `cargo xtask aep-corpus` +/corpus/ diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index c97d32ed9..93b34b9bf 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -64,6 +64,8 @@ and licence texts are in [`NOTICE`](NOTICE). | `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/cube.mtl` | EffectCraft contributors | Original work: MTL test fixture (materials of cube.obj) | MIT OR Apache-2.0 | +One small After Effects test project of `crates/aep` is in this repository: `crates/aep/tests/fixtures/synthetic.aep` (original work, made with After Effects 2026 from own data, working space None, the embedded colour profile removed; MIT OR Apache-2.0, with its `.attribution` sidecar). The other After Effects test projects of `crates/aep` are not in this repository. They are original work of an EffectCraft contributor, generated with After Effects 2026 as own test data (MIT OR Apache-2.0, each file with its `.attribution` sidecar); they are in [storytold/effectcraft-corpus](https://github.com/storytold/effectcraft-corpus) with their sidecars, fetched by `cargo xtask aep-corpus` (commit-pinned, SHA-256-verified), and the tests read them only from the folder named by `EFFECTCRAFT_AEP_FIXTURES` (opt-in). Their embedded colour profiles are replaced by stubs made with `crates/aep/examples/stubicc.rs` (a header and a description, no colour data), so no vendor profile data is in them. + ## Screenshots Screenshots of EffectCraft itself, rendered headlessly, showing only procedurally generated demo content. diff --git a/Cargo.lock b/Cargo.lock index 232fa6b18..9a9f78ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1436,6 +1436,20 @@ dependencies = [ "zbus", ] +[[package]] +name = "effectcraft-aep" +version = "0.21.0" +dependencies = [ + "effectcraft-color", + "effectcraft-effects", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-time", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "effectcraft-automation" version = "0.21.0" @@ -1507,6 +1521,7 @@ version = "0.21.0" dependencies = [ "chrono", "craft-docpath", + "effectcraft-aep", "effectcraft-color", "effectcraft-effects", "effectcraft-geom", @@ -7630,6 +7645,7 @@ dependencies = [ "chrono", "pkg-config", "serde_json", + "sha2", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index f5f84775f..fa1870e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ effectcraft-lottie = { path = "crates/lottie" } effectcraft-interchange = { path = "crates/interchange" } effectcraft-pdf = { path = "crates/pdf" } effectcraft-psd = { path = "crates/psd" } +effectcraft-aep = { path = "crates/aep" } effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } diff --git a/crates/aep/Cargo.toml b/crates/aep/Cargo.toml new file mode 100644 index 000000000..c538dd4f5 --- /dev/null +++ b/crates/aep/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "effectcraft-aep" +description = "After Effects projects: the .aep (RIFX 'Egg!') / .aepx container as a lossless chunk tree with a typed view, and its import into an EffectCraft project, which reports everything it does not carry over" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-time = { workspace = true } +effectcraft-color = { workspace = true } +effectcraft-keyframe = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-effects = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } + +# The tool's own tests run with the crate's (examples are only built by default). +[[example]] +name = "stubicc" +test = true diff --git a/crates/aep/README.md b/crates/aep/README.md new file mode 100644 index 000000000..a9b3b807d --- /dev/null +++ b/crates/aep/README.md @@ -0,0 +1,89 @@ +# effectcraft-aep + +After Effects projects (`.aep`, `.aepx`) in one crate, the way `effectcraft-lottie` handles Lottie: + +- `container`: reads and writes the container as a lossless chunk tree, with a typed, read-only + view (below; its API is re-exported at the crate root); +- `import`: maps the typed view into an EffectCraft project and reports everything it does not + import ([`docs/aep-import.md`](../../docs/aep-import.md)). + +Writing an EffectCraft project as an `.aep` (export) is not part of this crate yet. + +No file access: callers pass bytes, so the crate builds for wasm32. + +## Container + +An `.aep` is a `RIFX` container (big-endian RIFF) with form type `Egg!`: chunks of a 4-byte id, a +big-endian `u32` size and a body padded to an even length; `LIST` chunks start with a 4-byte list +type followed by chunks. `parse` builds the tree, `write` serialises it, and for every input that +parses, `write(parse(x)) == x` byte for byte (container and chunk sizes are recomputed from the +tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, are kept). + +The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded +XML tokenizer, no dependency); `parse_any` picks the reader by content, so the typed view and the +importer take either form. The mapping is in `docs/aep-format.md`, section 1. + +Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay +opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining +input, nesting and chunk count are bounded (`Limits`), nothing is allocated from a declared size, +and truncated or damaged files return `Err` (or, inside the container, an opaque body). + +The container layer decodes no chunk body. On top of it, a typed view (`Project`, `Item`, `Comp`, `Layer`, +`Property`, ...) reads the fields whose layout was verified against the values the test files were made +with: layer and composition headers (switches, exact times and time stretch, size, frame rate, work area, shutter phase, display start, renderer), +static transform values in After Effects' units with the "absent = default" rule, keyframe records +(interpolation sides, ease per component, plain and spatial templates), Time Remapping, expressions with their enable switch, effect +instances and parameter values, masks and bezier paths with tangents, shape layer groups, text documents +(paragraph and style runs, box, justification, tracking, faux styles, caps, leading), markers with +duration, strings and cue data, cameras and lights with their options, footage (stills, audio, image +sequences), solids and precomp links. Every accessor returns `Option`/an empty list on damaged data and never +panics. Which fields are verified, which are not decoded, and how it was found is in +[`docs/aep-format.md`](../../docs/aep-format.md). + +## Specifications + +Public specifications used for parts the format embeds: + +- ICC.1:2010: the profile description of a working-space profile (a v2 `desc` or a v4 `mluc` + tag), from which the working space's name is read (`container/settings.rs`). +- RFC 4648 (base64): the profile data in the project's colour settings JSON. + +## Clean room + +Everything here comes from our own observations: project files that we generated ourselves with +After Effects, one change per file, compared byte by byte. No source code or format +documentation of another `.aep` reader or writer was used, and no Adobe asset is used. The test fixtures are such generated files +(own test data): a few combined projects, one composition per case (`tests/fixtures/CASES.md`; +the generator is in `tests/fixtures/tools/`). They are not in this repository but in +[storytold/effectcraft-corpus](https://github.com/storytold/effectcraft-corpus), with their +attribution, the generation notes and the inspection of their embedded content. +`cargo xtask aep-corpus` fetches them at the commit pinned in `tests/fixtures/corpus.lock`, checks +every file against the SHA-256 listed there and only then installs them in `corpus/aep` (ignored by +git). The tests read them from the folder named by the `EFFECTCRAFT_AEP_FIXTURES` environment +variable (opt-in; set `EFFECTCRAFT_AEP_FIXTURES_REQUIRED=1` too, so that a missing project fails). The +tests reach them only through `tests/it/common/mod.rs`, which hands a test its case as a project +of its own cut from the combined file; a test whose fixture is missing (or with the variable not +set) prints `SKIPPED: fixture missing` and passes. + +`tests/fixtures/synthetic.aep` (115 KB) is the one project in the repository: made in After Effects 2026 +with the working space set to None (one composition, an image sequence of three 8x8 PNGs, a +solid, a folder, a Levels and a Glow effect with expressions). Every After Effects project embeds +the working space's ICC profile (HP sRGB IEC61966-2.1 for the default) in a JSON settings chunk; +the one `Utf8` chunk of this file that held a profile was replaced by `{}`, and nothing else was +changed. The tests in `tests/it/import/synthetic.rs` damage copies of it and need no other file. + +## Tools + +- `tests/fixtures/tools/`: the generator of the test projects (regenerates the corpus with After + Effects; its README says how). +- `examples/aepdump.rs`: prints the chunk tree of a file (ids, sizes, short data, match names): + `cargo run -p effectcraft-aep --example aepdump -- file.aep`. +- `examples/stubicc.rs`: replaces every embedded ICC profile of a saved project by a stub that keeps + only the profile's description: the JSON `colorProfileData` of the working space (any white space + around the colon, `\/` escapes, quotes as `"`) in an `.aep` chunk or the `.aepx` text, and a raw + profile of an imported image (an `.aep` chunk, or the `bdata` hex of an `.aepx`): + `cargo run -p effectcraft-aep --example stubicc -- in.aep out.aep`. A profile it cannot replace + stops it before anything is written. The importer reads the working space's name from the + description only, so the project imports the same; no vendor colour data stays in the test + project. Check the result independently, not by the tool's count (the corpus repository has + `tools/aep/inspect-embedded.py`, which scans the raw bytes, base64 and hex). diff --git a/crates/aep/examples/aepdump.rs b/crates/aep/examples/aepdump.rs new file mode 100644 index 000000000..2fe492696 --- /dev/null +++ b/crates/aep/examples/aepdump.rs @@ -0,0 +1,75 @@ +//! Developer tool: prints the chunk tree of an .aep (ids, sizes, short data), match names inline. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +use effectcraft_aep::{Body, Chunk}; + +/// Ends this developer tool with a message instead of a panic. +trait OrExit { + fn or_exit(self, what: &str) -> T; +} + +impl OrExit for Option { + fn or_exit(self, what: &str) -> T { + match self { + Some(v) => v, + None => exit(what), + } + } +} + +impl OrExit for Result { + fn or_exit(self, what: &str) -> T { + match self { + Ok(v) => v, + Err(e) => exit(&format!("{what}: {e:?}")), + } + } +} + +fn exit(msg: &str) -> ! { + eprintln!("{msg}"); + std::process::exit(1) +} + +fn show(c: &Chunk<'_>, depth: usize, max: usize, hex: usize) { + let ind = " ".repeat(depth); + let id = String::from_utf8_lossy(&c.id).to_string(); + match &c.body { + Body::Data(d) if &c.id == b"tdmn" => { + let n: String = d.iter().take_while(|b| **b != 0).map(|b| *b as char).collect(); + println!("{ind}tdmn {n}"); + } + Body::Data(d) => { + let h: String = d.iter().take(hex).map(|b| format!("{b:02x}")).collect(); + let a: String = d.iter().take(hex).map(|b| if (32..127).contains(b) { *b as char } else { '.' }).collect(); + println!("{ind}{id} [{}] {h} |{a}|", d.len()); + } + Body::Chunks(k) => { + println!("{ind}{id} (nested)"); + if depth < max { + for x in k { + show(x, depth + 1, max, hex); + } + } + } + Body::List { list_type, children } => { + println!("{ind}LIST {} ({} children)", String::from_utf8_lossy(list_type), children.len()); + if depth < max { + for x in children { + show(x, depth + 1, max, hex); + } + } + } + Body::OpaqueList { list_type, data } => println!("{ind}LIST {} opaque [{}]", String::from_utf8_lossy(list_type), data.len()), + } +} +fn main() { + let a: Vec = std::env::args().collect(); + let bytes = std::fs::read(a.get(1).or_exit("usage: see the comment at the top of this file")).or_exit("read"); + let max: usize = a.get(2).and_then(|s| s.parse().ok()).unwrap_or(4); + let hex: usize = a.get(3).and_then(|s| s.parse().ok()).unwrap_or(32); + let f = effectcraft_aep::parse(&bytes).or_exit("parse"); + for c in &f.chunks { + show(c, 0, max, hex); + } + println!("trailer {} bytes", f.trailer.len()); +} diff --git a/crates/aep/examples/stubicc.rs b/crates/aep/examples/stubicc.rs new file mode 100644 index 000000000..9438da06d --- /dev/null +++ b/crates/aep/examples/stubicc.rs @@ -0,0 +1,359 @@ +//! Developer tool: replaces every embedded ICC profile of an .aep / .aepx by a stub that keeps only the +//! profile's description. Fixture hygiene: a saved project carries the vendor's working-space profile +//! (a few KB of colour data); the importer reads only the profile's name, the `desc` tag, so the stub +//! (a 128-byte header, one `desc` tag, no colour data, made here) imports identically. +//! +//! `cargo run -p effectcraft-aep --example stubicc -- in.aep out.aep` +//! +//! It handles: +//! - the JSON value `"colorProfileData": ""` in any chunk of an .aep and anywhere in the text +//! of an .aepx (white space around the colon, `\/` escapes and quotes written as `"` allowed); +//! - a raw ICC profile stored as a chunk of its own (the profile of an imported image), in an .aep +//! as the chunk's bytes and in an .aepx as its `bdata=""` attribute. +//! +//! A profile it cannot replace (no readable description, data that does not decode) stops the tool +//! before anything is written. Check the result independently (the corpus has +//! `tools/aep/inspect-embedded.py`), not by this tool's own count. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +use effectcraft_aep::{Body, Chunk}; + +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// Ends this developer tool with a message instead of a panic. +trait OrExit { + fn or_exit(self, what: &str) -> T; +} + +impl OrExit for Option { + fn or_exit(self, what: &str) -> T { + match self { + Some(v) => v, + None => exit(what), + } + } +} + +impl OrExit for Result { + fn or_exit(self, what: &str) -> T { + match self { + Ok(v) => v, + Err(e) => exit(&format!("{what}: {e:?}")), + } + } +} + +fn exit(msg: &str) -> ! { + eprintln!("{msg}"); + std::process::exit(1) +} + +fn b64_decode(text: &[u8]) -> Option> { + let (mut out, mut acc, mut bits) = (Vec::new(), 0u32, 0u32); + for &c in text.iter().filter(|c| **c != b'=') { + let v = u32::try_from(B64.iter().position(|b| *b == c)?).ok()?; + acc = (acc << 6 | v) & 0x00ff_ffff; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push(((acc >> bits) & 0xff) as u8); + } + } + Some(out) +} + +fn b64_encode(data: &[u8]) -> Vec { + let mut out = Vec::new(); + for ch in data.chunks(3) { + let at = |i: usize| u32::from(ch.get(i).copied().unwrap_or(0)); + let n = (at(0) << 16) | (at(1) << 8) | at(2); + for i in 0..4usize { + if i <= ch.len() { + out.push(B64.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'A')); + } else { + out.push(b'='); + } + } + } + out +} + +fn be32(d: &[u8], at: usize) -> Option { + let b: [u8; 4] = d.get(at..at.checked_add(4)?)?.try_into().ok()?; + usize::try_from(u32::from_be_bytes(b)).ok() +} + +/// The profile description (ICC.1 `desc`: v2 ASCII or v4 `mluc`, first record). +fn description(icc: &[u8]) -> Option { + let count = be32(icc, 128)?; + let tag = (0..count.min(1024)).find_map(|i| { + let at = 132usize.checked_add(i.checked_mul(12)?)?; + (icc.get(at..at.checked_add(4)?)? == b"desc").then_some(at) + })?; + let off = be32(icc, tag.checked_add(4)?)?; + let body = icc.get(off..off.checked_add(be32(icc, tag.checked_add(8)?)?)?)?; + let text = match body.get(..4)? { + b"desc" => { + let n = be32(body, 8)?; + String::from_utf8_lossy(body.get(12..12usize.checked_add(n)?)?.split(|b| *b == 0).next()?).into_owned() + } + b"mluc" => { + let (n, at) = (be32(body, 20)?, be32(body, 24)?); + let units: Vec = body.get(at..at.checked_add(n)?)?.chunks(2).filter_map(|p| Some(u16::from_be_bytes([*p.first()?, *p.get(1)?]))).collect(); + String::from_utf16_lossy(&units) + } + _ => return None, + }; + Some(text.trim_end_matches('\0').trim().to_owned()) +} + +/// A profile with a header, one `desc` tag and no colour data. +fn stub(desc: &str) -> Option> { + let mut body = b"desc\0\0\0\0".to_vec(); + body.extend_from_slice(&u32::try_from(desc.len().checked_add(1)?).ok()?.to_be_bytes()); + body.extend_from_slice(desc.as_bytes()); + body.push(0); + body.extend_from_slice(&[0; 4 + 4 + 2 + 1 + 67]); + let mut p = vec![0u8; 128]; + for (at, v) in [(8, &0x0240_0000u32.to_be_bytes()[..]), (12, b"mntr"), (16, b"RGB "), (20, b"XYZ "), (36, b"acsp")] { + p.get_mut(at..at + v.len())?.copy_from_slice(v); + } + p.extend_from_slice(&1u32.to_be_bytes()); + p.extend_from_slice(b"desc"); + p.extend_from_slice(&144u32.to_be_bytes()); + p.extend_from_slice(&u32::try_from(body.len()).ok()?.to_be_bytes()); + p.extend_from_slice(&body); + let size = u32::try_from(p.len()).ok()?.to_be_bytes(); + p.get_mut(..4)?.copy_from_slice(&size); + Some(p) +} + +/// The JSON key, without its quotes. +const KEY: &[u8] = b"colorProfileData"; +/// How a JSON quote is written: as is (a `Utf8` chunk, the `` text of an .aepx) and as an +/// XML entity (JSON inside an attribute of an .aepx). +const QUOTES: [&[u8]; 2] = [b"\"", b"""]; + +/// What the tool did, counted. +#[derive(Default)] +struct Tally { + stubbed: usize, + /// Values too short to be a profile (no ICC header), kept as they are. + kept_short: usize, + /// Profiles that could not be replaced (no readable description, undecodable data): the tool + /// fails, so no profile is ever left behind silently. + failed: Vec, +} + +fn is_profile(d: &[u8]) -> bool { + d.len() > 132 && d.get(36..40) == Some(b"acsp") +} + +/// The stub for the bytes `d` of a value, or `None` (counted) when it holds no profile or the +/// profile has no readable description. +fn stub_of(d: &[u8], at: usize, tally: &mut Tally) -> Option> { + if !is_profile(d) { + tally.kept_short += 1; + return None; + } + let s = description(d).and_then(|t| stub(&t)); + if s.is_none() { + tally.failed.push(format!("profile at byte {at}: no readable description")); + } + s +} + +/// At the start of `t`: the key in quotes, optional white space, a colon, optional white space and +/// the opening quote of the value. Returns the length up to and including that quote, and the quote. +fn value_start(t: &[u8]) -> Option<(usize, &'static [u8])> { + let ws = |mut i: usize| { + while t.get(i).is_some_and(u8::is_ascii_whitespace) { + i += 1; + } + i + }; + QUOTES.into_iter().find_map(|q| { + let key_end = q.len().checked_mul(2)?.checked_add(KEY.len())?; + let quoted = t.get(..key_end)?; + if !(quoted.starts_with(q) && quoted.get(q.len()..q.len() + KEY.len())? == KEY && quoted.ends_with(q)) { + return None; + } + let colon = ws(key_end); + if t.get(colon) != Some(&b':') { + return None; + } + let open = ws(colon + 1); + t.get(open..)?.starts_with(q).then_some((open + q.len(), q)) + }) +} + +fn hex_decode(h: &[u8]) -> Option> { + if !h.len().is_multiple_of(2) { + return None; + } + h.chunks(2).map(|p| u8::from_str_radix(std::str::from_utf8(p).ok()?, 16).ok()).collect() +} + +fn hex_encode(d: &[u8], upper: bool) -> Vec { + d.iter().flat_map(|b| if upper { format!("{b:02X}") } else { format!("{b:02x}") }.into_bytes()).collect() +} + +/// Replaces, in `text`, every `colorProfileData` JSON value (base64; `\/` escapes, white space +/// around the colon and quotes written as `"` allowed) and every .aepx `bdata=""` +/// attribute that holds a raw profile. +fn stub_text(text: &[u8], tally: &mut Tally) -> Vec { + const BDATA: &[u8] = b"bdata=\""; + let (mut out, mut i) = (Vec::with_capacity(text.len()), 0); + while let Some(&b) = text.get(i) { + let rest = text.get(i..).unwrap_or_default(); + if let Some((open, q)) = value_start(rest) + && let Some(len) = rest.get(open..).and_then(|v| v.windows(q.len()).position(|w| w == q)) + { + let raw = rest.get(open..open + len).unwrap_or_default(); + let unescaped: Vec = raw.iter().copied().filter(|c| *c != b'\\').collect(); + let replaced = match b64_decode(&unescaped) { + Some(d) => stub_of(&d, i, tally).map(|s| b64_encode(&s)), + None => { + tally.failed.push(format!("colorProfileData at byte {i}: not base64")); + None + } + }; + out.extend_from_slice(rest.get(..open).unwrap_or_default()); + match replaced { + Some(enc) => { + out.extend_from_slice(&enc); + tally.stubbed += 1; + } + None => out.extend_from_slice(raw), + } + i += open + len; + continue; + } + if rest.starts_with(BDATA) + && let Some(len) = rest.get(BDATA.len()..).and_then(|v| v.iter().position(|c| *c == b'"')) + { + let hex = rest.get(BDATA.len()..BDATA.len() + len).unwrap_or_default(); + let replaced = hex_decode(hex).filter(|d| is_profile(d)).and_then(|d| stub_of(&d, i, tally)); + out.extend_from_slice(BDATA); + match replaced { + Some(s) => { + out.extend_from_slice(&hex_encode(&s, hex.iter().any(u8::is_ascii_uppercase))); + tally.stubbed += 1; + } + None => out.extend_from_slice(hex), + } + i += BDATA.len() + len; + continue; + } + out.push(b); + i += 1; + } + out +} + +fn walk(chunks: &mut [Chunk<'_>], tally: &mut Tally) { + for c in chunks.iter_mut() { + // Only a data chunk is rewritten as one: an opaque `LIST` also has bytes, but replacing it by + // a data chunk would drop its list type. + let data = if let Body::Data(d) = &c.body { Some(d.to_vec()) } else { None }; + match data { + Some(d) if d.windows(KEY.len()).any(|w| w == KEY) => { + let new = stub_text(&d, tally); + *c = Chunk::data(c.id, new); + } + Some(d) if is_profile(&d) => { + if let Some(s) = stub_of(&d, 0, tally) { + *c = Chunk::data(c.id, s); + tally.stubbed += 1; + } + } + _ => { + if let Body::List { children, .. } | Body::Chunks(children) = &mut c.body { + walk(children, tally); + } + } + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let (Some(input), Some(output)) = (args.get(1), args.get(2)) else { exit("usage: stubicc ") }; + let bytes = std::fs::read(input).or_exit("read the input"); + let mut tally = Tally::default(); + let out = if input.ends_with(".aepx") { + stub_text(&bytes, &mut tally) + } else { + let mut file = effectcraft_aep::parse(&bytes).or_exit("parse the input"); + walk(&mut file.chunks, &mut tally); + effectcraft_aep::write(&file).or_exit("write the output") + }; + if !tally.failed.is_empty() { + exit(&format!("nothing written: {} profile(s) could not be replaced:\n{}", tally.failed.len(), tally.failed.join("\n"))); + } + std::fs::write(output, out).or_exit("write the output"); + println!("stubbed {} profile(s); kept {} short value(s) that hold no profile", tally.stubbed, tally.kept_short); +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn profile(desc: &str) -> Vec { + // A profile with colour data after the description (like a real one): the stub of `desc` + // plus 3000 bytes, its size field updated. + let mut p = stub(desc).unwrap(); + p.extend(std::iter::repeat_n(0x5a, 3000)); + let size = u32::try_from(p.len()).unwrap().to_be_bytes(); + p[..4].copy_from_slice(&size); + p + } + + fn stubbed(text: &[u8]) -> (Vec, Tally) { + let mut tally = Tally::default(); + (stub_text(text, &mut tally), tally) + } + + #[test] + fn json_values_in_every_spelling_are_replaced() { + let b64 = String::from_utf8(b64_encode(&profile("sRGB IEC61966-2.1"))).unwrap(); + let want = String::from_utf8(b64_encode(&stub("sRGB IEC61966-2.1").unwrap())).unwrap(); + for (before, after) in [ + (format!("{{\"colorProfileData\":\"{b64}\"}}"), format!("{{\"colorProfileData\":\"{want}\"}}")), + (format!("{{\"colorProfileData\" : \n\t\"{b64}\"}}"), format!("{{\"colorProfileData\" : \n\t\"{want}\"}}")), + (format!("{{\"colorProfileData\":\"{}\"}}", b64.replace('/', "\\/")), format!("{{\"colorProfileData\":\"{want}\"}}")), + (format!("{{"colorProfileData": "{b64}"}}"), format!("{{"colorProfileData": "{want}"}}")), + ] { + let (out, tally) = stubbed(before.as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), after); + assert_eq!((tally.stubbed, tally.failed.len()), (1, 0)); + } + } + + #[test] + fn a_raw_profile_in_aepx_hex_is_replaced_and_other_hex_kept() { + let hex = String::from_utf8(hex_encode(&profile("Rec.709 Gamma 2.4"), false)).unwrap(); + let want = String::from_utf8(hex_encode(&stub("Rec.709 Gamma 2.4").unwrap(), false)).unwrap(); + let text = format!("", "ab".repeat(200)); + let (out, tally) = stubbed(text.as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), format!("", "ab".repeat(200))); + assert_eq!((tally.stubbed, tally.failed.len()), (1, 0)); + // Upper-case hex stays upper case. + let (out, _) = stubbed(format!("bdata=\"{}\"", hex.to_uppercase()).as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), format!("bdata=\"{}\"", want.to_uppercase())); + } + + #[test] + fn a_profile_that_cannot_be_replaced_is_a_failure_not_a_silent_copy() { + // An ICC header whose tag table has no `desc`. + let mut p = profile("x"); + p[132..136].copy_from_slice(b"wtpt"); + let (_, tally) = stubbed(format!("\"colorProfileData\":\"{}\"", String::from_utf8(b64_encode(&p)).unwrap()).as_bytes()); + assert_eq!((tally.stubbed, tally.failed.len()), (0, 1)); + let (_, tally) = stubbed(b"\"colorProfileData\":\"not*base64\""); + assert_eq!(tally.failed.len(), 1); + // A short value holds no profile: kept and counted, not a failure. + let (out, tally) = stubbed(b"\"colorProfileData\":\"AAAA\""); + assert_eq!((out.as_slice(), tally.kept_short, tally.failed.len()), (&b"\"colorProfileData\":\"AAAA\""[..], 1, 0)); + } +} diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs new file mode 100644 index 000000000..4aa0f393a --- /dev/null +++ b/crates/aep/src/container.rs @@ -0,0 +1,57 @@ +//! After Effects project (`.aep`) container reader and writer. +//! +//! An `.aep` is a `RIFX` container (big-endian RIFF) with form type `Egg!`. This crate reads it +//! into a lossless chunk tree ([`AepFile`]) and writes it back byte for byte. The XML form of a +//! project (`.aepx`) is read into the same tree by [`read_aepx`]; [`parse_any`] picks the reader +//! by content. See +//! `docs/aep-format.md` for the observed format and how it was established (clean room: only +//! files we generated ourselves, one change per file). +//! +//! [`Project::new`] opens a typed, read-only view over the tree: items, compositions +//! ([`CompHeader`]), layers ([`LayerHeader`], [`Transform`], effects, masks, text documents, +//! markers), properties, keyframes and footage. Only fields whose layout was verified against the +//! values the test files were made with are decoded; every accessor returns `None` (or an empty +//! list) on damaged data instead of failing. + +mod aepx; +mod arbitrary; +mod bytes; +mod camera; +mod chunk; +pub mod effect_map; +mod effects; +mod error; +mod footage; +mod header; +mod keys; +mod masks; +mod matte; +mod read; +mod settings; +mod shapes; +pub mod style_map; +mod textdoc; +mod typed; +mod values; +mod write; + +pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; +pub use arbitrary::{CURVE_IDENTITY, CurvesData, HueSaturationData, LevelsData, MeshData, curve_lut}; +pub use camera::{CameraPosition, DEFAULT_CAMERA_ZOOM}; +pub use chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue}; +pub use error::AepError; +pub use footage::{Footage, SequenceInfo}; +pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; +pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; +pub use matte::{MatteMode, TrackMatte}; +pub use read::{Limits, parse, parse_with_limits}; +pub use settings::{ProjectSettings, TimeDisplay, WorkingSpace}; +pub use shapes::{GradientType, PolystarType, ShapeDetail, ShapeGroup, ShapeItem, Stroke, StrokeCap, StrokeJoin}; +pub use textdoc::{ + AutoKern, Caps, CueType, FontBaseline, Justification, KernRun, Marker, ParagraphRun, StyleRun, TextBox, TextDocument, TextValue, parse_text_dict, +}; +pub use typed::{Comp, Item, ItemKind, KeyLayout, KeyList, Keyframe, Layer, ListHeader, Project, PropEntry, PropGroup, PropNode, Property, TimeBase}; +pub use values::{Orientation, OrientationKey, TimeRemap, TimeRemapKey, Tracked, Transform, ValueSource}; +pub use write::{write, write_to}; diff --git a/crates/aep/src/container/aepx.rs b/crates/aep/src/container/aepx.rs new file mode 100644 index 000000000..a45cbae1d --- /dev/null +++ b/crates/aep/src/container/aepx.rs @@ -0,0 +1,640 @@ +//! Reading the XML form of a project (`.aepx`) into the same chunk tree as an `.aep`. +//! +//! An `.aepx` is the `RIFX` chunk tree written as XML: one element per chunk, named by the chunk +//! id (or the list type of a `LIST`), with the body as hex in a `bdata` attribute. A few chunks +//! are written as text instead (`Utf8` as ``, the numbers `numS` and `ppSn`, the footage +//! alias `alas` as ``), and the XMP packet that follows the container of an `.aep` +//! is the element ``. The mapping, with the cases it was found on, is in +//! `docs/aep-format.md` (section 1.1). +//! +//! The reader is a small XML tokenizer for exactly that subset: elements, attributes, text, the +//! five predefined entities and character references, CDATA, comments and processing +//! instructions. A document type declaration is refused (no entity expansion). It runs without +//! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything +//! it allocates is bounded by the size of the input. + +use std::borrow::Cow; + +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::error::AepError; +use crate::container::read::{Limits, parse}; + +/// The root element of an `.aepx`. +const ROOT: &[u8] = b"AfterEffectsProject"; +/// The element that holds the XMP packet (the trailer of an `.aep`). +const XMP: &[u8] = b"ProjectXMPMetadata"; +/// The element of a `Utf8` chunk. +const STRING: &[u8] = b"string"; +/// The element of an `alas` chunk (footage alias), written as attributes instead of JSON. +const FILE_REFERENCE: &[u8] = b"fileReference"; + +/// List types whose body is not a chunk sequence: the `.aepx` writes them like data chunks (with +/// `bdata`), the `.aep` as an opaque `LIST`. +const OPAQUE_LIST_TYPES: [[u8; 4]; 1] = [*b"btdk"]; + +/// True when `data` looks like an `.aepx`: optional UTF-8 byte order mark and white space, then +/// an XML declaration or the `AfterEffectsProject` root element. +pub fn is_aepx(data: &[u8]) -> bool { + let rest = data.strip_prefix(&[0xef, 0xbb, 0xbf][..]).unwrap_or(data); + let start = rest.iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(rest.len()); + let rest = rest.get(start..).unwrap_or(&[]); + rest.starts_with(b" Result, AepError> { + if is_aepx(data) { read_aepx(data) } else { parse(data) } +} + +/// Reads an `.aepx` with the default [`Limits`] into the chunk tree an `.aep` of the same project +/// gives. +/// +/// # Errors +/// [`AepError::Xml`] for input that is not well-formed XML of the `.aepx` subset (bad UTF-8, an +/// unknown entity, an unclosed or mismatched tag, invalid or odd-length hex, a number that does +/// not parse, a document type declaration, a root element other than `AfterEffectsProject`); +/// [`AepError::TooDeep`] / [`AepError::TooManyChunks`] beyond the limits. +pub fn read_aepx(data: &[u8]) -> Result, AepError> { + read_aepx_with_limits(data, Limits::default()) +} + +/// Like [`read_aepx`] with explicit [`Limits`]: `max_depth` bounds the element nesting (the root +/// element's children are depth 1), `max_chunks` the number of elements. +/// +/// # Errors +/// See [`read_aepx`]. +pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +fn xml(offset: usize, what: &'static str) -> AepError { + AepError::Xml { offset, what } +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk id of an element name: 1 to 4 printable ASCII bytes, padded with spaces (`Pin` is +/// the list type `Pin `). +fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { + if name.is_empty() || name.len() > 4 || !name.iter().all(|b| b.is_ascii_graphic()) { + return None; + } + let mut id = [b' '; 4]; + for (d, s) in id.iter_mut().zip(name) { + *d = *s; + } + Some(id) +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} + +/// Appends `raw` with the predefined entities and character references decoded. +fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { + let mut i = 0; + while let Some(rest) = raw.get(i..) { + let Some(amp) = rest.iter().position(|b| *b == b'&') else { + out.extend_from_slice(rest); + break; + }; + out.extend_from_slice(rest.get(..amp).unwrap_or(&[])); + let at = offset.saturating_add(i).saturating_add(amp); + let after = rest.get(amp.saturating_add(1)..).unwrap_or(&[]); + let semi = after.iter().take(12).position(|b| *b == b';').ok_or(xml(at, "an entity is not terminated"))?; + let name = after.get(..semi).unwrap_or(&[]); + let ch = match name { + b"amp" => '&', + b"lt" => '<', + b"gt" => '>', + b"quot" => '"', + b"apos" => '\'', + _ => { + let code = if let Some(hex) = name.strip_prefix(b"#x").or_else(|| name.strip_prefix(b"#X")) { + std::str::from_utf8(hex).ok().and_then(|h| u32::from_str_radix(h, 16).ok()) + } else if let Some(dec) = name.strip_prefix(b"#") { + std::str::from_utf8(dec).ok().and_then(|d| d.parse::().ok()) + } else { + return Err(xml(at, "an unknown entity")); + }; + code.filter(|c| *c != 0).and_then(char::from_u32).ok_or(xml(at, "an invalid character reference"))? + } + }; + let mut buf = [0u8; 4]; + out.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); + i = i.saturating_add(amp).saturating_add(semi).saturating_add(2); + } + Ok(()) +} + +/// Decodes a `bdata` value (hex digits, two per byte). +fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) +} + +/// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document +/// order (alphabetical in every sample, as in the `.aep`): the two `ascendcount_*` values as +/// numbers, `platform` `Win` as `1`, `target_is_folder` `0` / `1` as `false` / `true`, everything +/// else as a string. +fn alias_json(attrs: &[(String, String)]) -> String { + let mut out = String::from("{"); + for (i, (key, value)) in attrs.iter().enumerate() { + if i > 0 { + out.push(','); + } + json_str(&mut out, key); + out.push(':'); + let number = value.parse::().ok(); + match (key.as_str(), value.as_str()) { + ("ascendcount_base" | "ascendcount_target", _) if number.is_some() => out.push_str(value), + ("platform", "Win") => out.push('1'), + ("platform", _) if number.is_some() => out.push_str(value), + ("target_is_folder", "0" | "false") => out.push_str("false"), + ("target_is_folder", "1" | "true") => out.push_str("true"), + _ => json_str(&mut out, value), + } + } + out.push('}'); + out +} + +fn json_str(out: &mut String, s: &str) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if u32::from(c) < 0x20 => out.push_str(&format!("\\u{:04x}", u32::from(c))), + c => out.push(c), + } + } + out.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn doc(body: &str) -> String { + format!("\n{body}\n") + } + + #[test] + fn maps_elements_to_chunks() { + let x = doc(concat!( + "\n", + "A & BA-3", + "150.000000000x", + "]]>" + )); + let f = read_aepx(x.as_bytes()).unwrap(); + assert_eq!(f.trailer.as_ref(), b""); + let ids: Vec<_> = f.chunks.iter().map(|c| (c.id, c.list_type())).collect(); + assert_eq!( + ids, + vec![ + (*b"head", None), + (LIST, Some(*b"Pin ")), + (LIST, Some(*b"Pefl")), + (*b"tdsn", None), + (*b"numS", None), + (*b"ppSn", None), + (LIST, Some(*b"btdk")) + ] + ); + assert_eq!(f.chunks[1].children()[0].bytes(), Some(&[1u8][..])); + assert_eq!(f.chunks[1].children()[0].pad, Some(0)); + assert_eq!(f.chunks[3].children()[0].bytes(), Some(&b"A & BA"[..])); + assert_eq!(f.chunks[4].bytes(), Some(&(-3i32).to_be_bytes()[..])); + assert_eq!(f.chunks[5].bytes(), Some(&150f64.to_be_bytes()[..])); + assert_eq!(f.chunks[6].bytes(), Some(&b"ABC"[..])); + // The tree writes as a valid `.aep` that parses back to itself. + let bin = crate::container::write(&f).unwrap(); + assert_eq!(parse(&bin).unwrap(), f); + } + + #[test] + fn file_reference_becomes_alias_json() { + let x = doc(concat!( + "" + )); + let f = read_aepx(x.as_bytes()).unwrap(); + let alas = f.chunks[0].children()[0].bytes().unwrap(); + assert_eq!( + std::str::from_utf8(alas).unwrap(), + r#"{"ascendcount_base":1,"ascendcount_target":1,"fullpath":"C:\\a\\\"b.png","platform":1,"server_name":"N","server_volume_name":"V","target_is_folder":false}"# + ); + } + + #[test] + fn nums_takes_signed_and_unsigned_32_bit_values() { + for (text, bytes) in [("-2147483648", [0x80, 0, 0, 0]), ("4294967295", [0xff; 4]), (" 7 ", [0, 0, 0, 7])] { + let f = read_aepx(doc(&format!("{text}")).as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&bytes[..]), "{text}"); + } + for text in ["-2147483649", "4294967296", "1.5"] { + assert!(read_aepx(doc(&format!("{text}")).as_bytes()).is_err(), "{text}"); + } + } + + #[test] + fn detection() { + assert!(is_aepx(b"\xef\xbb\xbf ")); + assert!(is_aepx(b"")); + assert!(!is_aepx(b"RIFX\0\0\0\x04Egg!")); + assert!(parse_any(b"RIFX\0\0\0\x04Egg!").is_ok()); + assert!(parse_any(doc("").as_bytes()).is_ok()); + } + + #[test] + fn bad_input_is_an_error() { + let bad = [ + doc(""), + doc(""), + doc(" "), + doc("�"), + doc("&"), + doc(""), + doc(""), + doc("x"), + doc(""), + doc(""), + doc(""), + "]>".into(), + "".into(), + "".into(), + "".into(), + "junk".into(), + "