diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..0394589c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,6 +1327,7 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "exr", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", @@ -1335,6 +1336,7 @@ dependencies = [ "image", "log", "rayon", + "serde_json", "thiserror 2.0.21", ] @@ -1768,6 +1770,7 @@ dependencies = [ "miniz_oxide 0.8.9", "num-complex", "pulp", + "rayon-core", "smallvec", "zune-inflate", ] diff --git a/Cargo.toml b/Cargo.toml index a1941cf45..61dd6d2e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ kurbo = { version = "0.13", features = ["serde"] } tiny-skia = { version = "0.12", default-features = false, features = ["std", "simd"] } skrifa = { version = "0.44", default-features = false, features = ["std"] } harfrust = { version = "0.12", default-features = false, features = ["std"] } +exr = "1.74" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs new file mode 100644 index 000000000..5fd4d715e --- /dev/null +++ b/crates/effects/src/channel3d.rs @@ -0,0 +1,681 @@ +//! 3D Channel effects: 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, +//! Fog 3D, ID Matte and IDentifier. +//! +//! They read the layer's auxiliary channels ([`AuxChannels`], via +//! [`crate::EffectHost::aux`]): multi-layer OpenEXR footage (depth, object/material IDs, +//! Cryptomatte layers, any named channel) or the compositor's depth / layer-ID / Cryptomatte +//! pass for a precomp of a 3D comp. Without auxiliary channels they pass the layer through +//! (EXtractoR still maps the layer's own R, G, B and A). +//! +//! Aux pixels are looked up nearest-neighbour (IDs must not be interpolated) at each buffer +//! pixel's layer position, so render scale and layer padding are respected. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_raster::{AuxChannels, Image, Px, gaussian_blur}; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, hash1, premul, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } +} + +fn aux(ctx: &EffectCtx) -> Option> { + ctx.env.host?.aux() +} + +/// Aux index for each buffer pixel (None outside the aux image). +fn index_map(b: &Buf, a: &AuxChannels) -> Vec> { + let (w, h) = (b.img.width as usize, b.img.height as usize); + let inv = 1.0 / b.scale.max(1e-9); + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + a.index_at((x - b.offset[0]) * inv, (y - b.offset[1]) * inv) + }) + .collect() +} + +/// A channel sampled into the buffer grid (`fill` outside). +fn plane_of(b: &Buf, idx: &[Option], ch: &[f32], fill: f32) -> Plane { + Plane { w: b.img.width as usize, h: b.img.height as usize, data: idx.par_iter().map(|i| i.map_or(fill, |i| ch[i])).collect() } +} + +/// Distinct colour for an ID (hash → hue). +fn id_color(id: f32) -> [f32; 3] { + let h = hash1(id.to_bits(), 0x1d, 0x5eed); + let (r, g, b) = effectcraft_color::hsl_to_rgb(h, 0.75, 0.5); + if id == 0.0 { [0.0; 3] } else { [r, g, b] } +} + +// ---------------------------------------------------------------- 3D Channel Extract + +pub const EXTRACT_CHANNELS: [&str; 8] = ["Z-Depth", "Object ID", "Texture UV", "Surface Normals", "Coverage", "Background RGB", "Unclamped RGB", "Material ID"]; + +fn channel_extract(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let idx = index_map(&b, &a); + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let lv = |v: f32| ((v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum()).clamp(0.0, 1.0); + let lv = |v: f32| if wp >= bp { lv(v) } else { 1.0 - ((v - wp) / (bp - wp).max(1e-9)).clamp(0.0, 1.0) }; + let get3 = |names: [&[&str]; 3]| -> Option<[&[f32]; 3]> { Some([a.find(names[0])?, a.find(names[1])?, a.find(names[2])?]) }; + let img: Option = match ctx.params.e("channel") { + 0 => a.depth().map(|z| map_grey(&b, &idx, |i| lv(z[i]))), + 1 => a.object_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 7 => a.material_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 4 => a.coverage().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 2 => get3([&["UV.U", "U", "uv.x"], &["UV.V", "V", "uv.y"], &["UV.W", "W", "uv.z"]]) + .or_else(|| Some([a.find(&["UV.U", "U", "uv.x"])?, a.find(&["UV.V", "V", "uv.y"])?, a.find(&["UV.V", "V", "uv.y"])?])) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), 0.0])), + 3 => get3([&["N.X", "Normal.X", "normals.x"], &["N.Y", "Normal.Y", "normals.y"], &["N.Z", "Normal.Z", "normals.z"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i] * 0.5 + 0.5), lv(c[1][i] * 0.5 + 0.5), lv(c[2][i] * 0.5 + 0.5)])), + 5 => get3([&["BG.R", "Background.R"], &["BG.G", "Background.G"], &["BG.B", "Background.B"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), lv(c[2][i])])), + _ => get3([&["R"], &["G"], &["B"]]).map(|c| map_rgb(&b, &idx, |i| [c[0][i], c[1][i], c[2][i]])), + }; + if let Some(i) = img { + b.img = i; + } + b +} + +fn map_grey(b: &Buf, idx: &[Option], f: impl Fn(usize) -> f32 + Sync) -> Image { + map_rgb(b, idx, |i| { + let v = f(i); + [v, v, v] + }) +} + +fn map_rgb(b: &Buf, idx: &[Option], f: impl Fn(usize) -> [f32; 3] + Sync) -> Image { + let mut img = Image::new(b.img.width, b.img.height); + img.data.par_iter_mut().zip(idx.par_iter()).for_each(|(p, i)| { + if let Some(i) = i { + let c = f(*i); + *p = [c[0], c[1], c[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- Depth Matte + +fn depth_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let idx = index_map(&b, &a); + let depth = ctx.params.f("depth") as f32; + let feather = ctx.params.f("feather").max(0.0) as f32; + let invert = ctx.params.b("invert"); + // Keeps what lies at or beyond Depth (Invert keeps what is in front), with a soft ramp of + // `feather` depth units centred on the threshold. + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().zip(zp.data.par_iter()).for_each(|(px, &zv)| { + let k = if feather > 0.0 { + ((zv - depth) / feather + 0.5).clamp(0.0, 1.0) + } else if zv >= depth { + 1.0 + } else { + 0.0 + }; + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +// ---------------------------------------------------------------- Depth of Field + +/// Blur radius (pixels) for depth `z`. +pub fn dof_radius(z: f32, focal: f32, thickness: f32, max_r: f32, bias: f32) -> f32 { + let d = ((z - focal).abs() - thickness * 0.5).max(0.0); + let range = focal.abs().max(1.0); + let n = (d / range).min(1.0); + // Bias 50 = linear; higher bias brings blur in sooner. + let gamma = 2f32.powf((50.0 - bias) / 25.0); + max_r * n.powf(gamma) +} + +fn depth_of_field(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let max_r = (ctx.params.f("maximumRadius").max(0.0) * b.scale) as f32; + if max_r <= 0.0 { + return b; + } + let focal = ctx.params.f("focalPlane") as f32; + let thick = ctx.params.f("focalPlaneThickness").max(0.0) as f32; + let bias = ctx.params.f("focalBias") as f32; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + // A stack of blurs at a few radii, interpolated per pixel by its circle of confusion. + const LEVELS: usize = 5; + let stack: Vec = (0..LEVELS) + .into_par_iter() + .map(|k| { + let r = max_r * k as f32 / (LEVELS - 1) as f32; + if r < 0.3 { b.img.clone() } else { gaussian_blur(&b.img, (r / 2.0) as f64, (r / 2.0) as f64, true) } + }) + .collect(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let r = dof_radius(zp.data[i], focal, thick, max_r, bias); + let f = (r / max_r * (LEVELS - 1) as f32).clamp(0.0, (LEVELS - 1) as f32); + let k0 = (f.floor() as usize).min(LEVELS - 2); + let t = f - k0 as f32; + let (a0, a1) = (stack[k0].data[i], stack[k0 + 1].data[i]); + *px = [0, 1, 2, 3].map(|c| a0[c] + (a1[c] - a0[c]) * t); + }); + b +} + +// ---------------------------------------------------------------- Fog 3D + +fn fog_3d(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let pr = ctx.params; + let fc = pr.color("fogColor"); + let (start, end) = (pr.f("fogStartDepth") as f32, pr.f("fogEndDepth") as f32); + let opacity = (pr.f("fogOpacity") / 100.0) as f32; + let density = (pr.f("scatteringDensity") / 100.0) as f32; + let foggy_bg = pr.b("foggyBackground"); + let contrib = (pr.f("layerContribution") / 100.0) as f32; + let grad = if contrib > 0.0 { ctx.layer_param("gradientLayer", true).map(|o| crate::util::fit_layer(ctx, &b, &o, true)) } else { None }; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let zv = zp.data[i]; + let bg = zv >= BACKGROUND_DEPTH * 0.5; + let mut f = if bg { + if foggy_bg { 1.0 } else { 0.0 } + } else { + let t = ((zv - start) / (end - start).abs().max(1e-6)).clamp(0.0, 1.0); + // Denser scattering thickens the fog sooner. + 1.0 - (1.0 - t).powf(1.0 + density * 3.0) + }; + if let Some(g) = &grad { + let (gc, ga) = unpremul(g.data[i]); + let l = effectcraft_color::luminance(gc[0], gc[1], gc[2]) * ga; + f *= 1.0 - contrib + contrib * l; + } + let f = (f * opacity).clamp(0.0, 1.0); + let (c, al) = unpremul(*px); + let al2 = if bg && foggy_bg { al + (1.0 - al) * f } else { al }; + let out = [0, 1, 2].map(|k| c[k] + (fc[k] - c[k]) * f); + *px = premul(out, al2); + }); + b +} + +// ---------------------------------------------------------------- ID Matte / IDentifier + +fn id_channel(a: &AuxChannels, which: u32) -> Option<&[f32]> { + if which == 1 { a.material_id() } else { a.object_id() } +} + +fn id_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("auxChannel")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("idSelection").round() as f32; + let cov = if ctx.params.b("useCoverage") { a.coverage() } else { None }; + let mut m = Plane { + w: b.img.width as usize, + h: b.img.height as usize, + data: idx.par_iter().map(|i| i.map_or(0.0, |i| if ids[i].round() == sel { cov.map_or(1.0, |c| c[i].clamp(0.0, 1.0)) } else { 0.0 })).collect(), + }; + let feather = ctx.params.f("feather").max(0.0) * b.scale; + if feather > 0.0 { + m = gauss_plane(&m, feather * 0.5, feather * 0.5); + } + let invert = ctx.params.b("invert"); + b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| { + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +fn identifier(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("channelType")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("id").round() as f32; + let display = ctx.params.e("display"); + let max_id = ids.iter().cloned().fold(1.0f32, f32::max); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let Some(k) = idx[i] else { + if display != 2 { + *px = [0.0; 4]; + } + return; + }; + let id = ids[k].round(); + let hit = id == sel; + *px = match display { + // Colors: every ID in its own colour, the selected one highlighted white. + 0 => { + let c = if hit && sel != 0.0 { [1.0; 3] } else { id_color(id) }; + [c[0], c[1], c[2], 1.0] + } + 1 => { + let v = if hit { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + } + 2 => { + let s = src.data[i]; + if hit { s } else { [0.0; 4] } + } + _ => { + let v = id / max_id; + [v, v, v, 1.0] + } + }; + }); + b +} + +// ---------------------------------------------------------------- Cryptomatte + +/// Does `name` match a selection pattern (`*` wildcards, case-sensitive like the spec)? +fn wild(pat: &str, name: &str) -> bool { + match pat.split_once('*') { + None => pat == name, + Some((pre, rest)) => { + let Some(tail) = name.strip_prefix(pre) else { return false }; + if rest.is_empty() { + return true; + } + (0..=tail.len()).filter(|&k| tail.is_char_boundary(k)).any(|k| wild(rest, &tail[k..])) + } + } +} + +/// Parse a selection: comma-separated names (optionally quoted) or `` hashes / wildcards. +pub fn selection_hashes(sel: &str, manifest: Option<&[(String, u32)]>) -> Vec { + let mut out = vec![]; + for tok in sel.split(',') { + let t = tok.trim().trim_matches('"').trim_matches('\''); + if t.is_empty() { + continue; + } + if t.contains('*') { + if let Some(m) = manifest { + out.extend(m.iter().filter(|(n, _)| wild(t, n)).map(|(_, h)| *h)); + } + continue; + } + if let Some(hex) = t.strip_prefix("<").and_then(|r| r.strip_suffix(">")) + && let Ok(h) = u32::from_str_radix(hex, 16) + { + out.push(h); + continue; + } + out.push(manifest.and_then(|m| m.iter().find(|(n, _)| n == t).map(|(_, h)| *h)).unwrap_or_else(|| crypto_hash(t))); + } + out +} + +pub const CRYPTO_LAYERS: [&str; 3] = ["CryptoObject", "CryptoMaterial", "CryptoAsset"]; + +fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let layers = a.crypto_layers(); + if layers.is_empty() { + return b; + } + let want = CRYPTO_LAYERS[(ctx.params.e("layer") as usize).min(2)]; + let layer = layers.iter().find(|l| l.as_str() == want || l.ends_with(want)).unwrap_or(&layers[0]).clone(); + let manifest = a.manifests.iter().find(|(l, _)| *l == layer).map(|(_, m)| m.as_slice()); + let sel = selection_hashes(ctx.params.s("selection"), manifest); + let idx = index_map(&b, &a); + let display = ctx.params.e("display"); + let matte_only = ctx.params.b("matteOnly"); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let ranks = idx[i].map(|k| a.crypto_ranks(&layer, k)).unwrap_or_default(); + let m: f32 = ranks.iter().filter(|(id, c)| *c > 0.0 && sel.contains(&id.to_bits())).map(|(_, c)| *c).sum::().clamp(0.0, 1.0); + if matte_only { + *px = [m, m, m, m]; + return; + } + *px = match display { + // Colors: coverage-weighted ID colours (the spec's preview), selection brighter. + 0 => { + let mut c = [0.0f32; 3]; + for (id, cov) in &ranks { + let k = id_color(*id); + for j in 0..3 { + c[j] += k[j] * cov; + } + } + let c = c.map(|v| v * (1.0 - 0.5 * m) + 0.5 * m); + [c[0], c[1], c[2], 1.0] + } + // Matte: the layer keyed by the selection. + 1 => src.data[i].map(|v| v * m), + // Preview: the layer with the selection tinted. + _ => { + let s = src.data[i]; + [s[0] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[1] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[2] * (1.0 - 0.5 * m), s[3]] + } + }; + }); + b +} + +// ---------------------------------------------------------------- EXtractoR + +fn extractor(ctx: &EffectCtx, mut b: Buf) -> Buf { + let a = aux(ctx); + let names = [ctx.params.s("red"), ctx.params.s("green"), ctx.params.s("blue"), ctx.params.s("alpha")]; + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let unmult = ctx.params.b("unMult"); + let clip = ctx.params.b("clip"); + let idx = a.as_ref().map(|a| index_map(&b, a)); + // Each output channel: an aux channel by name, or the layer's own R/G/B/A. + let own = |n: &str| match n.trim().to_ascii_uppercase().as_str() { + "R" | "RED" => Some(0), + "G" | "GREEN" => Some(1), + "B" | "BLUE" => Some(2), + "A" | "ALPHA" => Some(3), + "" => None, + _ => None, + }; + let src_ch: Vec> = names.iter().map(|n| a.as_ref().and_then(|a| if n.trim().is_empty() { None } else { a.get(n.trim()) })).collect(); + if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(n) == Some(k)) { + return b; + } + let src = b.img.clone(); + let lv = |v: f32| { + let o = (v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum(); + if clip { o.clamp(0.0, 1.0) } else { o } + }; + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let (sc, sa) = unpremul(src.data[i]); + let straight = [sc[0], sc[1], sc[2], sa]; + let mut o: Px = [0.0; 4]; + for k in 0..4 { + o[k] = match (src_ch[k], idx.as_ref().and_then(|m| m[i])) { + (Some(ch), Some(j)) => ch[j], + (Some(_), None) => 0.0, + (None, _) => match own(names[k]) { + Some(c) => straight[c], + None => { + if k == 3 { + 1.0 + } else { + 0.0 + } + } + }, + }; + if k < 3 { + o[k] = lv(o[k]); + } + } + let al = o[3].clamp(0.0, 1.0); + let c = if unmult && al > 1e-6 { [o[0] / al, o[1] / al, o[2] / al] } else { [o[0], o[1], o[2]] }; + *px = premul(c, al); + }); + b +} + +pub fn specs() -> Vec { + let text = |id: &'static str, name: &'static str, d: &str| p(id, name, Value::Str(d.into()), ParamUi::Text); + vec![ + spec( + "ec.3d.channelextract", + "3D Channel Extract", + vec![ + p("channel", "3D Channel", Value::Enum(0), popup(&EXTRACT_CHANNELS)), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("whitePoint", "White Point", num(10000.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + ], + channel_extract, + ), + spec( + "ec.3d.cryptomatte", + "Cryptomatte", + vec![ + p("layer", "Layer", Value::Enum(0), popup(&CRYPTO_LAYERS)), + text("selection", "Selection", ""), + p("display", "Display", Value::Enum(2), popup(&["Colors", "Matte", "Preview"])), + p("matteOnly", "Matte Only", Value::Bool(false), ParamUi::Checkbox), + ], + cryptomatte, + ), + spec( + "ec.3d.depthmatte", + "Depth Matte", + vec![ + p("depth", "Depth", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("feather", "Feather", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + ], + depth_matte, + ), + spec( + "ec.3d.depthoffield", + "Depth of Field", + vec![ + p("focalPlane", "Focal Plane", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("maximumRadius", "Maximum Radius", num(0.0), slider(0.0, 500.0, 0.0, 50.0, 2)), + p("focalPlaneThickness", "Focal Plane Thickness", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("focalBias", "Focal Bias", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 2)), + ], + depth_of_field, + ), + spec( + "ec.3d.extractor", + "EXtractoR", + vec![ + text("red", "Red", "R"), + text("green", "Green", "G"), + text("blue", "Blue", "B"), + text("alpha", "Alpha", "A"), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("whitePoint", "White Point", num(1.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("unMult", "UnMult", Value::Bool(false), ParamUi::Checkbox), + p("clip", "Clip", Value::Bool(false), ParamUi::Checkbox), + ], + extractor, + ), + spec( + "ec.3d.fog3d", + "Fog 3D", + vec![ + p("fogColor", "Fog Color", col(1.0, 1.0, 1.0), ParamUi::Color), + p("fogStartDepth", "Fog Start Depth", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogEndDepth", "Fog End Depth", num(3000.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogOpacity", "Fog Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scatteringDensity", "Scattering Density", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("foggyBackground", "Foggy Background", Value::Bool(true), ParamUi::Checkbox), + p("gradientLayer", "Gradient Layer", Value::Layer(None), ParamUi::Layer), + p("layerContribution", "Layer Contribution", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + fog_3d, + ), + spec( + "ec.3d.idmatte", + "ID Matte", + vec![ + p("auxChannel", "Aux. Channel", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("idSelection", "ID Selection", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("useCoverage", "Use Coverage", Value::Bool(false), ParamUi::Checkbox), + ], + id_matte, + ), + spec( + "ec.3d.identifier", + "IDentifier", + vec![ + p("channelType", "Channel Type", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("display", "Display", Value::Enum(0), popup(&["Colors", "Luma Matte", "Alpha Matte", "Raw"])), + p("id", "ID", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + ], + identifier, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + use effectcraft_raster::channels3d::crypto_float; + + /// A 20×10 layer: left half object 1 at depth 100, right half object 2 at depth 500; the + /// top-right corner is background. Cryptomatte: "left" / "right". + struct AuxHost(Arc); + impl EffectHost for AuxHost { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + Some(self.0.clone()) + } + } + + fn scene() -> AuxHost { + let (w, h) = (20usize, 10usize); + let mut a = AuxChannels::new(w as u32, h as u32, 1.0); + let (mut z, mut id, mut r, mut g) = (vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h]); + let (hl, hr) = (crypto_hash("left"), crypto_hash("right")); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if x >= 15 && y < 3 { + z[i] = BACKGROUND_DEPTH; + } else if x < 10 { + z[i] = 100.0; + id[i] = 1.0; + r[i] = crypto_float(hl); + g[i] = 1.0; + } else { + z[i] = 500.0; + id[i] = 2.0; + r[i] = crypto_float(hr); + g[i] = 1.0; + } + } + } + a.channels = vec![("Z".into(), z), ("ObjectID".into(), id), ("CryptoObject00.R".into(), r), ("CryptoObject00.G".into(), g)]; + a.manifests = vec![("CryptoObject".into(), vec![("left".into(), hl), ("right".into(), hr)])]; + AuxHost(Arc::new(a)) + } + + fn img() -> Image { + Image::filled(20, 10, [0.5, 0.5, 0.5, 1.0]) + } + + fn run(id: &str, vals: &[(&str, Value)], host: &AuxHost) -> Image { + run_fx(id, vals, img(), 0.0, EffectEnv { host: Some(host), ..Default::default() }).img + } + + #[test] + fn passes_through_without_aux_channels() { + for s in specs() { + let out = run_fx(s.id, &[], img(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, img().data, "{}", s.id); + } + } + + #[test] + fn depth_matte_thresholds() { + let h = scene(); + let o = run("ec.3d.depthmatte", &[("depth", num(300.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let inv = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("invert", Value::Bool(true))], &h); + assert_eq!(inv.get(2, 5)[3], 1.0); + assert_eq!(inv.get(12, 5)[3], 0.0); + // Feather: a ramp of 800 units centred on 300 → depth 100 at 0.25, 500 at 0.75. + let f = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("feather", num(800.0))], &h); + assert!((f.get(2, 5)[3] - 0.25).abs() < 1e-5 && (f.get(12, 5)[3] - 0.75).abs() < 1e-5); + } + + #[test] + fn channel_extract_maps_depth_and_ids() { + let h = scene(); + let o = run("ec.3d.channelextract", &[("blackPoint", num(0.0)), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6 && (o.get(12, 5)[0] - 0.5).abs() < 1e-6); + assert_eq!(o.get(17, 1)[0], 1.0); + let ids = run("ec.3d.channelextract", &[("channel", Value::Enum(1)), ("whitePoint", num(2.0))], &h); + assert_eq!(ids.get(2, 5)[0], 0.5); + } + + #[test] + fn id_matte_and_identifier() { + let h = scene(); + let o = run("ec.3d.idmatte", &[("idSelection", num(2.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let l = run("ec.3d.identifier", &[("display", Value::Enum(1)), ("id", num(1.0))], &h); + assert_eq!(l.get(2, 5)[0], 1.0); + assert_eq!(l.get(12, 5)[0], 0.0); + let c = run("ec.3d.identifier", &[], &h); + assert_ne!(c.get(2, 5), c.get(12, 5)); + } + + #[test] + fn cryptomatte_selects_by_name_and_wildcard() { + let h = scene(); + let m = run("ec.3d.cryptomatte", &[("selection", Value::Str("right".into())), ("matteOnly", Value::Bool(true))], &h); + assert_eq!(m.get(2, 5)[3], 0.0); + assert_eq!(m.get(12, 5)[3], 1.0); + let w = run("ec.3d.cryptomatte", &[("selection", Value::Str("l*".into())), ("display", Value::Enum(1))], &h); + assert_eq!(w.get(2, 5)[3], 1.0); + assert_eq!(w.get(12, 5)[3], 0.0); + assert!(wild("*ight", "right") && wild("r*t", "right") && !wild("x*", "right")); + let hex = format!("<{:08x}>", crypto_hash("left")); + assert_eq!(selection_hashes(&hex, None), vec![crypto_hash("left")]); + } + + #[test] + fn fog_and_depth_of_field_follow_depth() { + let h = scene(); + let f = run("ec.3d.fog3d", &[("fogColor", col(1.0, 0.0, 0.0)), ("fogEndDepth", num(1000.0)), ("scatteringDensity", num(0.0))], &h); + // Linear fog: 10 % at depth 100, 50 % at 500, full on the background. + assert!((f.get(2, 5)[0] - 0.55).abs() < 1e-4, "{:?}", f.get(2, 5)); + assert!((f.get(12, 5)[0] - 0.75).abs() < 1e-4); + assert!((f.get(17, 1)[0] - 1.0).abs() < 1e-4 && f.get(17, 1)[1] < 1e-4); + assert!((dof_radius(100.0, 100.0, 0.0, 10.0, 50.0)).abs() < 1e-6); + assert!((dof_radius(200.0, 100.0, 0.0, 10.0, 50.0) - 10.0).abs() < 1e-6); + assert_eq!(dof_radius(140.0, 100.0, 100.0, 10.0, 50.0), 0.0); + // A sharp edge at the far object blurs when focused near. + let mut pic = Image::new(20, 10); + for y in 0..10 { + for x in 0..20 { + let v = if x % 2 == 0 { 1.0 } else { 0.0 }; + pic.set(x, y, [v, v, v, 1.0]); + } + } + let env = EffectEnv { host: Some(&h), ..Default::default() }; + let d = run_fx("ec.3d.depthoffield", &[("focalPlane", num(100.0)), ("maximumRadius", num(4.0))], pic.clone(), 0.0, env).img; + assert_eq!(d.get(4, 5), pic.get(4, 5)); + assert!((d.get(12, 5)[0] - 0.5).abs() < 0.3 && d.get(12, 5) != pic.get(12, 5)); + } + + #[test] + fn extractor_maps_named_channels() { + let h = scene(); + let o = run("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6); + assert!((o.get(2, 5)[1] - 0.0005).abs() < 1e-6); + // Without aux: the layer's own channels can be swapped. + let s = run_fx("ec.3d.extractor", &[("red", Value::Str("A".into()))], Image::filled(2, 2, [0.2, 0.3, 0.4, 1.0]), 0.0, EffectEnv::default()); + assert_eq!(s.img.get(0, 0), [1.0, 0.3, 0.4, 1.0]); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 2f6a463e8..c4666b946 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -11,6 +11,7 @@ mod blur2; mod blur3; mod channel; mod channel2; +mod channel3d; mod color2; mod color3; mod color_fx; @@ -57,7 +58,7 @@ pub use color2::Curve; use effectcraft_keyframe::Value; use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; -pub use effectcraft_raster::Image; +pub use effectcraft_raster::{AuxChannels, Image}; /// Effect categories in Effects & Presets order. pub const CATEGORIES: &[&str] = &[ @@ -238,6 +239,13 @@ pub trait EffectHost: Sync { fn layer_at(&self, _id: u64, _comp_time: f64, _masks_and_effects: bool) -> Option { None } + /// Auxiliary 3D channels (depth, object / material IDs, normals, Cryptomatte, any named + /// EXR channel) of the effect's own layer source at the current time, in layer space (see + /// [`AuxChannels`]). `None` when the source has none (the 3D Channel effects then pass the + /// layer through). + fn aux(&self) -> Option> { + None + } } /// Extra context the renderer may supply (all optional; `Default` is "nothing known"). @@ -366,6 +374,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(keylight::specs()); v.extend(mocha_shape::specs()); v.extend(sim3::specs()); + v.extend(channel3d::specs()); v.extend(time_fx::specs()); v.extend(audio_fx::specs()); v.extend(paint::specs()); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 899925693..9466a6cdd 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -19,7 +19,9 @@ filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } +exr = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr"] } rayon = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs new file mode 100644 index 000000000..ba0034c38 --- /dev/null +++ b/crates/media/src/exr_channels.rs @@ -0,0 +1,152 @@ +//! Auxiliary channels from multi-layer OpenEXR files (depth, object / material IDs, normals, +//! Cryptomatte, …) for the 3D Channel effects, read with the pure-Rust `exr` crate. +//! +//! Every channel of every layer becomes a plane named `layer.channel` (or just `channel` for +//! the unnamed layer), placed in the display window (pixels outside a layer's data window are +//! 0, or the background depth for depth channels). Cryptomatte manifests come from the +//! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published +//! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_raster::AuxChannels; +use effectcraft_raster::channels3d::BACKGROUND_DEPTH; + +fn is_depth(name: &str) -> bool { + let last = name.rsplit('.').next().unwrap_or(name); + last.eq_ignore_ascii_case("z") || name.to_ascii_lowercase().contains("depth") +} + +/// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). +pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { + let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; + let mut v: Vec<(String, u32)> = m.iter().filter_map(|(k, h)| Some((k.clone(), u32::from_str_radix(h.as_str()?.trim(), 16).ok()?))).collect(); + v.sort(); + v +} + +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + use exr::prelude::*; + let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let (w, h) = (dw.size.0, dw.size.1); + if w == 0 || h == 0 { + return None; + } + let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); + let mut crypto: HashMap, Option)> = HashMap::new(); + let mut collect_meta = |other: &HashMap| { + for (k, v) in other { + let key = k.to_string(); + let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; + let Some((id, field)) = rest.split_once('/') else { continue }; + let AttributeValue::Text(t) = v else { continue }; + let e = crypto.entry(id.to_string()).or_default(); + match field { + "name" => e.0 = Some(t.to_string()), + "manifest" => e.1 = Some(t.to_string()), + _ => {} + } + } + }; + collect_meta(&img.attributes.other); + for layer in img.layer_data.iter() { + collect_meta(&layer.attributes.other); + let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); + let (lw, lh) = (layer.size.0, layer.size.1); + let pos = layer.attributes.layer_position; + let (ox, oy) = (pos.0 as i64 - dw.position.0 as i64, pos.1 as i64 - dw.position.1 as i64); + for ch in layer.channel_data.list.iter() { + let name = match &prefix { + Some(p) => format!("{p}.{}", ch.name), + None => ch.name.to_string(), + }; + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + let mut plane = vec![fill; w * h]; + let vals: Vec = ch.sample_data.values_as_f32().collect(); + if vals.len() < lw * lh { + continue; + } + for y in 0..lh { + let ty = y as i64 + oy; + if ty < 0 || ty >= h as i64 { + continue; + } + for x in 0..lw { + let tx = x as i64 + ox; + if tx < 0 || tx >= w as i64 { + continue; + } + plane[ty as usize * w + tx as usize] = vals[y * lw + x]; + } + } + aux.channels.push((name, plane)); + } + } + for (_, (name, manifest)) in crypto { + if let (Some(n), Some(m)) = (name, manifest) { + aux.manifests.push((n, parse_manifest(&m))); + } + } + aux.manifests.sort(); + Some(aux) +} + +type Cache = Mutex>)>>; + +/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(Vec::new())); + if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { + return v; + } + let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); + if let Ok(mut c) = cache.lock() { + if c.len() >= 4 { + c.remove(0); + } + c.push((path.to_string(), v.clone())); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_layered_channels_and_cryptomatte_metadata() { + use exr::prelude::*; + let (w, h) = (4usize, 2usize); + let z: Vec = (0..w * h).map(|i| i as f32 * 10.0).collect(); + let id: Vec = (0..w * h).map(|i| (i % 2) as f32 + 1.0).collect(); + let r: Vec = vec![0.5; w * h]; + let channels = AnyChannels::sort( + vec![ + AnyChannel::new("R", FlatSamples::F32(r)), + AnyChannel::new("Z", FlatSamples::F32(z.clone())), + AnyChannel::new("ObjectID", FlatSamples::F32(id.clone())), + ] + .into(), + ); + let mut layer = Layer::new((w, h), LayerAttributes::named("depth"), Encoding::FAST_LOSSLESS, channels); + layer.attributes.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + layer + .attributes + .other + .insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + let image = Image::from_layer(layer); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + let aux = read_exr_channels(bytes.get_ref()).unwrap(); + assert_eq!((aux.width, aux.height), (4, 2)); + assert_eq!(aux.get("depth.Z").unwrap(), z.as_slice()); + assert_eq!(aux.depth().unwrap(), z.as_slice()); + assert_eq!(aux.object_id().unwrap(), id.as_slice()); + assert_eq!(aux.manifests, vec![("CryptoObject".to_string(), vec![("bunny".to_string(), 0x1385_1a76), ("floor".to_string(), 0x0a1b_2c3d)])]); + assert!(read_exr_channels(b"not an exr").is_none()); + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e3..aa4a8f497 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -13,6 +13,7 @@ //! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). mod convert; +pub mod exr_channels; mod pool; mod probe; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef7..caa0c4f90 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -470,6 +470,18 @@ impl FootageSource for MediaPool { } } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { + if footage.missing || !footage.has_video { + return None; + } + let loc = Inner::locate(footage, t); + if loc.media_t.is_some() || !loc.key.path.to_ascii_lowercase().ends_with(".exr") { + return None; + } + let path = loc.key.path.clone(); + crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/raster/src/channels3d.rs b/crates/raster/src/channels3d.rs new file mode 100644 index 000000000..0402349b0 --- /dev/null +++ b/crates/raster/src/channels3d.rs @@ -0,0 +1,206 @@ +//! Auxiliary (3D) channels that travel with a layer's pixels: depth, object / material IDs, +//! surface normals, texture UVs, coverage and Cryptomatte ID/coverage pairs. +//! +//! They come from multi-layer OpenEXR footage (any channel by its full `layer.channel` name) +//! or from the compositor's own render passes for precomps of 3D comps. The 3D Channel effects +//! read them through `EffectHost::aux`. +//! +//! Conventions (shared by the EXR reader and the render pass): +//! +//! * channel planes are row-major `width × height` `f32`s covering the layer's source rectangle, +//! with `scale` aux pixels per layer pixel; +//! * depth is camera-space distance in pixels (`Z`), background `BACKGROUND_DEPTH`; +//! * `ObjectID` / `MaterialID` are small integers (0 = background); +//! * Cryptomatte layers follow the published Cryptomatte specification (Psyop, 2015): a layer +//! named `CryptoObject` stores rank pairs in `CryptoObject00.R/G/B/A`, `CryptoObject01.R/…` +//! (ID float, coverage, ID float, coverage, …), IDs are MurmurHash3 (x86, 32-bit, seed 0) of +//! the object name reinterpreted as a float with the exponent nudged off 0 and 255, and the +//! manifest maps names to hashes. + +/// Depth written where nothing was rendered. +pub const BACKGROUND_DEPTH: f32 = 1.0e10; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AuxChannels { + pub width: u32, + pub height: u32, + /// Aux pixels per layer pixel. + pub scale: f64, + /// Named planes (`Z`, `ObjectID`, `CryptoObject00.R`, …). + pub channels: Vec<(String, Vec)>, + /// Cryptomatte manifests: (layer name, [(object name, hash)]). + pub manifests: Vec<(String, Vec<(String, u32)>)>, +} + +impl AuxChannels { + pub fn new(width: u32, height: u32, scale: f64) -> AuxChannels { + AuxChannels { width, height, scale, channels: vec![], manifests: vec![] } + } + + /// A channel by exact name (case-insensitive). + pub fn get(&self, name: &str) -> Option<&[f32]> { + self.channels.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_slice()) + } + + /// The first channel present among `names` (also matching `*.name` suffixes, so `Z` finds + /// `depth.Z`). + pub fn find(&self, names: &[&str]) -> Option<&[f32]> { + for n in names { + if let Some(v) = self.get(n) { + return Some(v); + } + } + for n in names { + let suffix = format!(".{}", n.to_ascii_lowercase()); + if let Some((_, v)) = self.channels.iter().find(|(c, _)| c.to_ascii_lowercase().ends_with(&suffix)) { + return Some(v); + } + } + None + } + + pub fn names(&self) -> Vec<&str> { + self.channels.iter().map(|(n, _)| n.as_str()).collect() + } + + pub fn depth(&self) -> Option<&[f32]> { + self.find(&["Z", "depth", "Depth", "zDepth", "Z-Depth"]) + } + pub fn object_id(&self) -> Option<&[f32]> { + self.find(&["ObjectID", "object_id", "objectId", "id"]) + } + pub fn material_id(&self) -> Option<&[f32]> { + self.find(&["MaterialID", "material_id", "materialId"]) + } + pub fn coverage(&self) -> Option<&[f32]> { + self.find(&["Coverage", "coverage"]) + } + + /// Cryptomatte layer names present (prefixes with a `00.R` channel). + pub fn crypto_layers(&self) -> Vec { + let mut v: Vec = + self.channels.iter().filter_map(|(n, _)| n.strip_suffix("00.R").or_else(|| n.strip_suffix("00.r")).map(str::to_string)).collect(); + v.sort(); + v.dedup(); + v + } + + /// Rank pairs (id, coverage) of Cryptomatte layer `layer` at aux pixel index `i`. + pub fn crypto_ranks(&self, layer: &str, i: usize) -> Vec<(f32, f32)> { + let mut out = vec![]; + for k in 0.. { + let (r, g, b, a) = ( + self.get(&format!("{layer}{k:02}.R")), + self.get(&format!("{layer}{k:02}.G")), + self.get(&format!("{layer}{k:02}.B")), + self.get(&format!("{layer}{k:02}.A")), + ); + let (Some(r), Some(g)) = (r, g) else { break }; + out.push((r[i], g[i])); + if let (Some(b), Some(a)) = (b, a) { + out.push((b[i], a[i])); + } + } + out + } + + /// Aux pixel index of a layer-space point (nearest; `None` outside). + #[inline] + pub fn index_at(&self, lx: f64, ly: f64) -> Option { + let x = (lx * self.scale).floor(); + let y = (ly * self.scale).floor(); + if x < 0.0 || y < 0.0 || x >= self.width as f64 || y >= self.height as f64 { + return None; + } + Some(y as usize * self.width as usize + x as usize) + } + + /// Manifest entry name for a hash in Cryptomatte layer `layer`. + pub fn manifest_name(&self, layer: &str, hash: u32) -> Option<&str> { + self.manifests.iter().find(|(l, _)| l == layer)?.1.iter().find(|(_, h)| *h == hash).map(|(n, _)| n.as_str()) + } +} + +/// MurmurHash3 x86 32-bit (public-domain algorithm by Austin Appleby). +pub fn murmur3_32(data: &[u8], seed: u32) -> u32 { + const C1: u32 = 0xcc9e_2d51; + const C2: u32 = 0x1b87_3593; + let mut h = seed; + let chunks = data.chunks_exact(4); + let tail = chunks.remainder(); + for c in chunks { + let mut k = u32::from_le_bytes([c[0], c[1], c[2], c[3]]); + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + h = h.rotate_left(13).wrapping_mul(5).wrapping_add(0xe654_6b64); + } + let mut k = 0u32; + for (i, b) in tail.iter().enumerate() { + k ^= (*b as u32) << (8 * i); + } + if !tail.is_empty() { + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + } + h ^= data.len() as u32; + h ^= h >> 16; + h = h.wrapping_mul(0x85eb_ca6b); + h ^= h >> 13; + h = h.wrapping_mul(0xc2b2_ae35); + h ^= h >> 16; + h +} + +/// Cryptomatte hash of a name (the manifest value). +pub fn crypto_hash(name: &str) -> u32 { + let h = murmur3_32(name.as_bytes(), 0); + // Keep the float finite and normal: flip the low exponent bit when it is all 0s or 1s. + let e = (h >> 23) & 0xff; + if e == 0 || e == 255 { h ^ (1 << 23) } else { h } +} + +/// The ID float stored in Cryptomatte channels for a hash. +pub fn crypto_float(hash: u32) -> f32 { + f32::from_bits(hash) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn murmur3_reference_values() { + // Published MurmurHash3_x86_32 test vectors. + assert_eq!(murmur3_32(b"", 0), 0); + assert_eq!(murmur3_32(b"", 1), 0x514e_28b7); + assert_eq!(murmur3_32(b"", 0xffff_ffff), 0x81f1_6f39); + assert_eq!(murmur3_32(&[0, 0, 0, 0], 0), 0x2362_f9de); + assert_eq!(murmur3_32(b"a", 0x9747_b28c), 0x7fa0_9ea6); + assert_eq!(murmur3_32(b"aaaa", 0x9747_b28c), 0x5a97_808a); + assert_eq!(murmur3_32(b"Hello, world!", 0x9747_b28c), 0x2488_4cba); + assert_eq!(murmur3_32(b"The quick brown fox jumps over the lazy dog", 0x9747_b28c), 0x2fa8_26cd); + } + + #[test] + fn crypto_floats_are_normal_and_round_trip() { + for n in ["bunny", "set", "ground", "a", ""] { + let h = crypto_hash(n); + let f = crypto_float(h); + assert!(f.is_finite() && (f == 0.0 || f.is_normal() || f.abs() > 0.0), "{n}"); + assert_eq!(f.to_bits(), h); + } + } + + #[test] + fn lookup_helpers() { + let mut a = AuxChannels::new(2, 1, 1.0); + a.channels.push(("depth.Z".into(), vec![5.0, 6.0])); + a.channels.push(("CryptoObject00.R".into(), vec![1.0, 2.0])); + a.channels.push(("CryptoObject00.G".into(), vec![0.5, 1.0])); + assert_eq!(a.depth(), Some(&[5.0, 6.0][..])); + assert_eq!(a.crypto_layers(), vec!["CryptoObject".to_string()]); + assert_eq!(a.crypto_ranks("CryptoObject", 0), vec![(1.0, 0.5)]); + assert_eq!(a.index_at(1.5, 0.2), Some(1)); + assert_eq!(a.index_at(2.5, 0.2), None); + } +} diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index dfe538486..4f2441996 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -4,6 +4,7 @@ //! heavy operation is row-parallel with rayon. pub mod blur; +pub mod channels3d; pub mod flow; pub mod warp; @@ -11,6 +12,7 @@ use effectcraft_color::{BlendMode, blend_pixel}; use rayon::prelude::*; pub use blur::{box_blur, directional_blur, gaussian_blur, radial_blur}; +pub use channels3d::AuxChannels; pub use warp::{Sampling, WarpOpts, accumulate_warp, composite_warp, composite_warp_reference, resample}; pub type Px = [f32; 4]; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index bd42d31a4..f09f1a6e2 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -39,6 +39,12 @@ pub trait FootageSource: Send + Sync { fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { None } + /// Auxiliary 3D channels of `item` at source time `t` (multi-layer OpenEXR: depth, IDs, + /// Cryptomatte, any named channel), with `scale` relative to the file's pixels. `None` when + /// the footage has none. + fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { + None + } /// Set the decoded-frame cache budget in bytes (Settings ▸ Memory & CPU); sources without /// a cache ignore it. fn set_cache_budget(&self, _bytes: usize) {} @@ -107,6 +113,29 @@ impl EffectHost for FxHost<'_, '_, '_> { sub.layer_input(&self.ctx.at(t), self.layer, n).map(|b| (*b).clone()) } + fn aux(&self) -> Option> { + match &self.layer.source { + LayerSource::Footage { item } => { + let ItemKind::Footage(f) = &self.r.project.item(*item)?.kind else { return None }; + let a = self.r.footage.aux(*item, f, self.ctx.source_time(self.layer))?; + // Aux pixels per layer pixel (the layer is the footage at its nominal size). + if f.width > 0 && a.width > 0 { + let mut a = (*a).clone(); + a.scale = a.width as f64 / f.width as f64; + return Some(Arc::new(a)); + } + Some(a) + } + LayerSource::Comp { item } => { + if self.r.depth > MAX_FX_DEPTH || self.r.project.comp_contains(*item, self.ctx.comp_id) { + return None; + } + self.r.nested().comp_aux(*item, self.ctx.source_time(self.layer)).map(Arc::new) + } + _ => None, + } + } + fn layer_at(&self, id: u64, comp_time: f64, masks_and_effects: bool) -> Option { let other = self.ctx.layer(effectcraft_project::LayerId(id))?; if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { @@ -1175,6 +1204,13 @@ impl<'a> Renderer<'a> { self.depth } + /// The auxiliary 3D channels of comp `comp_id` at comp time `t` (depth, layer IDs, normals, + /// UVs, Cryptomatte; see `three_d::compose::aux_pass`), at the renderer's output scale. + pub fn comp_aux(&self, comp_id: ItemId, t: Tick) -> Option { + let ctx = self.eval_ctx(comp_id, t)?; + Some(three_d::compose::aux_pass(self, &ctx)) + } + /// Evaluation context of comp `comp_id` at comp time `t`. pub fn eval_ctx(&self, comp_id: ItemId, t: Tick) -> Option> { Some(self.ctx(comp_id, self.project.comp(comp_id)?, t)) @@ -1295,6 +1331,8 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { #[cfg(test)] mod tests_audio_fx; #[cfg(test)] +mod tests_aux; +#[cfg(test)] mod tests_collapse; #[cfg(test)] mod tests_color; diff --git a/crates/render/src/tests_aux.rs b/crates/render/src/tests_aux.rs new file mode 100644 index 000000000..31bb18813 --- /dev/null +++ b/crates/render/src/tests_aux.rs @@ -0,0 +1,98 @@ +//! Auxiliary 3D channels: the comp's depth / ID / Cryptomatte pass and the 3D Channel effects on +//! a precomp of a 3D comp. + +use effectcraft_color::Label; +use effectcraft_keyframe::Value; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project, Solid}; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{NoFootage, RenderOpts, Renderer, render_frame}; + +const W: u32 = 200; +const H: u32 = 100; + +fn zoom() -> f64 { + W as f64 * 50.0 / 36.0 +} + +fn solid3(p: &mut Project, comp: &Comp, name: &str, color: [f32; 3], pos: [f64; 3]) -> Layer { + let sid = p.add_item(name, Label::Red, None, ItemKind::Solid(Solid { color, width: 60, height: 60, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, name, LayerSource::Solid { item: sid }, (60, 60), None); + l.name = name.to_string(); + l.switches.three_d = true; + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3(pos); + l +} + +/// Inner 3D comp: "Near" (red, z 0, left) above "Far" (green, z 500, right). Outer comp: the +/// precomp layer, optionally with an effect. +fn scene(effect: Option<(&str, Vec<(&str, Value)>)>) -> (Project, ItemId, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = effectcraft_project::BitDepth::Bpc32; + let inner = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let iid = p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into())); + let near = solid3(&mut p, &inner, "Near", [1.0, 0.0, 0.0], [60.0, 50.0, 0.0]); + let far = solid3(&mut p, &inner, "Far", [0.0, 1.0, 0.0], [140.0, 50.0, 500.0]); + p.comp_mut(iid).unwrap().layers.push(near); + p.comp_mut(iid).unwrap().layers.push(far); + let outer = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let oid = p.add_item("Outer", Label::Sandstone, None, ItemKind::Comp(outer.clone().into())); + let mut pre = build::layer(&mut p, &outer, "Inner", LayerSource::Comp { item: iid }, (W, H), None); + if let Some((id, vals)) = effect { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [W as f64, H as f64]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap().value = v.clone(); + } + pre.props.sub_mut("effects").unwrap().children.push(g.into()); + } + p.comp_mut(oid).unwrap().layers.push(pre); + (p, iid, oid) +} + +#[test] +fn comp_aux_pass_has_depth_ids_and_cryptomatte() { + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts::default()); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height), (W, H)); + let at = |name: &str, x: usize, y: usize| a.get(name).unwrap()[y * W as usize + x]; + // Near plane at the camera's zoom distance, the far one 500 px behind it. + assert!((at("Z", 60, 50) as f64 - zoom()).abs() < 1e-2, "{}", at("Z", 60, 50)); + assert!((at("Z", 114, 50) as f64 - (zoom() + 500.0)).abs() < 1e-2, "{}", at("Z", 114, 50)); + assert_eq!(at("Z", 5, 5), BACKGROUND_DEPTH); + assert_eq!((at("ObjectID", 60, 50), at("ObjectID", 114, 50), at("ObjectID", 5, 5)), (1.0, 2.0, 0.0)); + assert_eq!(at("Coverage", 60, 50), 1.0); + // Planes face the camera. + assert!((at("N.Z", 60, 50) + 1.0).abs() < 1e-5); + // Cryptomatte rank 0 = the layer under the pixel, full coverage; manifest lists names. + assert_eq!(at("CryptoObject00.R", 60, 50).to_bits(), crypto_hash("Near")); + assert_eq!(at("CryptoObject00.G", 60, 50), 1.0); + assert_eq!(a.manifest_name("CryptoObject", crypto_hash("Far")), Some("Far")); +} + +#[test] +fn three_d_channel_effects_work_on_a_precomp_of_a_3d_comp() { + let near = (60, 50); + let far = (114, 50); + let alpha = |p: &Project, oid: ItemId, (x, y): (i64, i64)| render_frame(p, oid, Tick::ZERO, 1.0).get(x, y)[3]; + let (p, _, oid) = scene(Some(("ec.3d.idmatte", vec![("idSelection", Value::Scalar(2.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.depthmatte", vec![("depth", Value::Scalar(zoom() + 250.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.cryptomatte", vec![("selection", Value::Str("Near".into())), ("display", Value::Enum(1))]))); + assert!(alpha(&p, oid, near) > 0.99); + assert_eq!(alpha(&p, oid, far), 0.0); + // Half resolution: the pass follows the render scale. + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts { scale: 0.5, ..Default::default() }); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height, a.scale), (W / 2, H / 2, 0.5)); + assert_eq!(a.get("ObjectID").unwrap()[25 * 100 + 30], 1.0); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 1ef7412b5..8f8e6ff57 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -583,3 +583,182 @@ fn occlusion(c: &Item, p: Vec3, l: &LightState) -> [f32; 3] { } out } + +// ------------------------------------------------------------------ auxiliary render pass + +/// One surface seen through an output pixel (aux pass). +#[derive(Clone, Copy)] +struct AuxFrag { + z: f32, + a: f32, + obj: f32, + mat: f32, + n: [f32; 3], + uv: [f32; 2], + obj_hash: u32, + mat_hash: u32, +} + +/// The comp's auxiliary channels at the context time, at output resolution: camera-space depth +/// (`Z`), `ObjectID` (layer number), `MaterialID` (source item id), `Coverage`, camera-space +/// normals (`N.X/Y/Z`), texture coordinates (`UV.U/V`) and Cryptomatte `CryptoObject` (layer +/// names) / `CryptoMaterial` (source names) with two ranks each, plus their manifests. +/// +/// Layers are gathered in stack order (top first); a run of 3D layers contributes its planes +/// sorted near → far at each pixel, a 2D layer its coverage on the plane of the comp. The +/// front-most surface with ≥ 50 % opacity (else the strongest one) provides depth, IDs, normal +/// and UV; Cryptomatte coverage is each surface's visible share (`α · Π(1 − α_front)`). +pub(crate) fn aux_pass(r: &Renderer, ctx: &EvalCtx) -> effectcraft_raster::AuxChannels { + use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_float, crypto_hash}; + let s = r.opts.scale; + let (w, h) = (((ctx.comp.width as f64 * s).round() as u32).max(1), ((ctx.comp.height as f64 * s).round() as u32).max(1)); + let out = (w, h); + let (wu, n) = (w as usize, w as usize * h as usize); + let cam = camera_for(r, ctx); + let visible: Vec<&Layer> = r.visible_layers(ctx).into_iter().filter(|l| !l.switches.adjustment).collect(); + let number = |l: &Layer| ctx.comp.layers.iter().position(|x| x.id == l.id).map_or(0.0, |i| i as f32 + 1.0); + let source_name = |l: &Layer| l.source.item().and_then(|i| r.project.item(i)).map(|it| it.name.clone()).unwrap_or_else(|| l.name.clone()); + let source_id = |l: &Layer| l.source.item().map_or(0.0, |i| i.0 as f32); + let mut manifest_obj: Vec<(String, u32)> = vec![]; + let mut manifest_mat: Vec<(String, u32)> = vec![]; + for l in &visible { + let (on, mn) = (l.name.clone(), source_name(l)); + if !manifest_obj.iter().any(|(x, _)| *x == on) { + manifest_obj.push((on.clone(), crypto_hash(&on))); + } + if !manifest_mat.iter().any(|(x, _)| *x == mn) { + manifest_mat.push((mn.clone(), crypto_hash(&mn))); + } + } + // Groups in stack order: runs of 3D layers, single 2D layers. + let mut groups: Vec<&[&Layer]> = vec![]; + let mut i = 0; + while i < visible.len() { + let mut j = i + 1; + if visible[i].is_3d() { + while j < visible.len() && visible[j].is_3d() { + j += 1; + } + } + groups.push(&visible[i..j]); + i = j; + } + let mut frags: Vec> = vec![Vec::new(); n]; + let plane_z = cam.depth(vec3(ctx.comp.width as f64 * 0.5, ctx.comp.height as f64 * 0.5, 0.0)) as f32; + for g in groups.iter().rev() { + if g[0].is_3d() { + let items: Vec = g.iter().enumerate().flat_map(|(k, l)| prepare_all(r, ctx, l, k, out, true)).collect(); + let meta: Vec<(f32, f32, u32, u32, [f32; 3], [f64; 2])> = items + .iter() + .map(|it| { + let nrm = it.geos.first().map_or(vec3(0.0, 0.0, -1.0), |gm| cam.view.apply_vec(gm.normal)); + let mut nv = [nrm.x as f32, nrm.y as f32, nrm.z as f32]; + if nv[2] > 0.0 { + nv = nv.map(|v| -v); + } + let (sw, sh) = crate::source_size(r.project, it.layer); + let size = if sw == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [sw as f64, sh as f64] }; + (number(it.layer), source_id(it.layer), crypto_hash(&it.layer.name), crypto_hash(&source_name(it.layer)), nv, size) + }) + .collect(); + frags.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + let yi = y as i64; + let mut local: Vec = Vec::new(); + for (x, cell) in row.iter_mut().enumerate() { + let xi = x as i64; + local.clear(); + for (it, m) in items.iter().zip(&meta) { + if xi < it.bbox[0] || xi >= it.bbox[2] || yi < it.bbox[1] || yi >= it.bbox[3] { + continue; + } + let Some(gm) = it.geos.first() else { continue }; + let Some((u, v, z)) = unproject(gm, x as f64 + 0.5, y as f64 + 0.5) else { continue }; + let a = it.texel(u, v)[3] * it.opacity; + if a > 1e-4 { + let uv = [(u / m.5[0]) as f32, (v / m.5[1]) as f32]; + local.push(AuxFrag { z: z as f32, a: a.min(1.0), obj: m.0, mat: m.1, n: m.4, uv, obj_hash: m.2, mat_hash: m.3 }); + } + } + local.sort_by(|p, q| p.z.total_cmp(&q.z)); + cell.extend_from_slice(&local); + } + }); + } else { + let l = g[0]; + let mut img = Image::new(w, h); + if let Some(b) = r.layer_buf(ctx, l) { + r.place(ctx, l, &b, &mut img, BlendMode::Normal, ctx.opacity(l) as f32); + } + let (obj, mat, oh, mh) = (number(l), source_id(l), crypto_hash(&l.name), crypto_hash(&source_name(l))); + frags.par_chunks_mut(wu).zip(img.data.par_chunks(wu)).enumerate().for_each(|(y, (row, px))| { + for (x, (cell, p)) in row.iter_mut().zip(px).enumerate() { + if p[3] > 1e-4 { + let uv = [(x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32]; + cell.push(AuxFrag { z: plane_z, a: p[3].min(1.0), obj, mat, n: [0.0, 0.0, -1.0], uv, obj_hash: oh, mat_hash: mh }); + } + } + }); + } + } + // Resolve per pixel. + const NAMES: [&str; 9] = ["Z", "ObjectID", "MaterialID", "Coverage", "N.X", "N.Y", "N.Z", "UV.U", "UV.V"]; + const CRYPTO: usize = 16; + let resolved: Vec<([f32; 9], [f32; CRYPTO])> = frags + .par_iter() + .map(|fs| { + let mut o = [BACKGROUND_DEPTH, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let mut crypto = [0.0f32; CRYPTO]; + if fs.is_empty() { + return (o, crypto); + } + let mut t = 1.0f32; + let mut obj: Vec<(u32, f32)> = vec![]; + let mut mat: Vec<(u32, f32)> = vec![]; + let mut best: Option<(usize, f32)> = None; + let mut front: Option = None; + for (k, f) in fs.iter().enumerate() { + let c = f.a * t; + if front.is_none() && f.a >= 0.5 { + front = Some(k); + } + if best.is_none_or(|(_, bc)| c > bc) { + best = Some((k, c)); + } + for (list, hsh) in [(&mut obj, f.obj_hash), (&mut mat, f.mat_hash)] { + match list.iter_mut().find(|(x, _)| *x == hsh) { + Some(e) => e.1 += c, + None => list.push((hsh, c)), + } + } + t *= 1.0 - f.a; + if t <= 1e-5 { + break; + } + } + let f = fs[front.or(best.map(|b| b.0)).unwrap_or(0)]; + o = [f.z, f.obj, f.mat, 1.0 - t, f.n[0], f.n[1], f.n[2], f.uv[0], f.uv[1]]; + for (base, list) in [(0usize, &mut obj), (8usize, &mut mat)] { + list.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + for (rank, (hsh, c)) in list.iter().take(4).enumerate() { + crypto[base + rank * 2] = crypto_float(*hsh); + crypto[base + rank * 2 + 1] = *c; + } + } + (o, crypto) + }) + .collect(); + let mut aux = effectcraft_raster::AuxChannels::new(w, h, s); + for (k, name) in NAMES.iter().enumerate() { + aux.channels.push((name.to_string(), resolved.iter().map(|r| r.0[k]).collect())); + } + for (base, layer) in [(0usize, "CryptoObject"), (8usize, "CryptoMaterial")] { + for rank_pair in 0..2 { + for (c, ch) in ["R", "G", "B", "A"].iter().enumerate() { + let k = base + rank_pair * 4 + c; + aux.channels.push((format!("{layer}{rank_pair:02}.{ch}"), resolved.iter().map(|r| r.1[k]).collect())); + } + } + } + aux.manifests = vec![("CryptoObject".into(), manifest_obj), ("CryptoMaterial".into(), manifest_mat)]; + aux +}