mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-31 21:48:20 -06:00
M9.11: 3D Channel effects on auxiliary channels
Aux-channel model (effectcraft_raster::channels3d: depth, object/material ID, normals, UV, coverage, Cryptomatte ranks + manifests, MurmurHash3); EffectHost::aux; multi-layer OpenEXR channels via the exr crate (FootageSource::aux); the compositor's depth / layer-ID / Cryptomatte pass (Renderer::comp_aux) so the effects work on precomps of 3D comps. 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, Fog 3D, ID Matte, IDentifier. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7b815bcbb2
commit
9a11c577f3
Generated
+3
@@ -1327,6 +1327,7 @@ dependencies = [
|
||||
"effectcraft-raster",
|
||||
"effectcraft-render",
|
||||
"effectcraft-time",
|
||||
"exr",
|
||||
"filmcraft-codecs",
|
||||
"filmcraft-color",
|
||||
"filmcraft-frame",
|
||||
@@ -1335,6 +1336,7 @@ dependencies = [
|
||||
"image",
|
||||
"log",
|
||||
"rayon",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
@@ -1768,6 +1770,7 @@ dependencies = [
|
||||
"miniz_oxide 0.8.9",
|
||||
"num-complex",
|
||||
"pulp",
|
||||
"rayon-core",
|
||||
"smallvec",
|
||||
"zune-inflate",
|
||||
]
|
||||
|
||||
@@ -52,6 +52,7 @@ kurbo = { version = "0.13", features = ["serde"] }
|
||||
tiny-skia = { version = "0.12", default-features = false, features = ["std", "simd"] }
|
||||
skrifa = { version = "0.44", default-features = false, features = ["std"] }
|
||||
harfrust = { version = "0.12", default-features = false, features = ["std"] }
|
||||
exr = "1.74"
|
||||
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
|
||||
|
||||
# media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001).
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
//! 3D Channel effects: 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR,
|
||||
//! Fog 3D, ID Matte and IDentifier.
|
||||
//!
|
||||
//! They read the layer's auxiliary channels ([`AuxChannels`], via
|
||||
//! [`crate::EffectHost::aux`]): multi-layer OpenEXR footage (depth, object/material IDs,
|
||||
//! Cryptomatte layers, any named channel) or the compositor's depth / layer-ID / Cryptomatte
|
||||
//! pass for a precomp of a 3D comp. Without auxiliary channels they pass the layer through
|
||||
//! (EXtractoR still maps the layer's own R, G, B and A).
|
||||
//!
|
||||
//! Aux pixels are looked up nearest-neighbour (IDs must not be interpolated) at each buffer
|
||||
//! pixel's layer position, so render scale and layer padding are respected.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash};
|
||||
use effectcraft_raster::{AuxChannels, Image, Px, gaussian_blur};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::util::{Plane, gauss_plane, hash1, premul, unpremul};
|
||||
use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider};
|
||||
|
||||
fn spec(id: &'static str, name: &'static str, params: Vec<crate::ParamSpec>, render: crate::RenderFn) -> EffectSpec {
|
||||
EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true }
|
||||
}
|
||||
|
||||
fn aux(ctx: &EffectCtx) -> Option<Arc<AuxChannels>> {
|
||||
ctx.env.host?.aux()
|
||||
}
|
||||
|
||||
/// Aux index for each buffer pixel (None outside the aux image).
|
||||
fn index_map(b: &Buf, a: &AuxChannels) -> Vec<Option<usize>> {
|
||||
let (w, h) = (b.img.width as usize, b.img.height as usize);
|
||||
let inv = 1.0 / b.scale.max(1e-9);
|
||||
(0..w * h)
|
||||
.into_par_iter()
|
||||
.map(|i| {
|
||||
let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5);
|
||||
a.index_at((x - b.offset[0]) * inv, (y - b.offset[1]) * inv)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A channel sampled into the buffer grid (`fill` outside).
|
||||
fn plane_of(b: &Buf, idx: &[Option<usize>], ch: &[f32], fill: f32) -> Plane {
|
||||
Plane { w: b.img.width as usize, h: b.img.height as usize, data: idx.par_iter().map(|i| i.map_or(fill, |i| ch[i])).collect() }
|
||||
}
|
||||
|
||||
/// Distinct colour for an ID (hash → hue).
|
||||
fn id_color(id: f32) -> [f32; 3] {
|
||||
let h = hash1(id.to_bits(), 0x1d, 0x5eed);
|
||||
let (r, g, b) = effectcraft_color::hsl_to_rgb(h, 0.75, 0.5);
|
||||
if id == 0.0 { [0.0; 3] } else { [r, g, b] }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 3D Channel Extract
|
||||
|
||||
pub const EXTRACT_CHANNELS: [&str; 8] = ["Z-Depth", "Object ID", "Texture UV", "Surface Normals", "Coverage", "Background RGB", "Unclamped RGB", "Material ID"];
|
||||
|
||||
fn channel_extract(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let idx = index_map(&b, &a);
|
||||
let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32);
|
||||
let lv = |v: f32| ((v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum()).clamp(0.0, 1.0);
|
||||
let lv = |v: f32| if wp >= bp { lv(v) } else { 1.0 - ((v - wp) / (bp - wp).max(1e-9)).clamp(0.0, 1.0) };
|
||||
let get3 = |names: [&[&str]; 3]| -> Option<[&[f32]; 3]> { Some([a.find(names[0])?, a.find(names[1])?, a.find(names[2])?]) };
|
||||
let img: Option<Image> = match ctx.params.e("channel") {
|
||||
0 => a.depth().map(|z| map_grey(&b, &idx, |i| lv(z[i]))),
|
||||
1 => a.object_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))),
|
||||
7 => a.material_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))),
|
||||
4 => a.coverage().map(|c| map_grey(&b, &idx, |i| lv(c[i]))),
|
||||
2 => get3([&["UV.U", "U", "uv.x"], &["UV.V", "V", "uv.y"], &["UV.W", "W", "uv.z"]])
|
||||
.or_else(|| Some([a.find(&["UV.U", "U", "uv.x"])?, a.find(&["UV.V", "V", "uv.y"])?, a.find(&["UV.V", "V", "uv.y"])?]))
|
||||
.map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), 0.0])),
|
||||
3 => get3([&["N.X", "Normal.X", "normals.x"], &["N.Y", "Normal.Y", "normals.y"], &["N.Z", "Normal.Z", "normals.z"]])
|
||||
.map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i] * 0.5 + 0.5), lv(c[1][i] * 0.5 + 0.5), lv(c[2][i] * 0.5 + 0.5)])),
|
||||
5 => get3([&["BG.R", "Background.R"], &["BG.G", "Background.G"], &["BG.B", "Background.B"]])
|
||||
.map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), lv(c[2][i])])),
|
||||
_ => get3([&["R"], &["G"], &["B"]]).map(|c| map_rgb(&b, &idx, |i| [c[0][i], c[1][i], c[2][i]])),
|
||||
};
|
||||
if let Some(i) = img {
|
||||
b.img = i;
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
fn map_grey(b: &Buf, idx: &[Option<usize>], f: impl Fn(usize) -> f32 + Sync) -> Image {
|
||||
map_rgb(b, idx, |i| {
|
||||
let v = f(i);
|
||||
[v, v, v]
|
||||
})
|
||||
}
|
||||
|
||||
fn map_rgb(b: &Buf, idx: &[Option<usize>], f: impl Fn(usize) -> [f32; 3] + Sync) -> Image {
|
||||
let mut img = Image::new(b.img.width, b.img.height);
|
||||
img.data.par_iter_mut().zip(idx.par_iter()).for_each(|(p, i)| {
|
||||
if let Some(i) = i {
|
||||
let c = f(*i);
|
||||
*p = [c[0], c[1], c[2], 1.0];
|
||||
}
|
||||
});
|
||||
img
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Depth Matte
|
||||
|
||||
fn depth_matte(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let Some(z) = a.depth() else { return b };
|
||||
let idx = index_map(&b, &a);
|
||||
let depth = ctx.params.f("depth") as f32;
|
||||
let feather = ctx.params.f("feather").max(0.0) as f32;
|
||||
let invert = ctx.params.b("invert");
|
||||
// Keeps what lies at or beyond Depth (Invert keeps what is in front), with a soft ramp of
|
||||
// `feather` depth units centred on the threshold.
|
||||
let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH);
|
||||
b.img.data.par_iter_mut().zip(zp.data.par_iter()).for_each(|(px, &zv)| {
|
||||
let k = if feather > 0.0 {
|
||||
((zv - depth) / feather + 0.5).clamp(0.0, 1.0)
|
||||
} else if zv >= depth {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let k = if invert { 1.0 - k } else { k };
|
||||
*px = px.map(|v| v * k);
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Depth of Field
|
||||
|
||||
/// Blur radius (pixels) for depth `z`.
|
||||
pub fn dof_radius(z: f32, focal: f32, thickness: f32, max_r: f32, bias: f32) -> f32 {
|
||||
let d = ((z - focal).abs() - thickness * 0.5).max(0.0);
|
||||
let range = focal.abs().max(1.0);
|
||||
let n = (d / range).min(1.0);
|
||||
// Bias 50 = linear; higher bias brings blur in sooner.
|
||||
let gamma = 2f32.powf((50.0 - bias) / 25.0);
|
||||
max_r * n.powf(gamma)
|
||||
}
|
||||
|
||||
fn depth_of_field(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let Some(z) = a.depth() else { return b };
|
||||
let max_r = (ctx.params.f("maximumRadius").max(0.0) * b.scale) as f32;
|
||||
if max_r <= 0.0 {
|
||||
return b;
|
||||
}
|
||||
let focal = ctx.params.f("focalPlane") as f32;
|
||||
let thick = ctx.params.f("focalPlaneThickness").max(0.0) as f32;
|
||||
let bias = ctx.params.f("focalBias") as f32;
|
||||
let idx = index_map(&b, &a);
|
||||
let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH);
|
||||
// A stack of blurs at a few radii, interpolated per pixel by its circle of confusion.
|
||||
const LEVELS: usize = 5;
|
||||
let stack: Vec<Image> = (0..LEVELS)
|
||||
.into_par_iter()
|
||||
.map(|k| {
|
||||
let r = max_r * k as f32 / (LEVELS - 1) as f32;
|
||||
if r < 0.3 { b.img.clone() } else { gaussian_blur(&b.img, (r / 2.0) as f64, (r / 2.0) as f64, true) }
|
||||
})
|
||||
.collect();
|
||||
b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| {
|
||||
let r = dof_radius(zp.data[i], focal, thick, max_r, bias);
|
||||
let f = (r / max_r * (LEVELS - 1) as f32).clamp(0.0, (LEVELS - 1) as f32);
|
||||
let k0 = (f.floor() as usize).min(LEVELS - 2);
|
||||
let t = f - k0 as f32;
|
||||
let (a0, a1) = (stack[k0].data[i], stack[k0 + 1].data[i]);
|
||||
*px = [0, 1, 2, 3].map(|c| a0[c] + (a1[c] - a0[c]) * t);
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Fog 3D
|
||||
|
||||
fn fog_3d(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let Some(z) = a.depth() else { return b };
|
||||
let pr = ctx.params;
|
||||
let fc = pr.color("fogColor");
|
||||
let (start, end) = (pr.f("fogStartDepth") as f32, pr.f("fogEndDepth") as f32);
|
||||
let opacity = (pr.f("fogOpacity") / 100.0) as f32;
|
||||
let density = (pr.f("scatteringDensity") / 100.0) as f32;
|
||||
let foggy_bg = pr.b("foggyBackground");
|
||||
let contrib = (pr.f("layerContribution") / 100.0) as f32;
|
||||
let grad = if contrib > 0.0 { ctx.layer_param("gradientLayer", true).map(|o| crate::util::fit_layer(ctx, &b, &o, true)) } else { None };
|
||||
let idx = index_map(&b, &a);
|
||||
let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH);
|
||||
b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| {
|
||||
let zv = zp.data[i];
|
||||
let bg = zv >= BACKGROUND_DEPTH * 0.5;
|
||||
let mut f = if bg {
|
||||
if foggy_bg { 1.0 } else { 0.0 }
|
||||
} else {
|
||||
let t = ((zv - start) / (end - start).abs().max(1e-6)).clamp(0.0, 1.0);
|
||||
// Denser scattering thickens the fog sooner.
|
||||
1.0 - (1.0 - t).powf(1.0 + density * 3.0)
|
||||
};
|
||||
if let Some(g) = &grad {
|
||||
let (gc, ga) = unpremul(g.data[i]);
|
||||
let l = effectcraft_color::luminance(gc[0], gc[1], gc[2]) * ga;
|
||||
f *= 1.0 - contrib + contrib * l;
|
||||
}
|
||||
let f = (f * opacity).clamp(0.0, 1.0);
|
||||
let (c, al) = unpremul(*px);
|
||||
let al2 = if bg && foggy_bg { al + (1.0 - al) * f } else { al };
|
||||
let out = [0, 1, 2].map(|k| c[k] + (fc[k] - c[k]) * f);
|
||||
*px = premul(out, al2);
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- ID Matte / IDentifier
|
||||
|
||||
fn id_channel(a: &AuxChannels, which: u32) -> Option<&[f32]> {
|
||||
if which == 1 { a.material_id() } else { a.object_id() }
|
||||
}
|
||||
|
||||
fn id_matte(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let Some(ids) = id_channel(&a, ctx.params.e("auxChannel")) else { return b };
|
||||
let idx = index_map(&b, &a);
|
||||
let sel = ctx.params.f("idSelection").round() as f32;
|
||||
let cov = if ctx.params.b("useCoverage") { a.coverage() } else { None };
|
||||
let mut m = Plane {
|
||||
w: b.img.width as usize,
|
||||
h: b.img.height as usize,
|
||||
data: idx.par_iter().map(|i| i.map_or(0.0, |i| if ids[i].round() == sel { cov.map_or(1.0, |c| c[i].clamp(0.0, 1.0)) } else { 0.0 })).collect(),
|
||||
};
|
||||
let feather = ctx.params.f("feather").max(0.0) * b.scale;
|
||||
if feather > 0.0 {
|
||||
m = gauss_plane(&m, feather * 0.5, feather * 0.5);
|
||||
}
|
||||
let invert = ctx.params.b("invert");
|
||||
b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| {
|
||||
let k = if invert { 1.0 - k } else { k };
|
||||
*px = px.map(|v| v * k);
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
fn identifier(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let Some(ids) = id_channel(&a, ctx.params.e("channelType")) else { return b };
|
||||
let idx = index_map(&b, &a);
|
||||
let sel = ctx.params.f("id").round() as f32;
|
||||
let display = ctx.params.e("display");
|
||||
let max_id = ids.iter().cloned().fold(1.0f32, f32::max);
|
||||
let src = b.img.clone();
|
||||
b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| {
|
||||
let Some(k) = idx[i] else {
|
||||
if display != 2 {
|
||||
*px = [0.0; 4];
|
||||
}
|
||||
return;
|
||||
};
|
||||
let id = ids[k].round();
|
||||
let hit = id == sel;
|
||||
*px = match display {
|
||||
// Colors: every ID in its own colour, the selected one highlighted white.
|
||||
0 => {
|
||||
let c = if hit && sel != 0.0 { [1.0; 3] } else { id_color(id) };
|
||||
[c[0], c[1], c[2], 1.0]
|
||||
}
|
||||
1 => {
|
||||
let v = if hit { 1.0 } else { 0.0 };
|
||||
[v, v, v, 1.0]
|
||||
}
|
||||
2 => {
|
||||
let s = src.data[i];
|
||||
if hit { s } else { [0.0; 4] }
|
||||
}
|
||||
_ => {
|
||||
let v = id / max_id;
|
||||
[v, v, v, 1.0]
|
||||
}
|
||||
};
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Cryptomatte
|
||||
|
||||
/// Does `name` match a selection pattern (`*` wildcards, case-sensitive like the spec)?
|
||||
fn wild(pat: &str, name: &str) -> bool {
|
||||
match pat.split_once('*') {
|
||||
None => pat == name,
|
||||
Some((pre, rest)) => {
|
||||
let Some(tail) = name.strip_prefix(pre) else { return false };
|
||||
if rest.is_empty() {
|
||||
return true;
|
||||
}
|
||||
(0..=tail.len()).filter(|&k| tail.is_char_boundary(k)).any(|k| wild(rest, &tail[k..]))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a selection: comma-separated names (optionally quoted) or `<hex>` hashes / wildcards.
|
||||
pub fn selection_hashes(sel: &str, manifest: Option<&[(String, u32)]>) -> Vec<u32> {
|
||||
let mut out = vec![];
|
||||
for tok in sel.split(',') {
|
||||
let t = tok.trim().trim_matches('"').trim_matches('\'');
|
||||
if t.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if t.contains('*') {
|
||||
if let Some(m) = manifest {
|
||||
out.extend(m.iter().filter(|(n, _)| wild(t, n)).map(|(_, h)| *h));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(hex) = t.strip_prefix("<").and_then(|r| r.strip_suffix(">"))
|
||||
&& let Ok(h) = u32::from_str_radix(hex, 16)
|
||||
{
|
||||
out.push(h);
|
||||
continue;
|
||||
}
|
||||
out.push(manifest.and_then(|m| m.iter().find(|(n, _)| n == t).map(|(_, h)| *h)).unwrap_or_else(|| crypto_hash(t)));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub const CRYPTO_LAYERS: [&str; 3] = ["CryptoObject", "CryptoMaterial", "CryptoAsset"];
|
||||
|
||||
fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let Some(a) = aux(ctx) else { return b };
|
||||
let layers = a.crypto_layers();
|
||||
if layers.is_empty() {
|
||||
return b;
|
||||
}
|
||||
let want = CRYPTO_LAYERS[(ctx.params.e("layer") as usize).min(2)];
|
||||
let layer = layers.iter().find(|l| l.as_str() == want || l.ends_with(want)).unwrap_or(&layers[0]).clone();
|
||||
let manifest = a.manifests.iter().find(|(l, _)| *l == layer).map(|(_, m)| m.as_slice());
|
||||
let sel = selection_hashes(ctx.params.s("selection"), manifest);
|
||||
let idx = index_map(&b, &a);
|
||||
let display = ctx.params.e("display");
|
||||
let matte_only = ctx.params.b("matteOnly");
|
||||
let src = b.img.clone();
|
||||
b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| {
|
||||
let ranks = idx[i].map(|k| a.crypto_ranks(&layer, k)).unwrap_or_default();
|
||||
let m: f32 = ranks.iter().filter(|(id, c)| *c > 0.0 && sel.contains(&id.to_bits())).map(|(_, c)| *c).sum::<f32>().clamp(0.0, 1.0);
|
||||
if matte_only {
|
||||
*px = [m, m, m, m];
|
||||
return;
|
||||
}
|
||||
*px = match display {
|
||||
// Colors: coverage-weighted ID colours (the spec's preview), selection brighter.
|
||||
0 => {
|
||||
let mut c = [0.0f32; 3];
|
||||
for (id, cov) in &ranks {
|
||||
let k = id_color(*id);
|
||||
for j in 0..3 {
|
||||
c[j] += k[j] * cov;
|
||||
}
|
||||
}
|
||||
let c = c.map(|v| v * (1.0 - 0.5 * m) + 0.5 * m);
|
||||
[c[0], c[1], c[2], 1.0]
|
||||
}
|
||||
// Matte: the layer keyed by the selection.
|
||||
1 => src.data[i].map(|v| v * m),
|
||||
// Preview: the layer with the selection tinted.
|
||||
_ => {
|
||||
let s = src.data[i];
|
||||
[s[0] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[1] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[2] * (1.0 - 0.5 * m), s[3]]
|
||||
}
|
||||
};
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- EXtractoR
|
||||
|
||||
fn extractor(ctx: &EffectCtx, mut b: Buf) -> Buf {
|
||||
let a = aux(ctx);
|
||||
let names = [ctx.params.s("red"), ctx.params.s("green"), ctx.params.s("blue"), ctx.params.s("alpha")];
|
||||
let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32);
|
||||
let unmult = ctx.params.b("unMult");
|
||||
let clip = ctx.params.b("clip");
|
||||
let idx = a.as_ref().map(|a| index_map(&b, a));
|
||||
// Each output channel: an aux channel by name, or the layer's own R/G/B/A.
|
||||
let own = |n: &str| match n.trim().to_ascii_uppercase().as_str() {
|
||||
"R" | "RED" => Some(0),
|
||||
"G" | "GREEN" => Some(1),
|
||||
"B" | "BLUE" => Some(2),
|
||||
"A" | "ALPHA" => Some(3),
|
||||
"" => None,
|
||||
_ => None,
|
||||
};
|
||||
let src_ch: Vec<Option<&[f32]>> = names.iter().map(|n| a.as_ref().and_then(|a| if n.trim().is_empty() { None } else { a.get(n.trim()) })).collect();
|
||||
if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(n) == Some(k)) {
|
||||
return b;
|
||||
}
|
||||
let src = b.img.clone();
|
||||
let lv = |v: f32| {
|
||||
let o = (v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum();
|
||||
if clip { o.clamp(0.0, 1.0) } else { o }
|
||||
};
|
||||
b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| {
|
||||
let (sc, sa) = unpremul(src.data[i]);
|
||||
let straight = [sc[0], sc[1], sc[2], sa];
|
||||
let mut o: Px = [0.0; 4];
|
||||
for k in 0..4 {
|
||||
o[k] = match (src_ch[k], idx.as_ref().and_then(|m| m[i])) {
|
||||
(Some(ch), Some(j)) => ch[j],
|
||||
(Some(_), None) => 0.0,
|
||||
(None, _) => match own(names[k]) {
|
||||
Some(c) => straight[c],
|
||||
None => {
|
||||
if k == 3 {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
if k < 3 {
|
||||
o[k] = lv(o[k]);
|
||||
}
|
||||
}
|
||||
let al = o[3].clamp(0.0, 1.0);
|
||||
let c = if unmult && al > 1e-6 { [o[0] / al, o[1] / al, o[2] / al] } else { [o[0], o[1], o[2]] };
|
||||
*px = premul(c, al);
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
pub fn specs() -> Vec<EffectSpec> {
|
||||
let text = |id: &'static str, name: &'static str, d: &str| p(id, name, Value::Str(d.into()), ParamUi::Text);
|
||||
vec![
|
||||
spec(
|
||||
"ec.3d.channelextract",
|
||||
"3D Channel Extract",
|
||||
vec![
|
||||
p("channel", "3D Channel", Value::Enum(0), popup(&EXTRACT_CHANNELS)),
|
||||
p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)),
|
||||
p("whitePoint", "White Point", num(10000.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)),
|
||||
],
|
||||
channel_extract,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.cryptomatte",
|
||||
"Cryptomatte",
|
||||
vec![
|
||||
p("layer", "Layer", Value::Enum(0), popup(&CRYPTO_LAYERS)),
|
||||
text("selection", "Selection", ""),
|
||||
p("display", "Display", Value::Enum(2), popup(&["Colors", "Matte", "Preview"])),
|
||||
p("matteOnly", "Matte Only", Value::Bool(false), ParamUi::Checkbox),
|
||||
],
|
||||
cryptomatte,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.depthmatte",
|
||||
"Depth Matte",
|
||||
vec![
|
||||
p("depth", "Depth", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)),
|
||||
p("feather", "Feather", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)),
|
||||
p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox),
|
||||
],
|
||||
depth_matte,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.depthoffield",
|
||||
"Depth of Field",
|
||||
vec![
|
||||
p("focalPlane", "Focal Plane", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)),
|
||||
p("maximumRadius", "Maximum Radius", num(0.0), slider(0.0, 500.0, 0.0, 50.0, 2)),
|
||||
p("focalPlaneThickness", "Focal Plane Thickness", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)),
|
||||
p("focalBias", "Focal Bias", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 2)),
|
||||
],
|
||||
depth_of_field,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.extractor",
|
||||
"EXtractoR",
|
||||
vec![
|
||||
text("red", "Red", "R"),
|
||||
text("green", "Green", "G"),
|
||||
text("blue", "Blue", "B"),
|
||||
text("alpha", "Alpha", "A"),
|
||||
p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)),
|
||||
p("whitePoint", "White Point", num(1.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)),
|
||||
p("unMult", "UnMult", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("clip", "Clip", Value::Bool(false), ParamUi::Checkbox),
|
||||
],
|
||||
extractor,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.fog3d",
|
||||
"Fog 3D",
|
||||
vec![
|
||||
p("fogColor", "Fog Color", col(1.0, 1.0, 1.0), ParamUi::Color),
|
||||
p("fogStartDepth", "Fog Start Depth", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)),
|
||||
p("fogEndDepth", "Fog End Depth", num(3000.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)),
|
||||
p("fogOpacity", "Fog Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)),
|
||||
p("scatteringDensity", "Scattering Density", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)),
|
||||
p("foggyBackground", "Foggy Background", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("gradientLayer", "Gradient Layer", Value::Layer(None), ParamUi::Layer),
|
||||
p("layerContribution", "Layer Contribution", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)),
|
||||
],
|
||||
fog_3d,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.idmatte",
|
||||
"ID Matte",
|
||||
vec![
|
||||
p("auxChannel", "Aux. Channel", Value::Enum(0), popup(&["Object ID", "Material ID"])),
|
||||
p("idSelection", "ID Selection", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)),
|
||||
p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 2)),
|
||||
p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("useCoverage", "Use Coverage", Value::Bool(false), ParamUi::Checkbox),
|
||||
],
|
||||
id_matte,
|
||||
),
|
||||
spec(
|
||||
"ec.3d.identifier",
|
||||
"IDentifier",
|
||||
vec![
|
||||
p("channelType", "Channel Type", Value::Enum(0), popup(&["Object ID", "Material ID"])),
|
||||
p("display", "Display", Value::Enum(0), popup(&["Colors", "Luma Matte", "Alpha Matte", "Raw"])),
|
||||
p("id", "ID", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)),
|
||||
],
|
||||
identifier,
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{EffectEnv, EffectHost, LayerPixels, run_fx};
|
||||
use effectcraft_raster::channels3d::crypto_float;
|
||||
|
||||
/// A 20×10 layer: left half object 1 at depth 100, right half object 2 at depth 500; the
|
||||
/// top-right corner is background. Cryptomatte: "left" / "right".
|
||||
struct AuxHost(Arc<AuxChannels>);
|
||||
impl EffectHost for AuxHost {
|
||||
fn layer(&self, _: u64, _: bool) -> Option<LayerPixels> {
|
||||
None
|
||||
}
|
||||
fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option<Vec<f32>> {
|
||||
None
|
||||
}
|
||||
fn aux(&self) -> Option<Arc<AuxChannels>> {
|
||||
Some(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn scene() -> AuxHost {
|
||||
let (w, h) = (20usize, 10usize);
|
||||
let mut a = AuxChannels::new(w as u32, h as u32, 1.0);
|
||||
let (mut z, mut id, mut r, mut g) = (vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h]);
|
||||
let (hl, hr) = (crypto_hash("left"), crypto_hash("right"));
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let i = y * w + x;
|
||||
if x >= 15 && y < 3 {
|
||||
z[i] = BACKGROUND_DEPTH;
|
||||
} else if x < 10 {
|
||||
z[i] = 100.0;
|
||||
id[i] = 1.0;
|
||||
r[i] = crypto_float(hl);
|
||||
g[i] = 1.0;
|
||||
} else {
|
||||
z[i] = 500.0;
|
||||
id[i] = 2.0;
|
||||
r[i] = crypto_float(hr);
|
||||
g[i] = 1.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
a.channels = vec![("Z".into(), z), ("ObjectID".into(), id), ("CryptoObject00.R".into(), r), ("CryptoObject00.G".into(), g)];
|
||||
a.manifests = vec![("CryptoObject".into(), vec![("left".into(), hl), ("right".into(), hr)])];
|
||||
AuxHost(Arc::new(a))
|
||||
}
|
||||
|
||||
fn img() -> Image {
|
||||
Image::filled(20, 10, [0.5, 0.5, 0.5, 1.0])
|
||||
}
|
||||
|
||||
fn run(id: &str, vals: &[(&str, Value)], host: &AuxHost) -> Image {
|
||||
run_fx(id, vals, img(), 0.0, EffectEnv { host: Some(host), ..Default::default() }).img
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passes_through_without_aux_channels() {
|
||||
for s in specs() {
|
||||
let out = run_fx(s.id, &[], img(), 0.0, EffectEnv::default());
|
||||
assert_eq!(out.img.data, img().data, "{}", s.id);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn depth_matte_thresholds() {
|
||||
let h = scene();
|
||||
let o = run("ec.3d.depthmatte", &[("depth", num(300.0))], &h);
|
||||
assert_eq!(o.get(2, 5)[3], 0.0);
|
||||
assert_eq!(o.get(12, 5)[3], 1.0);
|
||||
let inv = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("invert", Value::Bool(true))], &h);
|
||||
assert_eq!(inv.get(2, 5)[3], 1.0);
|
||||
assert_eq!(inv.get(12, 5)[3], 0.0);
|
||||
// Feather: a ramp of 800 units centred on 300 → depth 100 at 0.25, 500 at 0.75.
|
||||
let f = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("feather", num(800.0))], &h);
|
||||
assert!((f.get(2, 5)[3] - 0.25).abs() < 1e-5 && (f.get(12, 5)[3] - 0.75).abs() < 1e-5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn channel_extract_maps_depth_and_ids() {
|
||||
let h = scene();
|
||||
let o = run("ec.3d.channelextract", &[("blackPoint", num(0.0)), ("whitePoint", num(1000.0))], &h);
|
||||
assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6 && (o.get(12, 5)[0] - 0.5).abs() < 1e-6);
|
||||
assert_eq!(o.get(17, 1)[0], 1.0);
|
||||
let ids = run("ec.3d.channelextract", &[("channel", Value::Enum(1)), ("whitePoint", num(2.0))], &h);
|
||||
assert_eq!(ids.get(2, 5)[0], 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id_matte_and_identifier() {
|
||||
let h = scene();
|
||||
let o = run("ec.3d.idmatte", &[("idSelection", num(2.0))], &h);
|
||||
assert_eq!(o.get(2, 5)[3], 0.0);
|
||||
assert_eq!(o.get(12, 5)[3], 1.0);
|
||||
let l = run("ec.3d.identifier", &[("display", Value::Enum(1)), ("id", num(1.0))], &h);
|
||||
assert_eq!(l.get(2, 5)[0], 1.0);
|
||||
assert_eq!(l.get(12, 5)[0], 0.0);
|
||||
let c = run("ec.3d.identifier", &[], &h);
|
||||
assert_ne!(c.get(2, 5), c.get(12, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cryptomatte_selects_by_name_and_wildcard() {
|
||||
let h = scene();
|
||||
let m = run("ec.3d.cryptomatte", &[("selection", Value::Str("right".into())), ("matteOnly", Value::Bool(true))], &h);
|
||||
assert_eq!(m.get(2, 5)[3], 0.0);
|
||||
assert_eq!(m.get(12, 5)[3], 1.0);
|
||||
let w = run("ec.3d.cryptomatte", &[("selection", Value::Str("l*".into())), ("display", Value::Enum(1))], &h);
|
||||
assert_eq!(w.get(2, 5)[3], 1.0);
|
||||
assert_eq!(w.get(12, 5)[3], 0.0);
|
||||
assert!(wild("*ight", "right") && wild("r*t", "right") && !wild("x*", "right"));
|
||||
let hex = format!("<{:08x}>", crypto_hash("left"));
|
||||
assert_eq!(selection_hashes(&hex, None), vec![crypto_hash("left")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fog_and_depth_of_field_follow_depth() {
|
||||
let h = scene();
|
||||
let f = run("ec.3d.fog3d", &[("fogColor", col(1.0, 0.0, 0.0)), ("fogEndDepth", num(1000.0)), ("scatteringDensity", num(0.0))], &h);
|
||||
// Linear fog: 10 % at depth 100, 50 % at 500, full on the background.
|
||||
assert!((f.get(2, 5)[0] - 0.55).abs() < 1e-4, "{:?}", f.get(2, 5));
|
||||
assert!((f.get(12, 5)[0] - 0.75).abs() < 1e-4);
|
||||
assert!((f.get(17, 1)[0] - 1.0).abs() < 1e-4 && f.get(17, 1)[1] < 1e-4);
|
||||
assert!((dof_radius(100.0, 100.0, 0.0, 10.0, 50.0)).abs() < 1e-6);
|
||||
assert!((dof_radius(200.0, 100.0, 0.0, 10.0, 50.0) - 10.0).abs() < 1e-6);
|
||||
assert_eq!(dof_radius(140.0, 100.0, 100.0, 10.0, 50.0), 0.0);
|
||||
// A sharp edge at the far object blurs when focused near.
|
||||
let mut pic = Image::new(20, 10);
|
||||
for y in 0..10 {
|
||||
for x in 0..20 {
|
||||
let v = if x % 2 == 0 { 1.0 } else { 0.0 };
|
||||
pic.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
let env = EffectEnv { host: Some(&h), ..Default::default() };
|
||||
let d = run_fx("ec.3d.depthoffield", &[("focalPlane", num(100.0)), ("maximumRadius", num(4.0))], pic.clone(), 0.0, env).img;
|
||||
assert_eq!(d.get(4, 5), pic.get(4, 5));
|
||||
assert!((d.get(12, 5)[0] - 0.5).abs() < 0.3 && d.get(12, 5) != pic.get(12, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extractor_maps_named_channels() {
|
||||
let h = scene();
|
||||
let o = run("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("whitePoint", num(1000.0))], &h);
|
||||
assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6);
|
||||
assert!((o.get(2, 5)[1] - 0.0005).abs() < 1e-6);
|
||||
// Without aux: the layer's own channels can be swapped.
|
||||
let s = run_fx("ec.3d.extractor", &[("red", Value::Str("A".into()))], Image::filled(2, 2, [0.2, 0.3, 0.4, 1.0]), 0.0, EffectEnv::default());
|
||||
assert_eq!(s.img.get(0, 0), [1.0, 0.3, 0.4, 1.0]);
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ mod blur2;
|
||||
mod blur3;
|
||||
mod channel;
|
||||
mod channel2;
|
||||
mod channel3d;
|
||||
mod color2;
|
||||
mod color3;
|
||||
mod color_fx;
|
||||
@@ -57,7 +58,7 @@ pub use color2::Curve;
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::build::Ids;
|
||||
use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property};
|
||||
pub use effectcraft_raster::Image;
|
||||
pub use effectcraft_raster::{AuxChannels, Image};
|
||||
|
||||
/// Effect categories in Effects & Presets order.
|
||||
pub const CATEGORIES: &[&str] = &[
|
||||
@@ -238,6 +239,13 @@ pub trait EffectHost: Sync {
|
||||
fn layer_at(&self, _id: u64, _comp_time: f64, _masks_and_effects: bool) -> Option<LayerPixels> {
|
||||
None
|
||||
}
|
||||
/// Auxiliary 3D channels (depth, object / material IDs, normals, Cryptomatte, any named
|
||||
/// EXR channel) of the effect's own layer source at the current time, in layer space (see
|
||||
/// [`AuxChannels`]). `None` when the source has none (the 3D Channel effects then pass the
|
||||
/// layer through).
|
||||
fn aux(&self) -> Option<std::sync::Arc<AuxChannels>> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Extra context the renderer may supply (all optional; `Default` is "nothing known").
|
||||
@@ -366,6 +374,7 @@ pub fn registry() -> &'static [EffectSpec] {
|
||||
v.extend(keylight::specs());
|
||||
v.extend(mocha_shape::specs());
|
||||
v.extend(sim3::specs());
|
||||
v.extend(channel3d::specs());
|
||||
v.extend(time_fx::specs());
|
||||
v.extend(audio_fx::specs());
|
||||
v.extend(paint::specs());
|
||||
|
||||
@@ -19,7 +19,9 @@ filmcraft-media = { workspace = true }
|
||||
filmcraft-frame = { workspace = true }
|
||||
filmcraft-color = { workspace = true }
|
||||
filmcraft-time = { workspace = true }
|
||||
exr = { workspace = true }
|
||||
image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr"] }
|
||||
rayon = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Auxiliary channels from multi-layer OpenEXR files (depth, object / material IDs, normals,
|
||||
//! Cryptomatte, …) for the 3D Channel effects, read with the pure-Rust `exr` crate.
|
||||
//!
|
||||
//! Every channel of every layer becomes a plane named `layer.channel` (or just `channel` for
|
||||
//! the unnamed layer), placed in the display window (pixels outside a layer's data window are
|
||||
//! 0, or the background depth for depth channels). Cryptomatte manifests come from the
|
||||
//! `cryptomatte/<key>/name` and `cryptomatte/<key>/manifest` header attributes (the published
|
||||
//! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
use effectcraft_raster::AuxChannels;
|
||||
use effectcraft_raster::channels3d::BACKGROUND_DEPTH;
|
||||
|
||||
fn is_depth(name: &str) -> bool {
|
||||
let last = name.rsplit('.').next().unwrap_or(name);
|
||||
last.eq_ignore_ascii_case("z") || name.to_ascii_lowercase().contains("depth")
|
||||
}
|
||||
|
||||
/// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`).
|
||||
pub fn parse_manifest(json: &str) -> Vec<(String, u32)> {
|
||||
let Ok(serde_json::Value::Object(m)) = serde_json::from_str::<serde_json::Value>(json) else { return vec![] };
|
||||
let mut v: Vec<(String, u32)> = m.iter().filter_map(|(k, h)| Some((k.clone(), u32::from_str_radix(h.as_str()?.trim(), 16).ok()?))).collect();
|
||||
v.sort();
|
||||
v
|
||||
}
|
||||
|
||||
/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR.
|
||||
pub fn read_exr_channels(bytes: &[u8]) -> Option<AuxChannels> {
|
||||
use exr::prelude::*;
|
||||
let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?;
|
||||
let dw = img.attributes.display_window;
|
||||
let (w, h) = (dw.size.0, dw.size.1);
|
||||
if w == 0 || h == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut aux = AuxChannels::new(w as u32, h as u32, 1.0);
|
||||
let mut crypto: HashMap<String, (Option<String>, Option<String>)> = HashMap::new();
|
||||
let mut collect_meta = |other: &HashMap<Text, AttributeValue>| {
|
||||
for (k, v) in other {
|
||||
let key = k.to_string();
|
||||
let Some(rest) = key.strip_prefix("cryptomatte/") else { continue };
|
||||
let Some((id, field)) = rest.split_once('/') else { continue };
|
||||
let AttributeValue::Text(t) = v else { continue };
|
||||
let e = crypto.entry(id.to_string()).or_default();
|
||||
match field {
|
||||
"name" => e.0 = Some(t.to_string()),
|
||||
"manifest" => e.1 = Some(t.to_string()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
};
|
||||
collect_meta(&img.attributes.other);
|
||||
for layer in img.layer_data.iter() {
|
||||
collect_meta(&layer.attributes.other);
|
||||
let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty());
|
||||
let (lw, lh) = (layer.size.0, layer.size.1);
|
||||
let pos = layer.attributes.layer_position;
|
||||
let (ox, oy) = (pos.0 as i64 - dw.position.0 as i64, pos.1 as i64 - dw.position.1 as i64);
|
||||
for ch in layer.channel_data.list.iter() {
|
||||
let name = match &prefix {
|
||||
Some(p) => format!("{p}.{}", ch.name),
|
||||
None => ch.name.to_string(),
|
||||
};
|
||||
let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 };
|
||||
let mut plane = vec![fill; w * h];
|
||||
let vals: Vec<f32> = ch.sample_data.values_as_f32().collect();
|
||||
if vals.len() < lw * lh {
|
||||
continue;
|
||||
}
|
||||
for y in 0..lh {
|
||||
let ty = y as i64 + oy;
|
||||
if ty < 0 || ty >= h as i64 {
|
||||
continue;
|
||||
}
|
||||
for x in 0..lw {
|
||||
let tx = x as i64 + ox;
|
||||
if tx < 0 || tx >= w as i64 {
|
||||
continue;
|
||||
}
|
||||
plane[ty as usize * w + tx as usize] = vals[y * lw + x];
|
||||
}
|
||||
}
|
||||
aux.channels.push((name, plane));
|
||||
}
|
||||
}
|
||||
for (_, (name, manifest)) in crypto {
|
||||
if let (Some(n), Some(m)) = (name, manifest) {
|
||||
aux.manifests.push((n, parse_manifest(&m)));
|
||||
}
|
||||
}
|
||||
aux.manifests.sort();
|
||||
Some(aux)
|
||||
}
|
||||
|
||||
type Cache = Mutex<Vec<(String, Option<Arc<AuxChannels>>)>>;
|
||||
|
||||
/// Read (and cache, a few files) the channels of the EXR at `path` via `read`.
|
||||
pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option<Arc<[u8]>>) -> Option<Arc<AuxChannels>> {
|
||||
static CACHE: OnceLock<Cache> = OnceLock::new();
|
||||
let cache = CACHE.get_or_init(|| Mutex::new(Vec::new()));
|
||||
if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) {
|
||||
return v;
|
||||
}
|
||||
let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new);
|
||||
if let Ok(mut c) = cache.lock() {
|
||||
if c.len() >= 4 {
|
||||
c.remove(0);
|
||||
}
|
||||
c.push((path.to_string(), v.clone()));
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reads_layered_channels_and_cryptomatte_metadata() {
|
||||
use exr::prelude::*;
|
||||
let (w, h) = (4usize, 2usize);
|
||||
let z: Vec<f32> = (0..w * h).map(|i| i as f32 * 10.0).collect();
|
||||
let id: Vec<f32> = (0..w * h).map(|i| (i % 2) as f32 + 1.0).collect();
|
||||
let r: Vec<f32> = vec![0.5; w * h];
|
||||
let channels = AnyChannels::sort(
|
||||
vec![
|
||||
AnyChannel::new("R", FlatSamples::F32(r)),
|
||||
AnyChannel::new("Z", FlatSamples::F32(z.clone())),
|
||||
AnyChannel::new("ObjectID", FlatSamples::F32(id.clone())),
|
||||
]
|
||||
.into(),
|
||||
);
|
||||
let mut layer = Layer::new((w, h), LayerAttributes::named("depth"), Encoding::FAST_LOSSLESS, channels);
|
||||
layer.attributes.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject")));
|
||||
layer
|
||||
.attributes
|
||||
.other
|
||||
.insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#)));
|
||||
let image = Image::from_layer(layer);
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
image.write().to_buffered(&mut bytes).unwrap();
|
||||
let aux = read_exr_channels(bytes.get_ref()).unwrap();
|
||||
assert_eq!((aux.width, aux.height), (4, 2));
|
||||
assert_eq!(aux.get("depth.Z").unwrap(), z.as_slice());
|
||||
assert_eq!(aux.depth().unwrap(), z.as_slice());
|
||||
assert_eq!(aux.object_id().unwrap(), id.as_slice());
|
||||
assert_eq!(aux.manifests, vec![("CryptoObject".to_string(), vec![("bunny".to_string(), 0x1385_1a76), ("floor".to_string(), 0x0a1b_2c3d)])]);
|
||||
assert!(read_exr_channels(b"not an exr").is_none());
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR).
|
||||
|
||||
mod convert;
|
||||
pub mod exr_channels;
|
||||
mod pool;
|
||||
mod probe;
|
||||
|
||||
|
||||
@@ -470,6 +470,18 @@ impl FootageSource for MediaPool {
|
||||
}
|
||||
}
|
||||
|
||||
fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
if footage.missing || !footage.has_video {
|
||||
return None;
|
||||
}
|
||||
let loc = Inner::locate(footage, t);
|
||||
if loc.media_t.is_some() || !loc.key.path.to_ascii_lowercase().ends_with(".exr") {
|
||||
return None;
|
||||
}
|
||||
let path = loc.key.path.clone();
|
||||
crate::exr_channels::cached(&path, || self.inner.read(&path).ok())
|
||||
}
|
||||
|
||||
fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option<Vec<f32>> {
|
||||
if !footage.has_audio || footage.missing {
|
||||
return None;
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
//! Auxiliary (3D) channels that travel with a layer's pixels: depth, object / material IDs,
|
||||
//! surface normals, texture UVs, coverage and Cryptomatte ID/coverage pairs.
|
||||
//!
|
||||
//! They come from multi-layer OpenEXR footage (any channel by its full `layer.channel` name)
|
||||
//! or from the compositor's own render passes for precomps of 3D comps. The 3D Channel effects
|
||||
//! read them through `EffectHost::aux`.
|
||||
//!
|
||||
//! Conventions (shared by the EXR reader and the render pass):
|
||||
//!
|
||||
//! * channel planes are row-major `width × height` `f32`s covering the layer's source rectangle,
|
||||
//! with `scale` aux pixels per layer pixel;
|
||||
//! * depth is camera-space distance in pixels (`Z`), background `BACKGROUND_DEPTH`;
|
||||
//! * `ObjectID` / `MaterialID` are small integers (0 = background);
|
||||
//! * Cryptomatte layers follow the published Cryptomatte specification (Psyop, 2015): a layer
|
||||
//! named `CryptoObject` stores rank pairs in `CryptoObject00.R/G/B/A`, `CryptoObject01.R/…`
|
||||
//! (ID float, coverage, ID float, coverage, …), IDs are MurmurHash3 (x86, 32-bit, seed 0) of
|
||||
//! the object name reinterpreted as a float with the exponent nudged off 0 and 255, and the
|
||||
//! manifest maps names to hashes.
|
||||
|
||||
/// Depth written where nothing was rendered.
|
||||
pub const BACKGROUND_DEPTH: f32 = 1.0e10;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct AuxChannels {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Aux pixels per layer pixel.
|
||||
pub scale: f64,
|
||||
/// Named planes (`Z`, `ObjectID`, `CryptoObject00.R`, …).
|
||||
pub channels: Vec<(String, Vec<f32>)>,
|
||||
/// Cryptomatte manifests: (layer name, [(object name, hash)]).
|
||||
pub manifests: Vec<(String, Vec<(String, u32)>)>,
|
||||
}
|
||||
|
||||
impl AuxChannels {
|
||||
pub fn new(width: u32, height: u32, scale: f64) -> AuxChannels {
|
||||
AuxChannels { width, height, scale, channels: vec![], manifests: vec![] }
|
||||
}
|
||||
|
||||
/// A channel by exact name (case-insensitive).
|
||||
pub fn get(&self, name: &str) -> Option<&[f32]> {
|
||||
self.channels.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_slice())
|
||||
}
|
||||
|
||||
/// The first channel present among `names` (also matching `*.name` suffixes, so `Z` finds
|
||||
/// `depth.Z`).
|
||||
pub fn find(&self, names: &[&str]) -> Option<&[f32]> {
|
||||
for n in names {
|
||||
if let Some(v) = self.get(n) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
for n in names {
|
||||
let suffix = format!(".{}", n.to_ascii_lowercase());
|
||||
if let Some((_, v)) = self.channels.iter().find(|(c, _)| c.to_ascii_lowercase().ends_with(&suffix)) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn names(&self) -> Vec<&str> {
|
||||
self.channels.iter().map(|(n, _)| n.as_str()).collect()
|
||||
}
|
||||
|
||||
pub fn depth(&self) -> Option<&[f32]> {
|
||||
self.find(&["Z", "depth", "Depth", "zDepth", "Z-Depth"])
|
||||
}
|
||||
pub fn object_id(&self) -> Option<&[f32]> {
|
||||
self.find(&["ObjectID", "object_id", "objectId", "id"])
|
||||
}
|
||||
pub fn material_id(&self) -> Option<&[f32]> {
|
||||
self.find(&["MaterialID", "material_id", "materialId"])
|
||||
}
|
||||
pub fn coverage(&self) -> Option<&[f32]> {
|
||||
self.find(&["Coverage", "coverage"])
|
||||
}
|
||||
|
||||
/// Cryptomatte layer names present (prefixes with a `00.R` channel).
|
||||
pub fn crypto_layers(&self) -> Vec<String> {
|
||||
let mut v: Vec<String> =
|
||||
self.channels.iter().filter_map(|(n, _)| n.strip_suffix("00.R").or_else(|| n.strip_suffix("00.r")).map(str::to_string)).collect();
|
||||
v.sort();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
|
||||
/// Rank pairs (id, coverage) of Cryptomatte layer `layer` at aux pixel index `i`.
|
||||
pub fn crypto_ranks(&self, layer: &str, i: usize) -> Vec<(f32, f32)> {
|
||||
let mut out = vec![];
|
||||
for k in 0.. {
|
||||
let (r, g, b, a) = (
|
||||
self.get(&format!("{layer}{k:02}.R")),
|
||||
self.get(&format!("{layer}{k:02}.G")),
|
||||
self.get(&format!("{layer}{k:02}.B")),
|
||||
self.get(&format!("{layer}{k:02}.A")),
|
||||
);
|
||||
let (Some(r), Some(g)) = (r, g) else { break };
|
||||
out.push((r[i], g[i]));
|
||||
if let (Some(b), Some(a)) = (b, a) {
|
||||
out.push((b[i], a[i]));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Aux pixel index of a layer-space point (nearest; `None` outside).
|
||||
#[inline]
|
||||
pub fn index_at(&self, lx: f64, ly: f64) -> Option<usize> {
|
||||
let x = (lx * self.scale).floor();
|
||||
let y = (ly * self.scale).floor();
|
||||
if x < 0.0 || y < 0.0 || x >= self.width as f64 || y >= self.height as f64 {
|
||||
return None;
|
||||
}
|
||||
Some(y as usize * self.width as usize + x as usize)
|
||||
}
|
||||
|
||||
/// Manifest entry name for a hash in Cryptomatte layer `layer`.
|
||||
pub fn manifest_name(&self, layer: &str, hash: u32) -> Option<&str> {
|
||||
self.manifests.iter().find(|(l, _)| l == layer)?.1.iter().find(|(_, h)| *h == hash).map(|(n, _)| n.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// MurmurHash3 x86 32-bit (public-domain algorithm by Austin Appleby).
|
||||
pub fn murmur3_32(data: &[u8], seed: u32) -> u32 {
|
||||
const C1: u32 = 0xcc9e_2d51;
|
||||
const C2: u32 = 0x1b87_3593;
|
||||
let mut h = seed;
|
||||
let chunks = data.chunks_exact(4);
|
||||
let tail = chunks.remainder();
|
||||
for c in chunks {
|
||||
let mut k = u32::from_le_bytes([c[0], c[1], c[2], c[3]]);
|
||||
k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2);
|
||||
h ^= k;
|
||||
h = h.rotate_left(13).wrapping_mul(5).wrapping_add(0xe654_6b64);
|
||||
}
|
||||
let mut k = 0u32;
|
||||
for (i, b) in tail.iter().enumerate() {
|
||||
k ^= (*b as u32) << (8 * i);
|
||||
}
|
||||
if !tail.is_empty() {
|
||||
k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2);
|
||||
h ^= k;
|
||||
}
|
||||
h ^= data.len() as u32;
|
||||
h ^= h >> 16;
|
||||
h = h.wrapping_mul(0x85eb_ca6b);
|
||||
h ^= h >> 13;
|
||||
h = h.wrapping_mul(0xc2b2_ae35);
|
||||
h ^= h >> 16;
|
||||
h
|
||||
}
|
||||
|
||||
/// Cryptomatte hash of a name (the manifest value).
|
||||
pub fn crypto_hash(name: &str) -> u32 {
|
||||
let h = murmur3_32(name.as_bytes(), 0);
|
||||
// Keep the float finite and normal: flip the low exponent bit when it is all 0s or 1s.
|
||||
let e = (h >> 23) & 0xff;
|
||||
if e == 0 || e == 255 { h ^ (1 << 23) } else { h }
|
||||
}
|
||||
|
||||
/// The ID float stored in Cryptomatte channels for a hash.
|
||||
pub fn crypto_float(hash: u32) -> f32 {
|
||||
f32::from_bits(hash)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn murmur3_reference_values() {
|
||||
// Published MurmurHash3_x86_32 test vectors.
|
||||
assert_eq!(murmur3_32(b"", 0), 0);
|
||||
assert_eq!(murmur3_32(b"", 1), 0x514e_28b7);
|
||||
assert_eq!(murmur3_32(b"", 0xffff_ffff), 0x81f1_6f39);
|
||||
assert_eq!(murmur3_32(&[0, 0, 0, 0], 0), 0x2362_f9de);
|
||||
assert_eq!(murmur3_32(b"a", 0x9747_b28c), 0x7fa0_9ea6);
|
||||
assert_eq!(murmur3_32(b"aaaa", 0x9747_b28c), 0x5a97_808a);
|
||||
assert_eq!(murmur3_32(b"Hello, world!", 0x9747_b28c), 0x2488_4cba);
|
||||
assert_eq!(murmur3_32(b"The quick brown fox jumps over the lazy dog", 0x9747_b28c), 0x2fa8_26cd);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crypto_floats_are_normal_and_round_trip() {
|
||||
for n in ["bunny", "set", "ground", "a", ""] {
|
||||
let h = crypto_hash(n);
|
||||
let f = crypto_float(h);
|
||||
assert!(f.is_finite() && (f == 0.0 || f.is_normal() || f.abs() > 0.0), "{n}");
|
||||
assert_eq!(f.to_bits(), h);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookup_helpers() {
|
||||
let mut a = AuxChannels::new(2, 1, 1.0);
|
||||
a.channels.push(("depth.Z".into(), vec![5.0, 6.0]));
|
||||
a.channels.push(("CryptoObject00.R".into(), vec![1.0, 2.0]));
|
||||
a.channels.push(("CryptoObject00.G".into(), vec![0.5, 1.0]));
|
||||
assert_eq!(a.depth(), Some(&[5.0, 6.0][..]));
|
||||
assert_eq!(a.crypto_layers(), vec!["CryptoObject".to_string()]);
|
||||
assert_eq!(a.crypto_ranks("CryptoObject", 0), vec![(1.0, 0.5)]);
|
||||
assert_eq!(a.index_at(1.5, 0.2), Some(1));
|
||||
assert_eq!(a.index_at(2.5, 0.2), None);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
//! heavy operation is row-parallel with rayon.
|
||||
|
||||
pub mod blur;
|
||||
pub mod channels3d;
|
||||
pub mod flow;
|
||||
pub mod warp;
|
||||
|
||||
@@ -11,6 +12,7 @@ use effectcraft_color::{BlendMode, blend_pixel};
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub use blur::{box_blur, directional_blur, gaussian_blur, radial_blur};
|
||||
pub use channels3d::AuxChannels;
|
||||
pub use warp::{Sampling, WarpOpts, accumulate_warp, composite_warp, composite_warp_reference, resample};
|
||||
|
||||
pub type Px = [f32; 4];
|
||||
|
||||
@@ -39,6 +39,12 @@ pub trait FootageSource: Send + Sync {
|
||||
fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option<Vec<f32>> {
|
||||
None
|
||||
}
|
||||
/// Auxiliary 3D channels of `item` at source time `t` (multi-layer OpenEXR: depth, IDs,
|
||||
/// Cryptomatte, any named channel), with `scale` relative to the file's pixels. `None` when
|
||||
/// the footage has none.
|
||||
fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
None
|
||||
}
|
||||
/// Set the decoded-frame cache budget in bytes (Settings ▸ Memory & CPU); sources without
|
||||
/// a cache ignore it.
|
||||
fn set_cache_budget(&self, _bytes: usize) {}
|
||||
@@ -107,6 +113,29 @@ impl EffectHost for FxHost<'_, '_, '_> {
|
||||
sub.layer_input(&self.ctx.at(t), self.layer, n).map(|b| (*b).clone())
|
||||
}
|
||||
|
||||
fn aux(&self) -> Option<Arc<effectcraft_raster::AuxChannels>> {
|
||||
match &self.layer.source {
|
||||
LayerSource::Footage { item } => {
|
||||
let ItemKind::Footage(f) = &self.r.project.item(*item)?.kind else { return None };
|
||||
let a = self.r.footage.aux(*item, f, self.ctx.source_time(self.layer))?;
|
||||
// Aux pixels per layer pixel (the layer is the footage at its nominal size).
|
||||
if f.width > 0 && a.width > 0 {
|
||||
let mut a = (*a).clone();
|
||||
a.scale = a.width as f64 / f.width as f64;
|
||||
return Some(Arc::new(a));
|
||||
}
|
||||
Some(a)
|
||||
}
|
||||
LayerSource::Comp { item } => {
|
||||
if self.r.depth > MAX_FX_DEPTH || self.r.project.comp_contains(*item, self.ctx.comp_id) {
|
||||
return None;
|
||||
}
|
||||
self.r.nested().comp_aux(*item, self.ctx.source_time(self.layer)).map(Arc::new)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn layer_at(&self, id: u64, comp_time: f64, masks_and_effects: bool) -> Option<LayerPixels> {
|
||||
let other = self.ctx.layer(effectcraft_project::LayerId(id))?;
|
||||
if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH {
|
||||
@@ -1175,6 +1204,13 @@ impl<'a> Renderer<'a> {
|
||||
self.depth
|
||||
}
|
||||
|
||||
/// The auxiliary 3D channels of comp `comp_id` at comp time `t` (depth, layer IDs, normals,
|
||||
/// UVs, Cryptomatte; see `three_d::compose::aux_pass`), at the renderer's output scale.
|
||||
pub fn comp_aux(&self, comp_id: ItemId, t: Tick) -> Option<effectcraft_raster::AuxChannels> {
|
||||
let ctx = self.eval_ctx(comp_id, t)?;
|
||||
Some(three_d::compose::aux_pass(self, &ctx))
|
||||
}
|
||||
|
||||
/// Evaluation context of comp `comp_id` at comp time `t`.
|
||||
pub fn eval_ctx(&self, comp_id: ItemId, t: Tick) -> Option<EvalCtx<'a>> {
|
||||
Some(self.ctx(comp_id, self.project.comp(comp_id)?, t))
|
||||
@@ -1295,6 +1331,8 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath {
|
||||
#[cfg(test)]
|
||||
mod tests_audio_fx;
|
||||
#[cfg(test)]
|
||||
mod tests_aux;
|
||||
#[cfg(test)]
|
||||
mod tests_collapse;
|
||||
#[cfg(test)]
|
||||
mod tests_color;
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
//! Auxiliary 3D channels: the comp's depth / ID / Cryptomatte pass and the 3D Channel effects on
|
||||
//! a precomp of a 3D comp.
|
||||
|
||||
use effectcraft_color::Label;
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::build::{self, Ids};
|
||||
use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project, Solid};
|
||||
use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash};
|
||||
use effectcraft_time::{FrameRate, Tick};
|
||||
|
||||
use crate::{NoFootage, RenderOpts, Renderer, render_frame};
|
||||
|
||||
const W: u32 = 200;
|
||||
const H: u32 = 100;
|
||||
|
||||
fn zoom() -> f64 {
|
||||
W as f64 * 50.0 / 36.0
|
||||
}
|
||||
|
||||
fn solid3(p: &mut Project, comp: &Comp, name: &str, color: [f32; 3], pos: [f64; 3]) -> Layer {
|
||||
let sid = p.add_item(name, Label::Red, None, ItemKind::Solid(Solid { color, width: 60, height: 60, pixel_aspect: 1.0 }));
|
||||
let mut l = build::layer(p, comp, name, LayerSource::Solid { item: sid }, (60, 60), None);
|
||||
l.name = name.to_string();
|
||||
l.switches.three_d = true;
|
||||
l.props.prop_mut("transform/position").unwrap().value = Value::Vec3(pos);
|
||||
l
|
||||
}
|
||||
|
||||
/// Inner 3D comp: "Near" (red, z 0, left) above "Far" (green, z 500, right). Outer comp: the
|
||||
/// precomp layer, optionally with an effect.
|
||||
fn scene(effect: Option<(&str, Vec<(&str, Value)>)>) -> (Project, ItemId, ItemId) {
|
||||
let mut p = Project::default();
|
||||
p.settings.bit_depth = effectcraft_project::BitDepth::Bpc32;
|
||||
let inner = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0));
|
||||
let iid = p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into()));
|
||||
let near = solid3(&mut p, &inner, "Near", [1.0, 0.0, 0.0], [60.0, 50.0, 0.0]);
|
||||
let far = solid3(&mut p, &inner, "Far", [0.0, 1.0, 0.0], [140.0, 50.0, 500.0]);
|
||||
p.comp_mut(iid).unwrap().layers.push(near);
|
||||
p.comp_mut(iid).unwrap().layers.push(far);
|
||||
let outer = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0));
|
||||
let oid = p.add_item("Outer", Label::Sandstone, None, ItemKind::Comp(outer.clone().into()));
|
||||
let mut pre = build::layer(&mut p, &outer, "Inner", LayerSource::Comp { item: iid }, (W, H), None);
|
||||
if let Some((id, vals)) = effect {
|
||||
let spec = effectcraft_effects::find(id).unwrap();
|
||||
let mut next = p.next_id;
|
||||
let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [W as f64, H as f64]);
|
||||
p.next_id = next;
|
||||
for (k, v) in vals {
|
||||
g.prop_mut(k).unwrap().value = v.clone();
|
||||
}
|
||||
pre.props.sub_mut("effects").unwrap().children.push(g.into());
|
||||
}
|
||||
p.comp_mut(oid).unwrap().layers.push(pre);
|
||||
(p, iid, oid)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comp_aux_pass_has_depth_ids_and_cryptomatte() {
|
||||
let (p, iid, _) = scene(None);
|
||||
let r = Renderer::new(&p, &NoFootage, RenderOpts::default());
|
||||
let a = r.comp_aux(iid, Tick::ZERO).unwrap();
|
||||
assert_eq!((a.width, a.height), (W, H));
|
||||
let at = |name: &str, x: usize, y: usize| a.get(name).unwrap()[y * W as usize + x];
|
||||
// Near plane at the camera's zoom distance, the far one 500 px behind it.
|
||||
assert!((at("Z", 60, 50) as f64 - zoom()).abs() < 1e-2, "{}", at("Z", 60, 50));
|
||||
assert!((at("Z", 114, 50) as f64 - (zoom() + 500.0)).abs() < 1e-2, "{}", at("Z", 114, 50));
|
||||
assert_eq!(at("Z", 5, 5), BACKGROUND_DEPTH);
|
||||
assert_eq!((at("ObjectID", 60, 50), at("ObjectID", 114, 50), at("ObjectID", 5, 5)), (1.0, 2.0, 0.0));
|
||||
assert_eq!(at("Coverage", 60, 50), 1.0);
|
||||
// Planes face the camera.
|
||||
assert!((at("N.Z", 60, 50) + 1.0).abs() < 1e-5);
|
||||
// Cryptomatte rank 0 = the layer under the pixel, full coverage; manifest lists names.
|
||||
assert_eq!(at("CryptoObject00.R", 60, 50).to_bits(), crypto_hash("Near"));
|
||||
assert_eq!(at("CryptoObject00.G", 60, 50), 1.0);
|
||||
assert_eq!(a.manifest_name("CryptoObject", crypto_hash("Far")), Some("Far"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn three_d_channel_effects_work_on_a_precomp_of_a_3d_comp() {
|
||||
let near = (60, 50);
|
||||
let far = (114, 50);
|
||||
let alpha = |p: &Project, oid: ItemId, (x, y): (i64, i64)| render_frame(p, oid, Tick::ZERO, 1.0).get(x, y)[3];
|
||||
let (p, _, oid) = scene(Some(("ec.3d.idmatte", vec![("idSelection", Value::Scalar(2.0))])));
|
||||
assert_eq!(alpha(&p, oid, near), 0.0);
|
||||
assert!(alpha(&p, oid, far) > 0.99);
|
||||
let (p, _, oid) = scene(Some(("ec.3d.depthmatte", vec![("depth", Value::Scalar(zoom() + 250.0))])));
|
||||
assert_eq!(alpha(&p, oid, near), 0.0);
|
||||
assert!(alpha(&p, oid, far) > 0.99);
|
||||
let (p, _, oid) = scene(Some(("ec.3d.cryptomatte", vec![("selection", Value::Str("Near".into())), ("display", Value::Enum(1))])));
|
||||
assert!(alpha(&p, oid, near) > 0.99);
|
||||
assert_eq!(alpha(&p, oid, far), 0.0);
|
||||
// Half resolution: the pass follows the render scale.
|
||||
let (p, iid, _) = scene(None);
|
||||
let r = Renderer::new(&p, &NoFootage, RenderOpts { scale: 0.5, ..Default::default() });
|
||||
let a = r.comp_aux(iid, Tick::ZERO).unwrap();
|
||||
assert_eq!((a.width, a.height, a.scale), (W / 2, H / 2, 0.5));
|
||||
assert_eq!(a.get("ObjectID").unwrap()[25 * 100 + 30], 1.0);
|
||||
}
|
||||
@@ -583,3 +583,182 @@ fn occlusion(c: &Item, p: Vec3, l: &LightState) -> [f32; 3] {
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ auxiliary render pass
|
||||
|
||||
/// One surface seen through an output pixel (aux pass).
|
||||
#[derive(Clone, Copy)]
|
||||
struct AuxFrag {
|
||||
z: f32,
|
||||
a: f32,
|
||||
obj: f32,
|
||||
mat: f32,
|
||||
n: [f32; 3],
|
||||
uv: [f32; 2],
|
||||
obj_hash: u32,
|
||||
mat_hash: u32,
|
||||
}
|
||||
|
||||
/// The comp's auxiliary channels at the context time, at output resolution: camera-space depth
|
||||
/// (`Z`), `ObjectID` (layer number), `MaterialID` (source item id), `Coverage`, camera-space
|
||||
/// normals (`N.X/Y/Z`), texture coordinates (`UV.U/V`) and Cryptomatte `CryptoObject` (layer
|
||||
/// names) / `CryptoMaterial` (source names) with two ranks each, plus their manifests.
|
||||
///
|
||||
/// Layers are gathered in stack order (top first); a run of 3D layers contributes its planes
|
||||
/// sorted near → far at each pixel, a 2D layer its coverage on the plane of the comp. The
|
||||
/// front-most surface with ≥ 50 % opacity (else the strongest one) provides depth, IDs, normal
|
||||
/// and UV; Cryptomatte coverage is each surface's visible share (`α · Π(1 − α_front)`).
|
||||
pub(crate) fn aux_pass(r: &Renderer, ctx: &EvalCtx) -> effectcraft_raster::AuxChannels {
|
||||
use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_float, crypto_hash};
|
||||
let s = r.opts.scale;
|
||||
let (w, h) = (((ctx.comp.width as f64 * s).round() as u32).max(1), ((ctx.comp.height as f64 * s).round() as u32).max(1));
|
||||
let out = (w, h);
|
||||
let (wu, n) = (w as usize, w as usize * h as usize);
|
||||
let cam = camera_for(r, ctx);
|
||||
let visible: Vec<&Layer> = r.visible_layers(ctx).into_iter().filter(|l| !l.switches.adjustment).collect();
|
||||
let number = |l: &Layer| ctx.comp.layers.iter().position(|x| x.id == l.id).map_or(0.0, |i| i as f32 + 1.0);
|
||||
let source_name = |l: &Layer| l.source.item().and_then(|i| r.project.item(i)).map(|it| it.name.clone()).unwrap_or_else(|| l.name.clone());
|
||||
let source_id = |l: &Layer| l.source.item().map_or(0.0, |i| i.0 as f32);
|
||||
let mut manifest_obj: Vec<(String, u32)> = vec![];
|
||||
let mut manifest_mat: Vec<(String, u32)> = vec![];
|
||||
for l in &visible {
|
||||
let (on, mn) = (l.name.clone(), source_name(l));
|
||||
if !manifest_obj.iter().any(|(x, _)| *x == on) {
|
||||
manifest_obj.push((on.clone(), crypto_hash(&on)));
|
||||
}
|
||||
if !manifest_mat.iter().any(|(x, _)| *x == mn) {
|
||||
manifest_mat.push((mn.clone(), crypto_hash(&mn)));
|
||||
}
|
||||
}
|
||||
// Groups in stack order: runs of 3D layers, single 2D layers.
|
||||
let mut groups: Vec<&[&Layer]> = vec![];
|
||||
let mut i = 0;
|
||||
while i < visible.len() {
|
||||
let mut j = i + 1;
|
||||
if visible[i].is_3d() {
|
||||
while j < visible.len() && visible[j].is_3d() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
groups.push(&visible[i..j]);
|
||||
i = j;
|
||||
}
|
||||
let mut frags: Vec<Vec<AuxFrag>> = vec![Vec::new(); n];
|
||||
let plane_z = cam.depth(vec3(ctx.comp.width as f64 * 0.5, ctx.comp.height as f64 * 0.5, 0.0)) as f32;
|
||||
for g in groups.iter().rev() {
|
||||
if g[0].is_3d() {
|
||||
let items: Vec<Item> = g.iter().enumerate().flat_map(|(k, l)| prepare_all(r, ctx, l, k, out, true)).collect();
|
||||
let meta: Vec<(f32, f32, u32, u32, [f32; 3], [f64; 2])> = items
|
||||
.iter()
|
||||
.map(|it| {
|
||||
let nrm = it.geos.first().map_or(vec3(0.0, 0.0, -1.0), |gm| cam.view.apply_vec(gm.normal));
|
||||
let mut nv = [nrm.x as f32, nrm.y as f32, nrm.z as f32];
|
||||
if nv[2] > 0.0 {
|
||||
nv = nv.map(|v| -v);
|
||||
}
|
||||
let (sw, sh) = crate::source_size(r.project, it.layer);
|
||||
let size = if sw == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [sw as f64, sh as f64] };
|
||||
(number(it.layer), source_id(it.layer), crypto_hash(&it.layer.name), crypto_hash(&source_name(it.layer)), nv, size)
|
||||
})
|
||||
.collect();
|
||||
frags.par_chunks_mut(wu).enumerate().for_each(|(y, row)| {
|
||||
let yi = y as i64;
|
||||
let mut local: Vec<AuxFrag> = Vec::new();
|
||||
for (x, cell) in row.iter_mut().enumerate() {
|
||||
let xi = x as i64;
|
||||
local.clear();
|
||||
for (it, m) in items.iter().zip(&meta) {
|
||||
if xi < it.bbox[0] || xi >= it.bbox[2] || yi < it.bbox[1] || yi >= it.bbox[3] {
|
||||
continue;
|
||||
}
|
||||
let Some(gm) = it.geos.first() else { continue };
|
||||
let Some((u, v, z)) = unproject(gm, x as f64 + 0.5, y as f64 + 0.5) else { continue };
|
||||
let a = it.texel(u, v)[3] * it.opacity;
|
||||
if a > 1e-4 {
|
||||
let uv = [(u / m.5[0]) as f32, (v / m.5[1]) as f32];
|
||||
local.push(AuxFrag { z: z as f32, a: a.min(1.0), obj: m.0, mat: m.1, n: m.4, uv, obj_hash: m.2, mat_hash: m.3 });
|
||||
}
|
||||
}
|
||||
local.sort_by(|p, q| p.z.total_cmp(&q.z));
|
||||
cell.extend_from_slice(&local);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
let l = g[0];
|
||||
let mut img = Image::new(w, h);
|
||||
if let Some(b) = r.layer_buf(ctx, l) {
|
||||
r.place(ctx, l, &b, &mut img, BlendMode::Normal, ctx.opacity(l) as f32);
|
||||
}
|
||||
let (obj, mat, oh, mh) = (number(l), source_id(l), crypto_hash(&l.name), crypto_hash(&source_name(l)));
|
||||
frags.par_chunks_mut(wu).zip(img.data.par_chunks(wu)).enumerate().for_each(|(y, (row, px))| {
|
||||
for (x, (cell, p)) in row.iter_mut().zip(px).enumerate() {
|
||||
if p[3] > 1e-4 {
|
||||
let uv = [(x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32];
|
||||
cell.push(AuxFrag { z: plane_z, a: p[3].min(1.0), obj, mat, n: [0.0, 0.0, -1.0], uv, obj_hash: oh, mat_hash: mh });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
// Resolve per pixel.
|
||||
const NAMES: [&str; 9] = ["Z", "ObjectID", "MaterialID", "Coverage", "N.X", "N.Y", "N.Z", "UV.U", "UV.V"];
|
||||
const CRYPTO: usize = 16;
|
||||
let resolved: Vec<([f32; 9], [f32; CRYPTO])> = frags
|
||||
.par_iter()
|
||||
.map(|fs| {
|
||||
let mut o = [BACKGROUND_DEPTH, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0];
|
||||
let mut crypto = [0.0f32; CRYPTO];
|
||||
if fs.is_empty() {
|
||||
return (o, crypto);
|
||||
}
|
||||
let mut t = 1.0f32;
|
||||
let mut obj: Vec<(u32, f32)> = vec![];
|
||||
let mut mat: Vec<(u32, f32)> = vec![];
|
||||
let mut best: Option<(usize, f32)> = None;
|
||||
let mut front: Option<usize> = None;
|
||||
for (k, f) in fs.iter().enumerate() {
|
||||
let c = f.a * t;
|
||||
if front.is_none() && f.a >= 0.5 {
|
||||
front = Some(k);
|
||||
}
|
||||
if best.is_none_or(|(_, bc)| c > bc) {
|
||||
best = Some((k, c));
|
||||
}
|
||||
for (list, hsh) in [(&mut obj, f.obj_hash), (&mut mat, f.mat_hash)] {
|
||||
match list.iter_mut().find(|(x, _)| *x == hsh) {
|
||||
Some(e) => e.1 += c,
|
||||
None => list.push((hsh, c)),
|
||||
}
|
||||
}
|
||||
t *= 1.0 - f.a;
|
||||
if t <= 1e-5 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let f = fs[front.or(best.map(|b| b.0)).unwrap_or(0)];
|
||||
o = [f.z, f.obj, f.mat, 1.0 - t, f.n[0], f.n[1], f.n[2], f.uv[0], f.uv[1]];
|
||||
for (base, list) in [(0usize, &mut obj), (8usize, &mut mat)] {
|
||||
list.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
|
||||
for (rank, (hsh, c)) in list.iter().take(4).enumerate() {
|
||||
crypto[base + rank * 2] = crypto_float(*hsh);
|
||||
crypto[base + rank * 2 + 1] = *c;
|
||||
}
|
||||
}
|
||||
(o, crypto)
|
||||
})
|
||||
.collect();
|
||||
let mut aux = effectcraft_raster::AuxChannels::new(w, h, s);
|
||||
for (k, name) in NAMES.iter().enumerate() {
|
||||
aux.channels.push((name.to_string(), resolved.iter().map(|r| r.0[k]).collect()));
|
||||
}
|
||||
for (base, layer) in [(0usize, "CryptoObject"), (8usize, "CryptoMaterial")] {
|
||||
for rank_pair in 0..2 {
|
||||
for (c, ch) in ["R", "G", "B", "A"].iter().enumerate() {
|
||||
let k = base + rank_pair * 4 + c;
|
||||
aux.channels.push((format!("{layer}{rank_pair:02}.{ch}"), resolved.iter().map(|r| r.1[k]).collect()));
|
||||
}
|
||||
}
|
||||
}
|
||||
aux.manifests = vec![("CryptoObject".into(), manifest_obj), ("CryptoMaterial".into(), manifest_mat)];
|
||||
aux
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user