diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9caf1c27..f89c0e87 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,10 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/quad.gltf` | EffectCraft contributors | Original work: glTF 2.0 test fixture (textured quad, node hierarchy, translation animation; data URIs) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/quad.glb` | EffectCraft contributors | Original work: binary glTF test fixture (the same quad with its texture in the BIN chunk) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | +| `crates/model/tests/fixtures/cube.mtl` | EffectCraft contributors | Original work: MTL test fixture (materials of cube.obj) | MIT OR Apache-2.0 | ## Screenshots diff --git a/Cargo.lock b/Cargo.lock index c1e86483..c2468e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "effectcraft-geom", "effectcraft-keyframe", "effectcraft-lottie", + "effectcraft-model", "effectcraft-path", "effectcraft-project", "effectcraft-raster", @@ -1325,10 +1326,12 @@ dependencies = [ name = "effectcraft-media" version = "0.1.0" dependencies = [ + "effectcraft-model", "effectcraft-project", "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "exr", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", @@ -1337,6 +1340,18 @@ dependencies = [ "image", "log", "rayon", + "serde_json", + "thiserror 2.0.21", +] + +[[package]] +name = "effectcraft-model" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "image", + "serde", + "serde_json", "thiserror 2.0.21", ] @@ -1382,6 +1397,7 @@ dependencies = [ "effectcraft-effects", "effectcraft-geom", "effectcraft-keyframe", + "effectcraft-model", "effectcraft-path", "effectcraft-project", "effectcraft-raster", @@ -1785,6 +1801,7 @@ dependencies = [ "miniz_oxide 0.8.9", "num-complex", "pulp", + "rayon-core", "smallvec", "zune-inflate", ] diff --git a/Cargo.toml b/Cargo.toml index 6c5ac99a..23fb4f29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ effectcraft-project = { path = "crates/project" } effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } effectcraft-track = { path = "crates/track" } +effectcraft-model = { path = "crates/model" } effectcraft-render = { path = "crates/render" } effectcraft-gpu = { path = "crates/gpu" } effectcraft-media = { path = "crates/media" } @@ -53,6 +54,7 @@ kurbo = { version = "0.13", features = ["serde"] } tiny-skia = { version = "0.12", default-features = false, features = ["std", "simd"] } skrifa = { version = "0.44", default-features = false, features = ["std"] } harfrust = { version = "0.12", default-features = false, features = ["std"] } +exr = "1.74" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index e321fa2f..ff5d3ed7 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -17,6 +17,5 @@ effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-track = { workspace = true } rayon = { workspace = true } - -[dev-dependencies] serde_json = { workspace = true } + diff --git a/crates/effects/src/audio_fx.rs b/crates/effects/src/audio_fx.rs index 855fd10e..bd80519a 100644 --- a/crates/effects/src/audio_fx.rs +++ b/crates/effects/src/audio_fx.rs @@ -1,5 +1,5 @@ -//! Audio effects (Effect > Audio): Backwards, Bass & Treble, Delay, Flange & Chorus, -//! High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer and Tone. +//! Audio effects (Effect > Audio): Backwards, Bass & Treble, Compressor, Delay, Distortion, +//! Flange & Chorus, Gate, High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer and Tone. //! //! They are ordinary effect instances on a layer (so their parameters animate and show in //! Effect Controls), pass video through unchanged, and are applied to the layer's audio by the @@ -12,7 +12,10 @@ //! DSP is textbook: second-order sections use the public RBJ "Audio EQ Cookbook" formulas; //! Reverb is a Schroeder/Moorer network (parallel low-pass feedback combs into series //! all-passes, as in the published Freeverb description); chorus/flange/vibrato are modulated -//! fractional delay lines. +//! fractional delay lines. Compressor and Gate are feed-forward dynamics processors on a +//! stereo-linked peak envelope (one-pole attack/release smoothing, the textbook quadratic soft-knee +//! gain computer); Distortion is a static waveshaper (soft clip, hard clip, fold-back, bit crush) +//! followed by a one-pole tone filter. use std::f64::consts::PI; @@ -60,6 +63,20 @@ pub fn specs() -> Vec { p("treble", "Treble", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), ], ), + spec( + "ec.audio.compressor", + "Compressor", + vec![ + p("threshold", "Threshold (dB)", num(-20.0), slider(-60.0, 0.0, -60.0, 0.0, 2)), + p("ratio", "Ratio", num(4.0), slider(1.0, 30.0, 1.0, 20.0, 2)), + p("attack", "Attack (ms)", num(10.0), slider(0.0, 500.0, 0.0, 200.0, 2)), + p("release", "Release (ms)", num(100.0), slider(1.0, 5000.0, 1.0, 1000.0, 2)), + p("knee", "Knee (dB)", num(0.0), slider(0.0, 24.0, 0.0, 24.0, 2)), + p("makeupGain", "Makeup Gain (dB)", num(0.0), slider(-24.0, 36.0, 0.0, 24.0, 2)), + pp("dryOut", "Dry Out", pct(0.0)), + pp("wetOut", "Wet Out", pct(100.0)), + ], + ), spec( "ec.audio.delay", "Delay", @@ -71,6 +88,19 @@ pub fn specs() -> Vec { pp("wetOut", "Wet Out", pct(75.0)), ], ), + spec( + "ec.audio.distortion", + "Distortion", + vec![ + p("distortionType", "Distortion Type", Value::Enum(0), popup(&["Soft Clip", "Hard Clip", "Fold Back", "Bit Crush"])), + p("drive", "Drive (dB)", num(12.0), slider(0.0, 48.0, 0.0, 48.0, 2)), + p("bitDepth", "Bit Depth", num(8.0), slider(1.0, 16.0, 1.0, 16.0, 0)), + p("tone", "Tone (Hz)", num(8000.0), slider(200.0, 20000.0, 200.0, 20000.0, 1)), + p("outputGain", "Output Gain (dB)", num(-6.0), slider(-48.0, 12.0, -24.0, 12.0, 2)), + pp("dryOut", "Dry Out", pct(0.0)), + pp("wetOut", "Wet Out", pct(100.0)), + ], + ), spec( "ec.audio.flangechorus", "Flange & Chorus", @@ -86,6 +116,17 @@ pub fn specs() -> Vec { pp("wetOut", "Wet Out", pct(50.0)), ], ), + spec( + "ec.audio.gate", + "Gate", + vec![ + p("threshold", "Threshold (dB)", num(-40.0), slider(-90.0, 0.0, -80.0, 0.0, 2)), + p("attack", "Attack (ms)", num(1.0), slider(0.0, 500.0, 0.0, 100.0, 2)), + p("hold", "Hold (ms)", num(50.0), slider(0.0, 5000.0, 0.0, 500.0, 2)), + p("release", "Release (ms)", num(100.0), slider(1.0, 5000.0, 1.0, 1000.0, 2)), + p("range", "Range (dB)", num(-80.0), slider(-120.0, 0.0, -90.0, 0.0, 2)), + ], + ), spec( "ec.audio.highlowpass", "High-Low Pass", @@ -167,6 +208,14 @@ pub fn preroll(id: &str, params: &Params) -> f64 { match id { "ec.audio.basstreble" | "ec.audio.highlowpass" | "ec.audio.parametriceq" => 0.05, "ec.audio.flangechorus" | "ec.audio.modulator" => 0.06, + "ec.audio.distortion" => 0.01, + // The envelope settles within a few attack/release time constants. + "ec.audio.compressor" | "ec.audio.gate" => { + let rel = params.f("release").max(1.0) / 1000.0; + let att = params.f("attack").max(0.0) / 1000.0; + let hold = if id == "ec.audio.gate" { params.f("hold").max(0.0) / 1000.0 } else { 0.0 }; + (9.0 * rel + 9.0 * att + hold + 0.01).min(10.0) + } "ec.audio.delay" => { let d = params.f("delayTime").max(1.0) / 1000.0; let fb = (params.f("feedback") / 100.0).clamp(0.0, 0.99); @@ -231,6 +280,9 @@ pub fn process(id: &str, params: &Params, buf: &mut [f32], rate: u32, start: f64 } } "ec.audio.delay" => delay(params, buf, sr), + "ec.audio.compressor" => compressor(params, buf, sr), + "ec.audio.distortion" => distortion(params, buf, sr), + "ec.audio.gate" => gate(params, buf, sr), "ec.audio.flangechorus" => flange_chorus(params, buf, sr, start), "ec.audio.modulator" => modulator(params, buf, sr, start), "ec.audio.reverb" => reverb(params, buf, sr), @@ -362,6 +414,126 @@ fn delay(params: &Params, buf: &mut [f32], sr: f64) { } } +#[inline] +fn db_to_lin(db: f64) -> f64 { + 10f64.powf(db / 20.0) +} + +#[inline] +fn lin_to_db(v: f64) -> f64 { + 20.0 * v.max(1e-12).log10() +} + +/// One-pole smoothing coefficient for a time constant of `ms` milliseconds. +fn smoothing(ms: f64, sr: f64) -> f64 { + if ms <= 0.0 { 0.0 } else { (-1.0 / (ms / 1000.0 * sr)).exp() } +} + +/// Static compressor curve: gain change (dB, ≤ 0) for an input level `x` dB (quadratic soft +/// knee of width `knee` around the threshold). +pub fn compressor_gain_db(x: f64, threshold: f64, ratio: f64, knee: f64) -> f64 { + let r = ratio.max(1.0); + let over = x - threshold; + let y = if knee > 0.0 && 2.0 * over.abs() <= knee { + x + (1.0 / r - 1.0) * (over + knee / 2.0).powi(2) / (2.0 * knee) + } else if over > 0.0 { + threshold + over / r + } else { + x + }; + y - x +} + +fn compressor(params: &Params, buf: &mut [f32], sr: f64) { + let th = params.f("threshold"); + let ratio = params.f("ratio").max(1.0); + let knee = params.f("knee").max(0.0); + let att = smoothing(params.f("attack"), sr); + let rel = smoothing(params.f("release").max(0.001), sr); + let makeup = params.f("makeupGain"); + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + // Smoothed gain change in dB (≤ 0): attack while the reduction grows, release as it shrinks. + let mut g = 0.0f64; + for f in buf.chunks_exact_mut(2) { + let peak = f[0].abs().max(f[1].abs()) as f64; + let target = compressor_gain_db(lin_to_db(peak), th, ratio, knee); + let k = if target < g { att } else { rel }; + g = target + (g - target) * k; + let lin = db_to_lin(g + makeup) as f32; + for v in f.iter_mut() { + *v = *v * dry + *v * lin * wet; + } + } +} + +fn gate(params: &Params, buf: &mut [f32], sr: f64) { + let th = db_to_lin(params.f("threshold")); + let att = smoothing(params.f("attack"), sr); + let rel = smoothing(params.f("release").max(0.001), sr); + let hold = (params.f("hold").max(0.0) / 1000.0 * sr) as usize; + let floor = db_to_lin(params.f("range").min(0.0)); + // Openness 0..1 (closed..open), mapped to floor..1. + let mut g = 0.0f64; + let mut held = 0usize; + for f in buf.chunks_exact_mut(2) { + let peak = f[0].abs().max(f[1].abs()) as f64; + let open = if peak >= th { + held = hold; + true + } else if held > 0 { + held -= 1; + true + } else { + false + }; + let target = if open { 1.0 } else { 0.0 }; + let k = if target > g { att } else { rel }; + g = target + (g - target) * k; + let lin = (floor + (1.0 - floor) * g) as f32; + f[0] *= lin; + f[1] *= lin; + } +} + +/// The Distortion effect's waveshaper for one sample (`drive` linear gain, `bits` for Bit Crush). +pub fn distort_sample(kind: u32, x: f32, drive: f32, bits: f32) -> f32 { + let v = x * drive; + match kind { + 1 => v.clamp(-1.0, 1.0), + 2 => { + // Fold back: reflect at ±1 (a triangle function of the input). + let t = (v + 1.0).rem_euclid(4.0); + if t < 2.0 { t - 1.0 } else { 3.0 - t } + } + 3 => { + let q = 2f32.powf(bits.clamp(1.0, 16.0) - 1.0); + (v.clamp(-1.0, 1.0) * q).round() / q + } + _ => v.tanh(), + } +} + +fn distortion(params: &Params, buf: &mut [f32], sr: f64) { + let kind = params.e("distortionType"); + let drive = db_to_lin(params.f("drive")) as f32; + let bits = params.f("bitDepth") as f32; + let out = db_to_lin(params.f("outputGain")) as f32; + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let fc = params.f("tone").clamp(20.0, sr * 0.49); + let a = (-2.0 * PI * fc / sr).exp() as f32; + let mut lp = [0.0f32; 2]; + for f in buf.chunks_exact_mut(2) { + for ch in 0..2 { + let x = f[ch]; + let y = distort_sample(kind, x, drive, bits); + lp[ch] = y + (lp[ch] - y) * a; + f[ch] = x * dry + lp[ch] * out * wet; + } + } +} + fn lfo(kind: u32, phase: f64) -> f64 { let ph = phase.rem_euclid(1.0); match kind { @@ -546,7 +718,10 @@ mod tests { for id in [ "ec.audio.backwards", "ec.audio.basstreble", + "ec.audio.compressor", "ec.audio.delay", + "ec.audio.distortion", + "ec.audio.gate", "ec.audio.flangechorus", "ec.audio.highlowpass", "ec.audio.modulator", @@ -679,4 +854,64 @@ mod tests { assert_eq!(b, vec![2.0, 1.0]); assert!(reverses("ec.audio.backwards")); } + + #[test] + fn compressor_reduces_gain_above_threshold() { + // Static curve: 10 dB over a −20 dB threshold at 4:1 comes out 7.5 dB lower. + assert!((compressor_gain_db(-10.0, -20.0, 4.0, 0.0) + 7.5).abs() < 1e-9); + assert_eq!(compressor_gain_db(-30.0, -20.0, 4.0, 0.0), 0.0); + // The soft knee meets the hard curve at its edges. + let k = 6.0; + assert!(compressor_gain_db(-23.0, -20.0, 4.0, k).abs() < 1e-9); + assert!((compressor_gain_db(-17.0, -20.0, 4.0, k) - compressor_gain_db(-17.0, -20.0, 4.0, 0.0)).abs() < 1e-9); + let sr = 48000.0; + let mut p = defaults("ec.audio.compressor"); + p.values.insert("attack".into(), num(1.0)); + let mut loud = sine(1000.0, sr, 9600); + process("ec.audio.compressor", &p, &mut loud, 48000, 0.0); + // A full-scale sine (peak 0 dB) is pulled down by ~15 dB once the envelope settles. + let peak = loud[18000..].iter().fold(0.0f32, |m, v| m.max(v.abs())); + let want = db_to_lin(compressor_gain_db(0.0, -20.0, 4.0, 0.0)) as f32; + assert!((peak - want).abs() < 0.05, "{peak} vs {want}"); + // Quiet input passes untouched. + let mut quiet: Vec = sine(1000.0, sr, 4800).iter().map(|v| v * 0.01).collect(); + let orig = quiet.clone(); + process("ec.audio.compressor", &p, &mut quiet, 48000, 0.0); + assert!(quiet.iter().zip(&orig).all(|(a, b)| (a - b).abs() < 1e-6)); + } + + #[test] + fn gate_mutes_below_threshold() { + let sr = 48000.0; + let p = defaults("ec.audio.gate"); + // A −60 dB signal under the −40 dB threshold is attenuated by the −80 dB range. + let mut quiet: Vec = sine(500.0, sr, 9600).iter().map(|v| v * 0.001).collect(); + process("ec.audio.gate", &p, &mut quiet, 48000, 0.0); + assert!(rms(&quiet, 4800) < 1e-6, "{}", rms(&quiet, 4800)); + // A loud signal opens the gate. + let mut loud: Vec = sine(500.0, sr, 9600).iter().map(|v| v * 0.5).collect(); + let r0 = rms(&loud, 4800); + process("ec.audio.gate", &p, &mut loud, 48000, 0.0); + assert!((rms(&loud, 4800) - r0).abs() < 0.01); + // Deterministic. + let mut a = sine(300.0, sr, 2000); + let mut b = a.clone(); + process("ec.audio.gate", &p, &mut a, 48000, 0.0); + process("ec.audio.gate", &p, &mut b, 48000, 0.0); + assert_eq!(a, b); + } + + #[test] + fn distortion_shapes_and_bounds() { + assert!((distort_sample(0, 0.5, 1.0, 8.0) - 0.5f32.tanh()).abs() < 1e-6); + assert_eq!(distort_sample(1, 0.8, 2.0, 8.0), 1.0); + assert!((distort_sample(2, 0.7, 2.0, 8.0) - 0.6).abs() < 1e-5); + assert_eq!(distort_sample(3, 0.3, 1.0, 2.0), 0.5); + let p = defaults("ec.audio.distortion"); + let mut b = sine(200.0, 48000.0, 4800); + process("ec.audio.distortion", &p, &mut b, 48000, 0.0); + assert!(b.iter().all(|v| v.abs() <= 1.0)); + assert!(rms(&b, 100) > 0.1); + assert!(preroll("ec.audio.compressor", &defaults("ec.audio.compressor")) > 0.4); + } } diff --git a/crates/effects/src/blur3.rs b/crates/effects/src/blur3.rs new file mode 100644 index 00000000..0140ac99 --- /dev/null +++ b/crates/effects/src/blur3.rs @@ -0,0 +1,338 @@ +//! Blur & Sharpen: CC Radial Blur and Camera-Shake Deblur. +//! +//! * **CC Radial Blur** — zoom (straight, fading, centred both ways) and rotational (plain or +//! fading) blurs around a centre point, sampled along the ray / arc through each pixel with a +//! sample count set by Quality. +//! * **Camera-Shake Deblur** — frame-based restoration in the spirit of published video deblurring +//! by "lucky frames": the frame is split into patches; for each patch the sharpness (mean +//! squared gradient) is compared with the same patch in the neighbouring frames within Blur +//! Duration; when a neighbour is clearly sharper, its patch is aligned to the current frame by +//! a small translation search (sum of squared differences on luma) and substituted with +//! feathered patch blending. Deterministic; neighbours come from [`crate::EffectHost::self_at`]. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::util::Plane; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Blur & Sharpen", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Radial Blur + +pub const RADIAL_TYPES: [&str; 5] = ["Straight Zoom", "Fading Zoom", "Centered", "Rotate", "Rotate Fading"]; + +fn cc_radial_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let amount = ctx.params.f("amount"); + if amount == 0.0 { + return b; + } + let kind = ctx.params.e("type"); + let quality = ctx.params.f("quality").clamp(1.0, 100.0); + let c = b.to_px(ctx.params.v2("center")); + let src = b.img.clone(); + // Zoom: fraction of the distance to the centre; rotate: degrees of arc. + let zoom = (amount / 100.0).clamp(-1.0, 1.0) * 0.5; + let arc = amount.to_radians() * 0.25; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let (dx, dy) = (x as f64 + 0.5 - c.0, y as f64 + 0.5 - c.1); + let r = (dx * dx + dy * dy).sqrt(); + let span = if kind >= 3 { r * arc.abs() } else { r * zoom.abs() }; + let n = ((span * quality / 50.0).ceil() as usize).clamp(1, 256); + if n <= 1 { + *o = src.get(x as i64, y as i64); + continue; + } + let mut acc = [0.0f32; 4]; + let mut wsum = 0.0f32; + for i in 0..n { + let t = i as f64 / (n - 1) as f64; + let (sx, sy, w) = match kind { + // Toward the centre, equal weights. + 0 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + // Toward the centre, weights fading with distance along the streak. + 1 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, (1.0 - t) as f32 + 0.05) + } + // Both ways around the pixel. + 2 => { + let k = 1.0 + zoom * (t - 0.5); + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + _ => { + let a = arc * (t - 0.5); + let (s, co) = a.sin_cos(); + let w = if kind == 4 { (1.0 - (2.0 * t - 1.0).abs()) as f32 + 0.05 } else { 1.0 }; + (c.0 + dx * co - dy * s, c.1 + dx * s + dy * co, w) + } + }; + let q = src.sample_bilinear(sx, sy); + for k in 0..4 { + acc[k] += q[k] * w; + } + wsum += w; + } + *o = acc.map(|v| v / wsum); + } + }); + b +} + +// ---------------------------------------------------------------- Camera-Shake Deblur + +/// Mean squared luma gradient over a rectangle (sharpness). +fn sharpness(l: &Plane, x0: usize, y0: usize, x1: usize, y1: usize) -> f64 { + let mut s = 0.0f64; + let mut n = 0usize; + for y in y0..y1.saturating_sub(1) { + for x in x0..x1.saturating_sub(1) { + let v = l.get(x, y); + let gx = l.get(x + 1, y) - v; + let gy = l.get(x, y + 1) - v; + s += (gx * gx + gy * gy) as f64; + n += 1; + } + } + if n == 0 { 0.0 } else { s / n as f64 } +} + +/// Best integer translation (dx, dy) within ±`r` aligning `b` to `a` on a patch (SSD). +fn align(a: &Plane, bp: &Plane, x0: usize, y0: usize, x1: usize, y1: usize, r: i64) -> (i64, i64) { + let mut best = (f64::INFINITY, 0i64, 0i64); + let step = ((x1 - x0).max(y1 - y0) / 24).max(1); + for dy in -r..=r { + for dx in -r..=r { + let mut s = 0.0f64; + for y in (y0..y1).step_by(step) { + for x in (x0..x1).step_by(step) { + let d = a.get(x, y) - bp.get_clamped(x as i64 + dx, y as i64 + dy); + s += (d * d) as f64; + } + } + // Prefer the smallest shift on ties. + let s = s * (1.0 + 1e-9 * (dx * dx + dy * dy) as f64); + if s < best.0 { + best = (s, dx, dy); + } + } + } + (best.1, best.2) +} + +fn camera_shake_deblur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let dur = ctx.params.f("blurDuration").clamp(1.0, 10.0).round() as i64; + let search = (ctx.params.f("searchRange").max(0.0) * b.scale).round() as i64; + let patch = (ctx.params.f("patchSize").max(8.0) * b.scale).round().max(4.0) as usize; + let thresh = 1.0 + ctx.params.f("threshold").max(0.0) / 100.0; + let show = ctx.params.b("showBlurryFrames"); + let fps = ctx.fps(); + let (w, h) = (b.img.width as usize, b.img.height as usize); + if w == 0 || h == 0 { + return b; + } + // Neighbouring frames in the effect's own input, resampled into this buffer's grid. + let mut neigh: Vec<(Image, Plane)> = vec![]; + for k in (-dur..=dur).filter(|k| *k != 0) { + let Some(nb) = host.self_at(ctx.time + k as f64 / fps, ctx.env.effect_index) else { continue }; + let img = if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + nb.img + } else { + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) + }; + let l = Plane::luma(&img); + neigh.push((img, l)); + } + if neigh.is_empty() { + return b; + } + let cur_l = Plane::luma(&b.img); + let (nx, ny) = (w.div_ceil(patch), h.div_ceil(patch)); + // Per patch: (neighbour index, shift) of the substitute, or None to keep. + let choice: Vec> = (0..nx * ny) + .into_par_iter() + .map(|i| { + let (px, py) = (i % nx, i / nx); + let (x0, y0) = (px * patch, py * patch); + let (x1, y1) = ((x0 + patch).min(w), (y0 + patch).min(h)); + let s0 = sharpness(&cur_l, x0, y0, x1, y1); + let mut best: Option<(f64, usize)> = None; + for (k, (_, l)) in neigh.iter().enumerate() { + let s = sharpness(l, x0, y0, x1, y1); + if s > s0 * thresh && best.is_none_or(|(bs, _)| s > bs) { + best = Some((s, k)); + } + } + let (_, k) = best?; + let (dx, dy) = align(&cur_l, &neigh[k].1, x0, y0, x1, y1, search); + Some((k, dx, dy)) + }) + .collect(); + if choice.iter().all(Option::is_none) { + return b; + } + let src = b.img.clone(); + // Feathered blending between patch decisions: each pixel mixes the four nearest patch + // centres bilinearly. + let pick = |i: usize, x: usize, y: usize| -> Px { + match choice[i] { + None => src.data[y * w + x], + Some((k, dx, dy)) => neigh[k].0.get_clamped(x as i64 + dx, y as i64 + dy), + } + }; + b.img.rows_mut().for_each(|(y, row)| { + let fy = ((y as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (ny - 1) as f64); + let (y0, ty) = (fy.floor() as usize, (fy - fy.floor()) as f32); + let y1 = (y0 + 1).min(ny - 1); + for (x, o) in row.iter_mut().enumerate() { + let fx = ((x as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (nx - 1) as f64); + let (x0, tx) = (fx.floor() as usize, (fx - fx.floor()) as f32); + let x1 = (x0 + 1).min(nx - 1); + let ids = [y0 * nx + x0, y0 * nx + x1, y1 * nx + x0, y1 * nx + x1]; + if ids.iter().all(|&i| choice[i].is_none()) { + continue; + } + let ws = [(1.0 - tx) * (1.0 - ty), tx * (1.0 - ty), (1.0 - tx) * ty, tx * ty]; + let mut acc = [0.0f32; 4]; + for (i, wgt) in ids.iter().zip(ws) { + let q = pick(*i, x, y); + for c in 0..4 { + acc[c] += q[c] * wgt; + } + } + if show { + acc[0] = (acc[0] + 0.3 * acc[3]).min(acc[3].max(acc[0])); + } + *o = acc; + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.blur.ccradial", + "CC Radial Blur", + vec![ + p("type", "Type", Value::Enum(1), popup(&RADIAL_TYPES)), + p("amount", "Amount", num(10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("quality", "Quality", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 1)), + p("center", "Center", Value::Vec2([0.5, 0.5]), ParamUi::Point), + ], + cc_radial_blur, + ), + spec( + "ec.blur.camerashakedeblur", + "Camera-Shake Deblur", + vec![ + p("blurDuration", "Blur Duration", num(2.0), slider(1.0, 10.0, 1.0, 10.0, 0)), + p("searchRange", "Search Range", num(8.0), slider(0.0, 64.0, 0.0, 32.0, 0)), + p("patchSize", "Patch Size", num(32.0), slider(8.0, 256.0, 8.0, 128.0, 0)), + p("threshold", "Threshold", num(20.0), slider(0.0, 500.0, 0.0, 100.0, 1)), + p("showBlurryFrames", "Show Blurry Frames", Value::Bool(false), ParamUi::Checkbox), + ], + camera_shake_deblur, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn checker(w: u32, h: u32, cell: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if ((x as u32 / cell) + (y as u32 / cell)).is_multiple_of(2) { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }) + } + + #[test] + fn cc_radial_blur_keeps_centre_and_smears_edges() { + let img = checker(64, 64, 4); + for kind in 0..5u32 { + let out = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // Far from the centre the checker is smeared toward grey. + let edge = out.img.get(2, 2)[0]; + assert!(edge > 0.05 && edge < 0.95, "type {kind}: {edge}"); + let again = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + // Zero amount is identity; a constant image stays constant. + let flat = Image::filled(16, 16, [0.4, 0.4, 0.4, 1.0]); + let o = run_fx("ec.blur.ccradial", &[("amount", num(80.0))], flat.clone(), 0.0, EffectEnv::default()); + assert!(o.img.data.iter().all(|p| (p[0] - 0.4).abs() < 1e-5)); + let o = run_fx("ec.blur.ccradial", &[("amount", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(o.img.data, img.data); + } + + /// Frames: sharp checker at even frames, a blurred one at odd frames (shifted by 2 px). + struct Shaky; + impl EffectHost for Shaky { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + let f = (t * 30.0).round() as i64; + Some(Buf { img: frame(f), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn frame(f: i64) -> Image { + let sharp = crate::util::gen_image(48, 48, |x, y| { + let v = if (((x + 2) / 6) + (y / 6)) % 2 == 0 { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }); + if f % 2 == 0 { sharp } else { effectcraft_raster::gaussian_blur(&sharp, 2.5, 2.5, true) } + } + + #[test] + fn camera_shake_deblur_substitutes_sharp_neighbours() { + let host = Shaky; + let env = EffectEnv { host: Some(&host), frame_rate: 30.0, ..Default::default() }; + let blurry = frame(1); + let out = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + let sharp = frame(0); + let err = |a: &Image| a.data.iter().zip(&sharp.data).map(|(p, q)| (p[0] - q[0]).abs()).sum::() / a.data.len() as f32; + assert!(err(&out.img) < err(&blurry) * 0.5, "{} vs {}", err(&out.img), err(&blurry)); + // A sharp frame is left alone (its neighbours are blurrier). + let keep = run_fx("ec.blur.camerashakedeblur", &[], sharp.clone(), 0.0, env); + assert_eq!(keep.img.data, sharp.data); + // Deterministic; no host = pass-through. + let again = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.blur.camerashakedeblur", &[], blurry.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, blurry.data); + } +} diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs new file mode 100644 index 00000000..5fd4d715 --- /dev/null +++ b/crates/effects/src/channel3d.rs @@ -0,0 +1,681 @@ +//! 3D Channel effects: 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, +//! Fog 3D, ID Matte and IDentifier. +//! +//! They read the layer's auxiliary channels ([`AuxChannels`], via +//! [`crate::EffectHost::aux`]): multi-layer OpenEXR footage (depth, object/material IDs, +//! Cryptomatte layers, any named channel) or the compositor's depth / layer-ID / Cryptomatte +//! pass for a precomp of a 3D comp. Without auxiliary channels they pass the layer through +//! (EXtractoR still maps the layer's own R, G, B and A). +//! +//! Aux pixels are looked up nearest-neighbour (IDs must not be interpolated) at each buffer +//! pixel's layer position, so render scale and layer padding are respected. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_raster::{AuxChannels, Image, Px, gaussian_blur}; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, hash1, premul, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } +} + +fn aux(ctx: &EffectCtx) -> Option> { + ctx.env.host?.aux() +} + +/// Aux index for each buffer pixel (None outside the aux image). +fn index_map(b: &Buf, a: &AuxChannels) -> Vec> { + let (w, h) = (b.img.width as usize, b.img.height as usize); + let inv = 1.0 / b.scale.max(1e-9); + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + a.index_at((x - b.offset[0]) * inv, (y - b.offset[1]) * inv) + }) + .collect() +} + +/// A channel sampled into the buffer grid (`fill` outside). +fn plane_of(b: &Buf, idx: &[Option], ch: &[f32], fill: f32) -> Plane { + Plane { w: b.img.width as usize, h: b.img.height as usize, data: idx.par_iter().map(|i| i.map_or(fill, |i| ch[i])).collect() } +} + +/// Distinct colour for an ID (hash → hue). +fn id_color(id: f32) -> [f32; 3] { + let h = hash1(id.to_bits(), 0x1d, 0x5eed); + let (r, g, b) = effectcraft_color::hsl_to_rgb(h, 0.75, 0.5); + if id == 0.0 { [0.0; 3] } else { [r, g, b] } +} + +// ---------------------------------------------------------------- 3D Channel Extract + +pub const EXTRACT_CHANNELS: [&str; 8] = ["Z-Depth", "Object ID", "Texture UV", "Surface Normals", "Coverage", "Background RGB", "Unclamped RGB", "Material ID"]; + +fn channel_extract(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let idx = index_map(&b, &a); + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let lv = |v: f32| ((v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum()).clamp(0.0, 1.0); + let lv = |v: f32| if wp >= bp { lv(v) } else { 1.0 - ((v - wp) / (bp - wp).max(1e-9)).clamp(0.0, 1.0) }; + let get3 = |names: [&[&str]; 3]| -> Option<[&[f32]; 3]> { Some([a.find(names[0])?, a.find(names[1])?, a.find(names[2])?]) }; + let img: Option = match ctx.params.e("channel") { + 0 => a.depth().map(|z| map_grey(&b, &idx, |i| lv(z[i]))), + 1 => a.object_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 7 => a.material_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 4 => a.coverage().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 2 => get3([&["UV.U", "U", "uv.x"], &["UV.V", "V", "uv.y"], &["UV.W", "W", "uv.z"]]) + .or_else(|| Some([a.find(&["UV.U", "U", "uv.x"])?, a.find(&["UV.V", "V", "uv.y"])?, a.find(&["UV.V", "V", "uv.y"])?])) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), 0.0])), + 3 => get3([&["N.X", "Normal.X", "normals.x"], &["N.Y", "Normal.Y", "normals.y"], &["N.Z", "Normal.Z", "normals.z"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i] * 0.5 + 0.5), lv(c[1][i] * 0.5 + 0.5), lv(c[2][i] * 0.5 + 0.5)])), + 5 => get3([&["BG.R", "Background.R"], &["BG.G", "Background.G"], &["BG.B", "Background.B"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), lv(c[2][i])])), + _ => get3([&["R"], &["G"], &["B"]]).map(|c| map_rgb(&b, &idx, |i| [c[0][i], c[1][i], c[2][i]])), + }; + if let Some(i) = img { + b.img = i; + } + b +} + +fn map_grey(b: &Buf, idx: &[Option], f: impl Fn(usize) -> f32 + Sync) -> Image { + map_rgb(b, idx, |i| { + let v = f(i); + [v, v, v] + }) +} + +fn map_rgb(b: &Buf, idx: &[Option], f: impl Fn(usize) -> [f32; 3] + Sync) -> Image { + let mut img = Image::new(b.img.width, b.img.height); + img.data.par_iter_mut().zip(idx.par_iter()).for_each(|(p, i)| { + if let Some(i) = i { + let c = f(*i); + *p = [c[0], c[1], c[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- Depth Matte + +fn depth_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let idx = index_map(&b, &a); + let depth = ctx.params.f("depth") as f32; + let feather = ctx.params.f("feather").max(0.0) as f32; + let invert = ctx.params.b("invert"); + // Keeps what lies at or beyond Depth (Invert keeps what is in front), with a soft ramp of + // `feather` depth units centred on the threshold. + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().zip(zp.data.par_iter()).for_each(|(px, &zv)| { + let k = if feather > 0.0 { + ((zv - depth) / feather + 0.5).clamp(0.0, 1.0) + } else if zv >= depth { + 1.0 + } else { + 0.0 + }; + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +// ---------------------------------------------------------------- Depth of Field + +/// Blur radius (pixels) for depth `z`. +pub fn dof_radius(z: f32, focal: f32, thickness: f32, max_r: f32, bias: f32) -> f32 { + let d = ((z - focal).abs() - thickness * 0.5).max(0.0); + let range = focal.abs().max(1.0); + let n = (d / range).min(1.0); + // Bias 50 = linear; higher bias brings blur in sooner. + let gamma = 2f32.powf((50.0 - bias) / 25.0); + max_r * n.powf(gamma) +} + +fn depth_of_field(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let max_r = (ctx.params.f("maximumRadius").max(0.0) * b.scale) as f32; + if max_r <= 0.0 { + return b; + } + let focal = ctx.params.f("focalPlane") as f32; + let thick = ctx.params.f("focalPlaneThickness").max(0.0) as f32; + let bias = ctx.params.f("focalBias") as f32; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + // A stack of blurs at a few radii, interpolated per pixel by its circle of confusion. + const LEVELS: usize = 5; + let stack: Vec = (0..LEVELS) + .into_par_iter() + .map(|k| { + let r = max_r * k as f32 / (LEVELS - 1) as f32; + if r < 0.3 { b.img.clone() } else { gaussian_blur(&b.img, (r / 2.0) as f64, (r / 2.0) as f64, true) } + }) + .collect(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let r = dof_radius(zp.data[i], focal, thick, max_r, bias); + let f = (r / max_r * (LEVELS - 1) as f32).clamp(0.0, (LEVELS - 1) as f32); + let k0 = (f.floor() as usize).min(LEVELS - 2); + let t = f - k0 as f32; + let (a0, a1) = (stack[k0].data[i], stack[k0 + 1].data[i]); + *px = [0, 1, 2, 3].map(|c| a0[c] + (a1[c] - a0[c]) * t); + }); + b +} + +// ---------------------------------------------------------------- Fog 3D + +fn fog_3d(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let pr = ctx.params; + let fc = pr.color("fogColor"); + let (start, end) = (pr.f("fogStartDepth") as f32, pr.f("fogEndDepth") as f32); + let opacity = (pr.f("fogOpacity") / 100.0) as f32; + let density = (pr.f("scatteringDensity") / 100.0) as f32; + let foggy_bg = pr.b("foggyBackground"); + let contrib = (pr.f("layerContribution") / 100.0) as f32; + let grad = if contrib > 0.0 { ctx.layer_param("gradientLayer", true).map(|o| crate::util::fit_layer(ctx, &b, &o, true)) } else { None }; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let zv = zp.data[i]; + let bg = zv >= BACKGROUND_DEPTH * 0.5; + let mut f = if bg { + if foggy_bg { 1.0 } else { 0.0 } + } else { + let t = ((zv - start) / (end - start).abs().max(1e-6)).clamp(0.0, 1.0); + // Denser scattering thickens the fog sooner. + 1.0 - (1.0 - t).powf(1.0 + density * 3.0) + }; + if let Some(g) = &grad { + let (gc, ga) = unpremul(g.data[i]); + let l = effectcraft_color::luminance(gc[0], gc[1], gc[2]) * ga; + f *= 1.0 - contrib + contrib * l; + } + let f = (f * opacity).clamp(0.0, 1.0); + let (c, al) = unpremul(*px); + let al2 = if bg && foggy_bg { al + (1.0 - al) * f } else { al }; + let out = [0, 1, 2].map(|k| c[k] + (fc[k] - c[k]) * f); + *px = premul(out, al2); + }); + b +} + +// ---------------------------------------------------------------- ID Matte / IDentifier + +fn id_channel(a: &AuxChannels, which: u32) -> Option<&[f32]> { + if which == 1 { a.material_id() } else { a.object_id() } +} + +fn id_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("auxChannel")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("idSelection").round() as f32; + let cov = if ctx.params.b("useCoverage") { a.coverage() } else { None }; + let mut m = Plane { + w: b.img.width as usize, + h: b.img.height as usize, + data: idx.par_iter().map(|i| i.map_or(0.0, |i| if ids[i].round() == sel { cov.map_or(1.0, |c| c[i].clamp(0.0, 1.0)) } else { 0.0 })).collect(), + }; + let feather = ctx.params.f("feather").max(0.0) * b.scale; + if feather > 0.0 { + m = gauss_plane(&m, feather * 0.5, feather * 0.5); + } + let invert = ctx.params.b("invert"); + b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| { + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +fn identifier(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("channelType")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("id").round() as f32; + let display = ctx.params.e("display"); + let max_id = ids.iter().cloned().fold(1.0f32, f32::max); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let Some(k) = idx[i] else { + if display != 2 { + *px = [0.0; 4]; + } + return; + }; + let id = ids[k].round(); + let hit = id == sel; + *px = match display { + // Colors: every ID in its own colour, the selected one highlighted white. + 0 => { + let c = if hit && sel != 0.0 { [1.0; 3] } else { id_color(id) }; + [c[0], c[1], c[2], 1.0] + } + 1 => { + let v = if hit { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + } + 2 => { + let s = src.data[i]; + if hit { s } else { [0.0; 4] } + } + _ => { + let v = id / max_id; + [v, v, v, 1.0] + } + }; + }); + b +} + +// ---------------------------------------------------------------- Cryptomatte + +/// Does `name` match a selection pattern (`*` wildcards, case-sensitive like the spec)? +fn wild(pat: &str, name: &str) -> bool { + match pat.split_once('*') { + None => pat == name, + Some((pre, rest)) => { + let Some(tail) = name.strip_prefix(pre) else { return false }; + if rest.is_empty() { + return true; + } + (0..=tail.len()).filter(|&k| tail.is_char_boundary(k)).any(|k| wild(rest, &tail[k..])) + } + } +} + +/// Parse a selection: comma-separated names (optionally quoted) or `` hashes / wildcards. +pub fn selection_hashes(sel: &str, manifest: Option<&[(String, u32)]>) -> Vec { + let mut out = vec![]; + for tok in sel.split(',') { + let t = tok.trim().trim_matches('"').trim_matches('\''); + if t.is_empty() { + continue; + } + if t.contains('*') { + if let Some(m) = manifest { + out.extend(m.iter().filter(|(n, _)| wild(t, n)).map(|(_, h)| *h)); + } + continue; + } + if let Some(hex) = t.strip_prefix("<").and_then(|r| r.strip_suffix(">")) + && let Ok(h) = u32::from_str_radix(hex, 16) + { + out.push(h); + continue; + } + out.push(manifest.and_then(|m| m.iter().find(|(n, _)| n == t).map(|(_, h)| *h)).unwrap_or_else(|| crypto_hash(t))); + } + out +} + +pub const CRYPTO_LAYERS: [&str; 3] = ["CryptoObject", "CryptoMaterial", "CryptoAsset"]; + +fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let layers = a.crypto_layers(); + if layers.is_empty() { + return b; + } + let want = CRYPTO_LAYERS[(ctx.params.e("layer") as usize).min(2)]; + let layer = layers.iter().find(|l| l.as_str() == want || l.ends_with(want)).unwrap_or(&layers[0]).clone(); + let manifest = a.manifests.iter().find(|(l, _)| *l == layer).map(|(_, m)| m.as_slice()); + let sel = selection_hashes(ctx.params.s("selection"), manifest); + let idx = index_map(&b, &a); + let display = ctx.params.e("display"); + let matte_only = ctx.params.b("matteOnly"); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let ranks = idx[i].map(|k| a.crypto_ranks(&layer, k)).unwrap_or_default(); + let m: f32 = ranks.iter().filter(|(id, c)| *c > 0.0 && sel.contains(&id.to_bits())).map(|(_, c)| *c).sum::().clamp(0.0, 1.0); + if matte_only { + *px = [m, m, m, m]; + return; + } + *px = match display { + // Colors: coverage-weighted ID colours (the spec's preview), selection brighter. + 0 => { + let mut c = [0.0f32; 3]; + for (id, cov) in &ranks { + let k = id_color(*id); + for j in 0..3 { + c[j] += k[j] * cov; + } + } + let c = c.map(|v| v * (1.0 - 0.5 * m) + 0.5 * m); + [c[0], c[1], c[2], 1.0] + } + // Matte: the layer keyed by the selection. + 1 => src.data[i].map(|v| v * m), + // Preview: the layer with the selection tinted. + _ => { + let s = src.data[i]; + [s[0] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[1] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[2] * (1.0 - 0.5 * m), s[3]] + } + }; + }); + b +} + +// ---------------------------------------------------------------- EXtractoR + +fn extractor(ctx: &EffectCtx, mut b: Buf) -> Buf { + let a = aux(ctx); + let names = [ctx.params.s("red"), ctx.params.s("green"), ctx.params.s("blue"), ctx.params.s("alpha")]; + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let unmult = ctx.params.b("unMult"); + let clip = ctx.params.b("clip"); + let idx = a.as_ref().map(|a| index_map(&b, a)); + // Each output channel: an aux channel by name, or the layer's own R/G/B/A. + let own = |n: &str| match n.trim().to_ascii_uppercase().as_str() { + "R" | "RED" => Some(0), + "G" | "GREEN" => Some(1), + "B" | "BLUE" => Some(2), + "A" | "ALPHA" => Some(3), + "" => None, + _ => None, + }; + let src_ch: Vec> = names.iter().map(|n| a.as_ref().and_then(|a| if n.trim().is_empty() { None } else { a.get(n.trim()) })).collect(); + if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(n) == Some(k)) { + return b; + } + let src = b.img.clone(); + let lv = |v: f32| { + let o = (v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum(); + if clip { o.clamp(0.0, 1.0) } else { o } + }; + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let (sc, sa) = unpremul(src.data[i]); + let straight = [sc[0], sc[1], sc[2], sa]; + let mut o: Px = [0.0; 4]; + for k in 0..4 { + o[k] = match (src_ch[k], idx.as_ref().and_then(|m| m[i])) { + (Some(ch), Some(j)) => ch[j], + (Some(_), None) => 0.0, + (None, _) => match own(names[k]) { + Some(c) => straight[c], + None => { + if k == 3 { + 1.0 + } else { + 0.0 + } + } + }, + }; + if k < 3 { + o[k] = lv(o[k]); + } + } + let al = o[3].clamp(0.0, 1.0); + let c = if unmult && al > 1e-6 { [o[0] / al, o[1] / al, o[2] / al] } else { [o[0], o[1], o[2]] }; + *px = premul(c, al); + }); + b +} + +pub fn specs() -> Vec { + let text = |id: &'static str, name: &'static str, d: &str| p(id, name, Value::Str(d.into()), ParamUi::Text); + vec![ + spec( + "ec.3d.channelextract", + "3D Channel Extract", + vec![ + p("channel", "3D Channel", Value::Enum(0), popup(&EXTRACT_CHANNELS)), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("whitePoint", "White Point", num(10000.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + ], + channel_extract, + ), + spec( + "ec.3d.cryptomatte", + "Cryptomatte", + vec![ + p("layer", "Layer", Value::Enum(0), popup(&CRYPTO_LAYERS)), + text("selection", "Selection", ""), + p("display", "Display", Value::Enum(2), popup(&["Colors", "Matte", "Preview"])), + p("matteOnly", "Matte Only", Value::Bool(false), ParamUi::Checkbox), + ], + cryptomatte, + ), + spec( + "ec.3d.depthmatte", + "Depth Matte", + vec![ + p("depth", "Depth", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("feather", "Feather", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + ], + depth_matte, + ), + spec( + "ec.3d.depthoffield", + "Depth of Field", + vec![ + p("focalPlane", "Focal Plane", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("maximumRadius", "Maximum Radius", num(0.0), slider(0.0, 500.0, 0.0, 50.0, 2)), + p("focalPlaneThickness", "Focal Plane Thickness", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("focalBias", "Focal Bias", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 2)), + ], + depth_of_field, + ), + spec( + "ec.3d.extractor", + "EXtractoR", + vec![ + text("red", "Red", "R"), + text("green", "Green", "G"), + text("blue", "Blue", "B"), + text("alpha", "Alpha", "A"), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("whitePoint", "White Point", num(1.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("unMult", "UnMult", Value::Bool(false), ParamUi::Checkbox), + p("clip", "Clip", Value::Bool(false), ParamUi::Checkbox), + ], + extractor, + ), + spec( + "ec.3d.fog3d", + "Fog 3D", + vec![ + p("fogColor", "Fog Color", col(1.0, 1.0, 1.0), ParamUi::Color), + p("fogStartDepth", "Fog Start Depth", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogEndDepth", "Fog End Depth", num(3000.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogOpacity", "Fog Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scatteringDensity", "Scattering Density", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("foggyBackground", "Foggy Background", Value::Bool(true), ParamUi::Checkbox), + p("gradientLayer", "Gradient Layer", Value::Layer(None), ParamUi::Layer), + p("layerContribution", "Layer Contribution", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + fog_3d, + ), + spec( + "ec.3d.idmatte", + "ID Matte", + vec![ + p("auxChannel", "Aux. Channel", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("idSelection", "ID Selection", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("useCoverage", "Use Coverage", Value::Bool(false), ParamUi::Checkbox), + ], + id_matte, + ), + spec( + "ec.3d.identifier", + "IDentifier", + vec![ + p("channelType", "Channel Type", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("display", "Display", Value::Enum(0), popup(&["Colors", "Luma Matte", "Alpha Matte", "Raw"])), + p("id", "ID", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + ], + identifier, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + use effectcraft_raster::channels3d::crypto_float; + + /// A 20×10 layer: left half object 1 at depth 100, right half object 2 at depth 500; the + /// top-right corner is background. Cryptomatte: "left" / "right". + struct AuxHost(Arc); + impl EffectHost for AuxHost { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + Some(self.0.clone()) + } + } + + fn scene() -> AuxHost { + let (w, h) = (20usize, 10usize); + let mut a = AuxChannels::new(w as u32, h as u32, 1.0); + let (mut z, mut id, mut r, mut g) = (vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h]); + let (hl, hr) = (crypto_hash("left"), crypto_hash("right")); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if x >= 15 && y < 3 { + z[i] = BACKGROUND_DEPTH; + } else if x < 10 { + z[i] = 100.0; + id[i] = 1.0; + r[i] = crypto_float(hl); + g[i] = 1.0; + } else { + z[i] = 500.0; + id[i] = 2.0; + r[i] = crypto_float(hr); + g[i] = 1.0; + } + } + } + a.channels = vec![("Z".into(), z), ("ObjectID".into(), id), ("CryptoObject00.R".into(), r), ("CryptoObject00.G".into(), g)]; + a.manifests = vec![("CryptoObject".into(), vec![("left".into(), hl), ("right".into(), hr)])]; + AuxHost(Arc::new(a)) + } + + fn img() -> Image { + Image::filled(20, 10, [0.5, 0.5, 0.5, 1.0]) + } + + fn run(id: &str, vals: &[(&str, Value)], host: &AuxHost) -> Image { + run_fx(id, vals, img(), 0.0, EffectEnv { host: Some(host), ..Default::default() }).img + } + + #[test] + fn passes_through_without_aux_channels() { + for s in specs() { + let out = run_fx(s.id, &[], img(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, img().data, "{}", s.id); + } + } + + #[test] + fn depth_matte_thresholds() { + let h = scene(); + let o = run("ec.3d.depthmatte", &[("depth", num(300.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let inv = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("invert", Value::Bool(true))], &h); + assert_eq!(inv.get(2, 5)[3], 1.0); + assert_eq!(inv.get(12, 5)[3], 0.0); + // Feather: a ramp of 800 units centred on 300 → depth 100 at 0.25, 500 at 0.75. + let f = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("feather", num(800.0))], &h); + assert!((f.get(2, 5)[3] - 0.25).abs() < 1e-5 && (f.get(12, 5)[3] - 0.75).abs() < 1e-5); + } + + #[test] + fn channel_extract_maps_depth_and_ids() { + let h = scene(); + let o = run("ec.3d.channelextract", &[("blackPoint", num(0.0)), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6 && (o.get(12, 5)[0] - 0.5).abs() < 1e-6); + assert_eq!(o.get(17, 1)[0], 1.0); + let ids = run("ec.3d.channelextract", &[("channel", Value::Enum(1)), ("whitePoint", num(2.0))], &h); + assert_eq!(ids.get(2, 5)[0], 0.5); + } + + #[test] + fn id_matte_and_identifier() { + let h = scene(); + let o = run("ec.3d.idmatte", &[("idSelection", num(2.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let l = run("ec.3d.identifier", &[("display", Value::Enum(1)), ("id", num(1.0))], &h); + assert_eq!(l.get(2, 5)[0], 1.0); + assert_eq!(l.get(12, 5)[0], 0.0); + let c = run("ec.3d.identifier", &[], &h); + assert_ne!(c.get(2, 5), c.get(12, 5)); + } + + #[test] + fn cryptomatte_selects_by_name_and_wildcard() { + let h = scene(); + let m = run("ec.3d.cryptomatte", &[("selection", Value::Str("right".into())), ("matteOnly", Value::Bool(true))], &h); + assert_eq!(m.get(2, 5)[3], 0.0); + assert_eq!(m.get(12, 5)[3], 1.0); + let w = run("ec.3d.cryptomatte", &[("selection", Value::Str("l*".into())), ("display", Value::Enum(1))], &h); + assert_eq!(w.get(2, 5)[3], 1.0); + assert_eq!(w.get(12, 5)[3], 0.0); + assert!(wild("*ight", "right") && wild("r*t", "right") && !wild("x*", "right")); + let hex = format!("<{:08x}>", crypto_hash("left")); + assert_eq!(selection_hashes(&hex, None), vec![crypto_hash("left")]); + } + + #[test] + fn fog_and_depth_of_field_follow_depth() { + let h = scene(); + let f = run("ec.3d.fog3d", &[("fogColor", col(1.0, 0.0, 0.0)), ("fogEndDepth", num(1000.0)), ("scatteringDensity", num(0.0))], &h); + // Linear fog: 10 % at depth 100, 50 % at 500, full on the background. + assert!((f.get(2, 5)[0] - 0.55).abs() < 1e-4, "{:?}", f.get(2, 5)); + assert!((f.get(12, 5)[0] - 0.75).abs() < 1e-4); + assert!((f.get(17, 1)[0] - 1.0).abs() < 1e-4 && f.get(17, 1)[1] < 1e-4); + assert!((dof_radius(100.0, 100.0, 0.0, 10.0, 50.0)).abs() < 1e-6); + assert!((dof_radius(200.0, 100.0, 0.0, 10.0, 50.0) - 10.0).abs() < 1e-6); + assert_eq!(dof_radius(140.0, 100.0, 100.0, 10.0, 50.0), 0.0); + // A sharp edge at the far object blurs when focused near. + let mut pic = Image::new(20, 10); + for y in 0..10 { + for x in 0..20 { + let v = if x % 2 == 0 { 1.0 } else { 0.0 }; + pic.set(x, y, [v, v, v, 1.0]); + } + } + let env = EffectEnv { host: Some(&h), ..Default::default() }; + let d = run_fx("ec.3d.depthoffield", &[("focalPlane", num(100.0)), ("maximumRadius", num(4.0))], pic.clone(), 0.0, env).img; + assert_eq!(d.get(4, 5), pic.get(4, 5)); + assert!((d.get(12, 5)[0] - 0.5).abs() < 0.3 && d.get(12, 5) != pic.get(12, 5)); + } + + #[test] + fn extractor_maps_named_channels() { + let h = scene(); + let o = run("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6); + assert!((o.get(2, 5)[1] - 0.0005).abs() < 1e-6); + // Without aux: the layer's own channels can be swapped. + let s = run_fx("ec.3d.extractor", &[("red", Value::Str("A".into()))], Image::filled(2, 2, [0.2, 0.3, 0.4, 1.0]), 0.0, EffectEnv::default()); + assert_eq!(s.img.get(0, 0), [1.0, 0.3, 0.4, 1.0]); + } +} diff --git a/crates/effects/src/distort4.rs b/crates/effects/src/distort4.rs new file mode 100644 index 00000000..4e6b5040 --- /dev/null +++ b/crates/effects/src/distort4.rs @@ -0,0 +1,719 @@ +//! Distort: CC Flo Motion, Liquify and Rolling Shutter Repair. +//! +//! * **CC Flo Motion** — two knots that pull the image toward themselves (positive amount) or +//! push it away (negative), with a falloff exponent, optional tiling and supersampled +//! antialiasing. +//! * **Liquify** — brush strokes build a displacement mesh (a grid of layer-space offsets). The +//! strokes are the effect's data (the hidden `distortionMesh` text, one stroke per line, so it +//! animates with hold keyframes and is written by the `liquify.stroke` command); replaying +//! them is deterministic and cached. Tools: warp, turbulence, twirl clockwise / counter- +//! clockwise, pucker, bloat, shift pixels, reflection, clone, reconstruction and freeze / thaw. +//! Distortion Percentage scales the mesh and Distortion Mesh Offset moves it. +//! * **Rolling Shutter Repair** — every scanline of a CMOS frame is exposed a little later than +//! the previous one, so moving content shears. Motion is measured between the neighbouring +//! frames with pyramidal Lucas–Kanade (`effectcraft-track`) on a grid of points; each pixel is +//! then resampled from where it was at the frame's mid-exposure time: +//! `out(p) = in(p + v(p)·rate·(s(p) − ½))`, with `s` the scan position (0…1 along the scan +//! direction) and `v` the per-frame velocity (Warp: one smooth velocity per scanline band; +//! Pixel Motion: a dense interpolated field). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use rayon::prelude::*; + +use crate::util::{hash1, point_in_poly}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Distort", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Flo Motion + +fn flo_motion(ctx: &EffectCtx, mut b: Buf) -> Buf { + let knots = [(ctx.params.v2("knot1"), ctx.params.f("amount1") / 100.0), (ctx.params.v2("knot2"), ctx.params.f("amount2") / 100.0)]; + if knots.iter().all(|k| k.1 == 0.0) { + return b; + } + let tile = ctx.params.b("tileEdges"); + let ss = match ctx.params.e("antialiasing") { + 0 => 1, + 1 => 2, + _ => 3, + }; + let falloff = ctx.params.f("falloff").max(0.01); + let (lw, lh) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + let radius = (lw * lw + lh * lh).sqrt() * 0.5; + let src = b.img.clone(); + let (scale, off) = (b.scale, b.offset); + let map = |lx: f64, ly: f64| -> (f64, f64) { + let (mut x, mut y) = (lx, ly); + for (k, a) in knots { + if a == 0.0 { + continue; + } + let (dx, dy) = (x - k[0], y - k[1]); + let r = (dx * dx + dy * dy).sqrt() / radius; + // Weight 1 at the knot, falling off with distance. + let w = (1.0 - r.min(1.0)).powf(falloff); + // Pull: sample closer to the knot (content flows out of it); push: farther. + let s = (1.0 - a.clamp(-4.0, 0.95) * w).max(0.02); + x = k[0] + dx * s; + y = k[1] + dy * s; + } + (x, y) + }; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 4]; + for sy in 0..ss { + for sx in 0..ss { + let px = x as f64 + (sx as f64 + 0.5) / ss as f64; + let py = y as f64 + (sy as f64 + 0.5) / ss as f64; + let (lx, ly) = map((px - off[0]) / scale, (py - off[1]) / scale); + let (lx, ly) = if tile { (lx.rem_euclid(lw), ly.rem_euclid(lh)) } else { (lx, ly) }; + let q = src.sample_bilinear(lx * scale + off[0], ly * scale + off[1]); + for c in 0..4 { + acc[c] += q[c]; + } + } + } + let n = (ss * ss) as f32; + *o = acc.map(|v| v / n); + } + }); + b +} + +// ---------------------------------------------------------------- Liquify + +/// Liquify tools (the stroke text uses these names). +pub const LIQUIFY_TOOLS: [&str; 12] = [ + "warp", + "turbulence", + "twirlClockwise", + "twirlCounterclockwise", + "pucker", + "bloat", + "shiftPixels", + "reflection", + "clone", + "reconstruction", + "freeze", + "thaw", +]; + +/// One recorded Liquify stroke. +#[derive(Clone, Debug, PartialEq)] +pub struct LiquifyStroke { + pub tool: usize, + /// Brush size (diameter) and pressure (0–100), turbulent jitter (0–100), layer pixels. + pub size: f64, + pub pressure: f64, + pub jitter: f64, + /// Clone offset (source − destination). + pub clone_offset: [f64; 2], + pub points: Vec<[f64; 2]>, +} + +impl LiquifyStroke { + /// Text form: `tool size pressure jitter cloneX cloneY x,y x,y …`. + pub fn to_line(&self) -> String { + let mut s = format!("{} {} {} {} {} {}", LIQUIFY_TOOLS[self.tool], self.size, self.pressure, self.jitter, self.clone_offset[0], self.clone_offset[1]); + for p in &self.points { + s.push_str(&format!(" {},{}", p[0], p[1])); + } + s + } + pub fn parse(line: &str) -> Option { + let mut it = line.split_whitespace(); + let name = it.next()?; + let tool = LIQUIFY_TOOLS.iter().position(|t| t.eq_ignore_ascii_case(name))?; + let mut f = || -> Option { it.next()?.parse().ok() }; + let (size, pressure, jitter, cx, cy) = (f()?, f()?, f()?, f()?, f()?); + let points: Vec<[f64; 2]> = it + .filter_map(|t| { + let (a, b) = t.split_once(',')?; + Some([a.parse().ok()?, b.parse().ok()?]) + }) + .collect(); + (!points.is_empty()).then_some(LiquifyStroke { tool, size, pressure, jitter, clone_offset: [cx, cy], points }) + } +} + +/// Parse the `distortionMesh` text into strokes (bad lines are skipped). +pub fn parse_strokes(s: &str) -> Vec { + s.lines().filter_map(LiquifyStroke::parse).collect() +} + +/// Displacement mesh: offsets (layer pixels) at grid nodes spaced `cell` apart. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub nx: usize, + pub ny: usize, + pub cell: f64, + pub d: Vec<[f64; 2]>, + pub freeze: Vec, +} + +impl Mesh { + fn new(w: f64, h: f64, cell: f64) -> Mesh { + let nx = (w / cell).ceil() as usize + 1; + let ny = (h / cell).ceil() as usize + 1; + Mesh { nx, ny, cell, d: vec![[0.0; 2]; nx * ny], freeze: vec![0.0; nx * ny] } + } + /// Bilinear displacement at a layer point (zero outside the mesh). + pub fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = x / self.cell; + let fy = y / self.cell; + if fx < 0.0 || fy < 0.0 || fx > (self.nx - 1) as f64 || fy > (self.ny - 1) as f64 { + return [0.0; 2]; + } + let (x0, y0) = ((fx.floor() as usize).min(self.nx - 2), (fy.floor() as usize).min(self.ny - 2)); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.d[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +/// Replay strokes into a displacement mesh over a `w`×`h` layer. `frozen(x, y)` reports +/// mask-frozen points (Freeze Area Mask). +pub fn build_mesh(strokes: &[LiquifyStroke], w: f64, h: f64, frozen: &(dyn Fn(f64, f64) -> bool + Sync)) -> Mesh { + let cell = (w.max(h) / 128.0).clamp(2.0, 16.0); + let mut m = Mesh::new(w, h, cell); + for (i, f) in m.freeze.iter_mut().enumerate() { + let (x, y) = ((i % m.nx) as f64 * cell, (i / m.nx) as f64 * cell); + if frozen(x, y) { + *f = 1.0; + } + } + for (si, s) in strokes.iter().enumerate() { + let r = (s.size * 0.5).max(1.0); + let pr = (s.pressure / 100.0).clamp(0.0, 1.0); + // Dabs every r/4 along the stroke; each dab knows the local motion of the brush. + let mut dabs: Vec<([f64; 2], [f64; 2])> = vec![]; + if s.points.len() == 1 { + dabs.push((s.points[0], [0.0; 2])); + } + for seg in s.points.windows(2) { + let (a, b) = (seg[0], seg[1]); + let len = ((b[0] - a[0]).powi(2) + (b[1] - a[1]).powi(2)).sqrt(); + let n = ((len / (r * 0.25)).ceil() as usize).max(1); + let step = [(b[0] - a[0]) / n as f64, (b[1] - a[1]) / n as f64]; + for k in 1..=n { + dabs.push(([a[0] + step[0] * k as f64, a[1] + step[1] * k as f64], step)); + } + } + for (di, (c, mv)) in dabs.iter().enumerate() { + let (gx0, gx1) = (((c[0] - r) / cell).floor().max(0.0) as usize, (((c[0] + r) / cell).ceil() as usize).min(m.nx - 1)); + let (gy0, gy1) = (((c[1] - r) / cell).floor().max(0.0) as usize, (((c[1] + r) / cell).ceil() as usize).min(m.ny - 1)); + let prev = m.d.clone(); + for gy in gy0..=gy1 { + for gx in gx0..=gx1 { + let (x, y) = (gx as f64 * cell, gy as f64 * cell); + let (dx, dy) = (x - c[0], y - c[1]); + let dist = (dx * dx + dy * dy).sqrt(); + if dist >= r { + continue; + } + let i = gy * m.nx + gx; + // Smooth brush falloff. + let fall = { + let t = 1.0 - dist / r; + t * t * (3.0 - 2.0 * t) + }; + let tool = LIQUIFY_TOOLS[s.tool]; + if tool == "freeze" || tool == "thaw" { + let v = pr * fall; + m.freeze[i] = if tool == "freeze" { m.freeze[i].max(v) } else { (m.freeze[i] - v).max(0.0) }; + continue; + } + let w = pr * fall * (1.0 - m.freeze[i]) * 0.5; + if w <= 0.0 { + continue; + } + let d = &mut m.d[i]; + match tool { + "warp" => { + d[0] -= mv[0] * w * 2.0; + d[1] -= mv[1] * w * 2.0; + } + "turbulence" => { + let j = s.jitter / 100.0 * r * 0.1; + let seed = (si as u32).wrapping_mul(7919) ^ di as u32; + d[0] += (hash1(gx as u32, gy as u32, seed) as f64 * 2.0 - 1.0) * j * w; + d[1] += (hash1(gx as u32, gy as u32, seed ^ 0x55) as f64 * 2.0 - 1.0) * j * w; + d[0] -= mv[0] * w; + d[1] -= mv[1] * w; + } + "twirlClockwise" | "twirlCounterclockwise" => { + let sgn = if tool == "twirlClockwise" { -1.0 } else { 1.0 }; + let a = sgn * 0.08 * w; + let (sn, cs) = a.sin_cos(); + d[0] += dx * cs - dy * sn - dx; + d[1] += dx * sn + dy * cs - dy; + } + "pucker" => { + d[0] += dx * 0.05 * w; + d[1] += dy * 0.05 * w; + } + "bloat" => { + d[0] -= dx * 0.05 * w; + d[1] -= dy * 0.05 * w; + } + "shiftPixels" => { + // Perpendicular to the stroke (to its left). + d[0] -= mv[1] * w * 2.0; + d[1] += mv[0] * w * 2.0; + } + "reflection" => { + // Mirror across the stroke line through the dab. + let l = (mv[0] * mv[0] + mv[1] * mv[1]).sqrt(); + if l > 1e-9 { + let n = [-mv[1] / l, mv[0] / l]; + let k = dx * n[0] + dy * n[1]; + let tgt = [-2.0 * k * n[0], -2.0 * k * n[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + } + "clone" => { + // Copy the distortion found at the clone source. + let sx = x + s.clone_offset[0]; + let sy = y + s.clone_offset[1]; + let fx = (sx / cell).round().clamp(0.0, (m.nx - 1) as f64) as usize; + let fy = (sy / cell).round().clamp(0.0, (m.ny - 1) as f64) as usize; + let src = prev[fy * m.nx + fx]; + let tgt = [src[0] + s.clone_offset[0], src[1] + s.clone_offset[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + // Reconstruction: relax toward no distortion. + _ => { + d[0] *= 1.0 - w; + d[1] *= 1.0 - w; + } + } + } + } + } + } + m +} + +type MeshCache = Mutex>>; + +fn cached_mesh(ctx: &EffectCtx) -> Arc { + use std::hash::{Hash, Hasher}; + let text = ctx.params.s("distortionMesh"); + let mask = ctx.params.f("freezeAreaMask").round() as usize; + let poly = mask.checked_sub(1).and_then(|i| ctx.env.masks.get(i)).cloned(); + let mut h = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut h); + ctx.layer_size[0].to_bits().hash(&mut h); + ctx.layer_size[1].to_bits().hash(&mut h); + if let Some(p) = &poly { + for q in &p.points { + q[0].to_bits().hash(&mut h); + q[1].to_bits().hash(&mut h); + } + p.inverted.hash(&mut h); + } + let key = h.finish(); + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(m) = cache.lock().ok().and_then(|c| c.get(&key).cloned()) { + return m; + } + let strokes = parse_strokes(text); + let frozen = |x: f64, y: f64| match &poly { + Some(p) => point_in_poly(&p.points, x, y) != p.inverted, + None => false, + }; + let m = Arc::new(build_mesh(&strokes, ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0), &frozen)); + if let Ok(mut c) = cache.lock() { + if c.len() > 32 { + c.clear(); + } + c.insert(key, m.clone()); + } + m +} + +fn liquify(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pct = ctx.params.f("distortionPercentage") / 100.0; + let show_freeze = ctx.params.b("viewFreezeAreaMask"); + let show_mesh = ctx.params.b("viewMesh"); + if ctx.params.s("distortionMesh").trim().is_empty() && !show_freeze && !show_mesh { + return b; + } + let mesh = cached_mesh(ctx); + let off = ctx.params.v2("distortionMeshOffset"); + let mesh_color = ctx.params.color("meshColor"); + let src = b.img.clone(); + let (scale, o) = (b.scale, b.offset); + let line_w = 0.6 / scale.max(1e-6); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - o[0]) / scale; + let ly = (y as f64 + 0.5 - o[1]) / scale; + let d = mesh.at(lx - off[0], ly - off[1]); + let (sx, sy) = (lx + d[0] * pct, ly + d[1] * pct); + let mut c = if d == [0.0; 2] { *px } else { src.sample_bilinear(sx * scale + o[0], sy * scale + o[1]) }; + if show_freeze { + let fx = ((lx - off[0]) / mesh.cell).round().clamp(0.0, (mesh.nx - 1) as f64) as usize; + let fy = ((ly - off[1]) / mesh.cell).round().clamp(0.0, (mesh.ny - 1) as f64) as usize; + let f = mesh.freeze[fy * mesh.nx + fx] as f32; + if f > 0.0 { + c[0] = c[0] * (1.0 - 0.5 * f) + 0.5 * f; + c[3] = c[3].max(0.5 * f); + } + } + if show_mesh { + // Grid lines every 4 mesh cells, displaced with the mesh. + let gs = mesh.cell * 4.0; + let (gx, gy) = ((lx + d[0] * pct - off[0]).rem_euclid(gs), (ly + d[1] * pct - off[1]).rem_euclid(gs)); + if gx < line_w || gy < line_w { + c = [mesh_color[0], mesh_color[1], mesh_color[2], 1.0]; + } + } + *px = c; + } + }); + b +} + +// ---------------------------------------------------------------- Rolling Shutter Repair + +/// Per-pixel velocity field (layer pixels per frame) on a coarse grid, from tracked points. +struct Flow { + nx: usize, + ny: usize, + cell: f64, + v: Vec<[f64; 2]>, +} + +impl Flow { + fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = (x / self.cell - 0.5).clamp(0.0, (self.nx - 1) as f64); + let fy = (y / self.cell - 0.5).clamp(0.0, (self.ny - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.v[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +fn median(v: &mut [f64]) -> f64 { + v.sort_by(f64::total_cmp); + v[v.len() / 2] +} + +/// Measure motion around the current frame (`cur`) from `prev` to `next` (all in the same +/// buffer grid). `per_row`: Warp method (one velocity per scanline band). +fn measure(cur: &Image, prev: Option<&Image>, next: Option<&Image>, scale: f64, detailed: bool, per_row: bool) -> Option { + use effectcraft_track::klt::{GrayPyramid, LkOpts, analysis_factor, track}; + let factor = analysis_factor(cur.width, cur.height, if detailed { 640 } else { 320 }); + let pc = GrayPyramid::from_image(cur, [0.0; 2], factor, 4); + let (w, h) = (pc.width(), pc.height()); + let n = if detailed { 24 } else { 12 }; + let cell_w = w as f64 / n as f64; + let rows = ((n as f64) * h as f64 / w.max(1) as f64).round().max(2.0) as usize; + let cell_h = h as f64 / rows as f64; + let pts: Vec<[f64; 2]> = (0..rows).flat_map(|j| (0..n).map(move |i| [(i as f64 + 0.5) * cell_w, (j as f64 + 0.5) * cell_h])).collect(); + let o = LkOpts { radius: 4, fb_max: 1.0, ..Default::default() }; + let go = |other: Option<&Image>| other.map(|img| track(&pc, &GrayPyramid::from_image(img, [0.0; 2], factor, 4), &pts, None, &o)); + let (tn, tp) = (go(next), go(prev)); + // Velocity in plane pixels per frame. + let vel: Vec> = (0..pts.len()) + .map(|i| { + let p = pts[i]; + let fw = tn.as_ref().and_then(|t| t[i]).map(|q| [q[0] - p[0], q[1] - p[1]]); + let bw = tp.as_ref().and_then(|t| t[i]).map(|q| [p[0] - q[0], p[1] - q[1]]); + match (fw, bw) { + (Some(a), Some(b)) => Some([(a[0] + b[0]) * 0.5, (a[1] + b[1]) * 0.5]), + (a, b) => a.or(b), + } + }) + .collect(); + let valid: Vec<[f64; 2]> = vel.iter().flatten().copied().collect(); + if valid.is_empty() { + return None; + } + let mut vx: Vec = valid.iter().map(|v| v[0]).collect(); + let mut vy: Vec = valid.iter().map(|v| v[1]).collect(); + let global = [median(&mut vx), median(&mut vy)]; + let k = factor as f64 / scale.max(1e-9); + let mut v = vec![[0.0; 2]; n * rows]; + for j in 0..rows { + // Row (band) median for Warp; per point (outliers replaced) for Pixel Motion. + let band: Vec<[f64; 2]> = (0..n).filter_map(|i| vel[j * n + i]).collect(); + let row_med = if band.is_empty() { + global + } else { + let mut a: Vec = band.iter().map(|v| v[0]).collect(); + let mut b: Vec = band.iter().map(|v| v[1]).collect(); + [median(&mut a), median(&mut b)] + }; + for i in 0..n { + let s = if per_row { + row_med + } else { + match vel[j * n + i] { + Some(q) if (q[0] - row_med[0]).abs() + (q[1] - row_med[1]).abs() < 3.0 + 0.5 * (row_med[0].abs() + row_med[1].abs()) => q, + _ => row_med, + } + }; + v[j * n + i] = [s[0] * k, s[1] * k]; + } + } + // Cell size in layer pixels (grid is uniform in plane pixels; assume square plane pixels). + Some(Flow { nx: n, ny: rows, cell: cell_w * k, v }) +} + +fn same_grid(nb: Buf, b: &Buf) -> Image { + if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + return nb.img; + } + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) +} + +fn rolling_shutter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let rate = ctx.params.f("rollingShutterRate") / 100.0; + if rate == 0.0 { + return b; + } + let Some(host) = ctx.env.host else { return b }; + let fps = ctx.fps(); + let prev = host.self_at(ctx.time - 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + let next = host.self_at(ctx.time + 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + if prev.is_none() && next.is_none() { + return b; + } + let per_row = ctx.params.e("method") == 0; + let Some(flow) = measure(&b.img, prev.as_ref(), next.as_ref(), 1.0, ctx.params.b("detailedAnalysis"), per_row) else { return b }; + let dir = ctx.params.e("scanDirection"); + let src = b.img.clone(); + let (w, h) = (b.img.width as f64, b.img.height as f64); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let (fx, fy) = (x as f64 + 0.5, y as f64 + 0.5); + let s = match dir { + 1 => 1.0 - fy / h, + 2 => fx / w, + 3 => 1.0 - fx / w, + _ => fy / h, + }; + let tau = rate * (s - 0.5); + let v = flow.at(fx, fy); + *px = src.sample_bilinear_clamped(fx + v[0] * tau, fy + v[1] * tau); + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.distort.ccflomotion", + "CC Flo Motion", + vec![ + p("knot1", "Knot 1", Value::Vec2([0.25, 0.5]), ParamUi::Point), + p("amount1", "Amount 1", num(50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("knot2", "Knot 2", Value::Vec2([0.75, 0.5]), ParamUi::Point), + p("amount2", "Amount 2", num(-50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("tileEdges", "Tile Edges", Value::Bool(false), ParamUi::Checkbox), + p("antialiasing", "Antialiasing", Value::Enum(1), popup(&["Low", "Medium", "High"])), + p("falloff", "Falloff", num(2.0), slider(0.01, 10.0, 0.1, 5.0, 2)), + ], + flo_motion, + ), + spec( + "ec.distort.liquify", + "Liquify", + vec![ + p("tool", "Tool", Value::Enum(0), popup(&LIQUIFY_TOOLS)), + p("brushSize", "Brush Size", num(64.0), slider(1.0, 600.0, 1.0, 600.0, 0)), + p("brushPressure", "Brush Pressure", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("freezeAreaMask", "Freeze Area Mask", num(0.0), ParamUi::Mask), + p("turbulentJitter", "Turbulent Jitter", num(70.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("cloneOffset", "Clone Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("reconstructionMode", "Reconstruction Mode", Value::Enum(0), popup(&["Revert", "Displace", "Amplitwist", "Affine"])), + p("viewFreezeAreaMask", "View Freeze Area Mask", Value::Bool(false), ParamUi::Checkbox), + p("viewMesh", "View Mesh", Value::Bool(false), ParamUi::Checkbox), + p("meshSize", "Mesh Size", Value::Enum(1), popup(&["Small", "Medium", "Large"])), + p("meshColor", "Mesh Color", col(0.5, 0.5, 0.5), ParamUi::Color), + // The strokes (see `LiquifyStroke`); written by the `liquify.stroke` command. + p("distortionMesh", "Distortion Mesh", Value::Str(String::new()), ParamUi::Hidden), + p("distortionMeshOffset", "Distortion Mesh Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("distortionPercentage", "Distortion Percentage", num(100.0), slider(0.0, 200.0, 0.0, 200.0, 1)), + p("distortionDisplayQuality", "Distortion Display Quality", Value::Enum(1), popup(&["Draft", "Smooth"])), + ], + liquify, + ), + spec( + "ec.distort.rollingshutterrepair", + "Rolling Shutter Repair", + vec![ + p("rollingShutterRate", "Rolling Shutter Rate", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scanDirection", "Scan Direction", Value::Enum(0), popup(&["Top → Bottom", "Bottom → Top", "Left → Right", "Right → Left"])), + p("method", "Method", Value::Enum(0), popup(&["Warp", "Pixel Motion"])), + p("detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("pixelMotionDetail", "Pixel Motion Detail", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + rolling_shutter, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn grid_img(w: u32, h: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if x % 8 == 0 || y % 8 == 0 { 1.0 } else { 0.2 }; + [v, v * 0.5, x as f32 / w as f32, 1.0] + }) + } + + #[test] + fn flo_motion_pulls_and_pushes() { + let img = grid_img(64, 64); + let out = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // The knot itself is fixed; pixels near it are magnified (sampled closer to the knot). + assert!((out.img.get(32, 32)[2] - img.get(32, 32)[2]).abs() < 0.01); + assert!((out.img.get(40, 32)[2] - img.get(40, 32)[2]).abs() > 0.01); + assert!(out.img.get(40, 32)[2] < img.get(40, 32)[2]); + let none = run_fx("ec.distort.ccflomotion", &[("amount1", num(0.0)), ("amount2", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + let again = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img, + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + + #[test] + fn liquify_strokes_round_trip_and_warp() { + let s = LiquifyStroke { tool: 0, size: 30.0, pressure: 80.0, jitter: 70.0, clone_offset: [0.0, 0.0], points: vec![[20.0, 32.0], [40.0, 32.0]] }; + assert_eq!(LiquifyStroke::parse(&s.to_line()), Some(s.clone())); + assert!(LiquifyStroke::parse("nope 1 2 3 4 5 1,2").is_none()); + let img = grid_img(64, 64); + let out = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_ne!(out.img.data, img.data); + // Warping to the right: a pixel right of the start now shows content from its left. + let at = |i: &Image| i.get(36, 32)[2]; + assert!(at(&out.img) < at(&img), "{} vs {}", at(&out.img), at(&img)); + // Far away, untouched. + assert_eq!(out.img.get(5, 5), img.get(5, 5)); + // Distortion Percentage 0 = identity; deterministic. + let zero = run_fx( + "ec.distort.liquify", + &[("distortionMesh", Value::Str(s.to_line())), ("distortionPercentage", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(zero.img.data, img.data); + let again = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + // Reconstruction undoes the warp; freeze protects. + let mut rec = s.clone(); + rec.tool = 9; + rec.pressure = 100.0; + let lines = format!("{}\n{}\n{}\n{}", s.to_line(), rec.to_line(), rec.to_line(), rec.to_line()); + let m1 = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| false); + let m2 = build_mesh(&parse_strokes(&lines), 64.0, 64.0, &|_, _| false); + let mag = |m: &Mesh| m.d.iter().map(|d| d[0].abs() + d[1].abs()).sum::(); + assert!(mag(&m2) < mag(&m1) * 0.5); + let frozen = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| true); + assert_eq!(mag(&frozen), 0.0); + // Every tool changes (or deliberately keeps) the mesh without NaNs. + for t in 0..LIQUIFY_TOOLS.len() { + let st = LiquifyStroke { tool: t, clone_offset: [10.0, 0.0], ..s.clone() }; + let m = build_mesh(&[s.clone(), st], 64.0, 64.0, &|_, _| false); + assert!(m.d.iter().all(|d| d[0].is_finite() && d[1].is_finite()), "{}", LIQUIFY_TOOLS[t]); + } + } + + /// A textured frame panning right at 3 px per frame, captured with a rolling shutter: row + /// `y` is exposed `(y / h − ½)` frames late (rate 100 %). + fn texture(x: f64, y: f64) -> f32 { + let a = ((x * 0.31).sin() * (y * 0.23).cos() + (x * 0.07 + y * 0.11).sin()) as f32; + 0.5 + 0.25 * a + } + fn rs_frame(f: f64, shutter: bool) -> Image { + let (w, h) = (96u32, 64u32); + crate::util::gen_image(w, h, |x, y| { + let tau = if shutter { (y as f64 + 0.5) / h as f64 - 0.5 } else { 0.0 }; + let shift = 3.0 * (f + tau); + let v = texture(x as f64 + 0.5 - shift, y as f64 + 0.5); + [v, v, v, 1.0] + }) + } + struct Pan; + impl EffectHost for Pan { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: rs_frame(t * 24.0, true), offset: [0.0; 2], scale: 1.0 }) + } + } + + #[test] + fn rolling_shutter_repair_unskews_a_pan() { + let host = Pan; + let env = EffectEnv { host: Some(&host), frame_rate: 24.0, ..Default::default() }; + let t = 5.0 / 24.0; + let skewed = rs_frame(5.0, true); + let truth = rs_frame(5.0, false); + let err = |a: &Image| { + let mut s = 0.0; + let mut n = 0.0; + for y in 4..60 { + for x in 12..84 { + s += (a.get(x, y)[0] - truth.get(x, y)[0]).abs(); + n += 1.0; + } + } + s / n + }; + for method in [0u32, 1] { + let out = run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert!(err(&out.img) < err(&skewed) * 0.4, "method {method}: {} vs {}", err(&out.img), err(&skewed)); + let again = + run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert_eq!(out.img.data, again.img.data); + } + let none = run_fx("ec.distort.rollingshutterrepair", &[], skewed.clone(), t, EffectEnv::default()); + assert_eq!(none.img.data, skewed.data); + } +} diff --git a/crates/effects/src/keylight.rs b/crates/effects/src/keylight.rs new file mode 100644 index 00000000..980a6169 --- /dev/null +++ b/crates/effects/src/keylight.rs @@ -0,0 +1,455 @@ +//! Key Light — a full screen-colour keyer with the controls of After Effects' bundled +//! "Keylight (1.2)" (that name is a trademark of its vendor, so the effect is registered as +//! **Key Light**; `lookup("Keylight (1.2)")` finds it through [`KEYLIGHT_ALIASES`]). +//! +//! The method is the textbook *screen difference* key, written from the public description of +//! the controls: +//! +//! * colours are first neutralised by the **Alpha Bias** / **Despill Bias** colours (each channel +//! scaled so the bias colour becomes grey — mid grey = no change); +//! * the **screen difference** of a pixel is its screen-primary channel minus a balance-weighted +//! mix of the other two (**Screen Balance**); the raw matte is +//! `1 − Screen Gain × diff(pixel) / diff(screen colour)`; +//! * the **Screen Matte** is then clipped (Clip Black / White, with Clip Rollback restoring edge +//! detail), despotted (morphological open/close: Despot Black / White), shrunk or grown and +//! softened, combined with the **Inside** / **Outside** masks and the source alpha; +//! * the foreground is the source with the screen colour removed in proportion to the +//! transparency (`(c − (1 − α)·S) / α`), then **despilled** (the primary is limited to the +//! balanced secondary), with **Replace Method** colouring areas whose alpha was raised by +//! clipping or the inside mask, and optional foreground and edge colour correction. +//! +//! Views: Source, Source Alpha, Corrected Source, Colour Correction Edges, Screen Matte, Inside +//! Mask, Outside Mask, Combined Matte, Status, Intermediate Result and Final Result. + +use effectcraft_color::luminance; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, gaussian_blur}; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, morph_frac, point_in_poly, premul, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +/// Other names the effect answers to in [`crate::lookup`]. +pub const KEYLIGHT_ALIASES: &[&str] = &["Keylight (1.2)", "Keylight"]; + +pub const VIEWS: [&str; 11] = [ + "Source", + "Source Alpha", + "Corrected Source", + "Colour Correction Edges", + "Screen Matte", + "Inside Mask", + "Outside Mask", + "Combined Matte", + "Status", + "Intermediate Result", + "Final Result", +]; + +const REPLACE: [&str; 4] = ["None", "Source", "Hard Colour", "Soft Colour"]; + +fn pct(d: f64) -> (Value, ParamUi) { + (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)) +} + +fn mask_popup() -> ParamUi { + popup(&["None", "Mask 1", "Mask 2", "Mask 3", "Mask 4", "Mask 5", "Mask 6", "Mask 7", "Mask 8", "Mask 9", "Mask 10"]) +} + +/// Index of the screen's primary channel and the two others (larger-in-screen first). +fn primary(s: [f32; 3]) -> (usize, usize, usize) { + let pi = if s[1] >= s[0] && s[1] >= s[2] { + 1 + } else if s[2] >= s[0] { + 2 + } else { + 0 + }; + let (a, b) = match pi { + 0 => (1, 2), + 1 => (0, 2), + _ => (0, 1), + }; + if s[a] >= s[b] { (pi, a, b) } else { (pi, b, a) } +} + +/// Screen difference: primary minus the balance-weighted secondaries. +#[inline] +pub fn screen_diff(c: [f32; 3], pi: usize, o1: usize, o2: usize, bal: f32) -> f32 { + c[pi] - (bal * c[o1] + (1.0 - bal) * c[o2]) +} + +/// Scale channels so `bias` becomes neutral (grey keeps everything as is). +#[inline] +fn neutralise(c: [f32; 3], bias: [f32; 3]) -> [f32; 3] { + let l = luminance(bias[0], bias[1], bias[2]).max(1e-4); + [0, 1, 2].map(|i| c[i] * l / bias[i].max(1e-4)) +} + +fn mask_coverage(ctx: &EffectCtx, b: &Buf, idx: u32, softness: f64, invert: bool) -> Option { + let shape = ctx.env.masks.get((idx as usize).checked_sub(1)?)?; + let (w, h) = (b.img.width as usize, b.img.height as usize); + let mut pl = Plane::new(w, h); + let inv = 1.0 / b.scale.max(1e-9); + pl.data.par_chunks_mut(w.max(1)).enumerate().for_each(|(y, row)| { + for (x, v) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - b.offset[0]) * inv; + let ly = (y as f64 + 0.5 - b.offset[1]) * inv; + *v = if point_in_poly(&shape.points, lx, ly) != shape.inverted { 1.0 } else { 0.0 }; + } + }); + let s = softness * b.scale; + if s > 0.0 { + pl = gauss_plane(&pl, s * 0.5, s * 0.5); + } + if invert { + pl = pl.map(|v| 1.0 - v); + } + Some(pl) +} + +/// Saturation / contrast / brightness correction of a straight colour. +fn correct(c: [f32; 3], sat: f32, contrast: f32, bright: f32) -> [f32; 3] { + let l = luminance(c[0], c[1], c[2]); + let c = c.map(|v| l + (v - l) * sat); + c.map(|v| ((v - 0.5) * contrast + 0.5) * bright) +} + +fn key_light(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let view = pr.e("view"); + if view == 0 { + return b; + } + let sc = pr.color("screenColour"); + let screen = [sc[0], sc[1], sc[2]]; + let (pi, o1, o2) = primary(screen); + let gain = pr.f("screenGain") as f32 / 100.0; + let bal = pr.f("screenBalance") as f32 / 100.0; + let dbias = pr.color("despillBias"); + let abias = if pr.b("lockBiasesTogether") { dbias } else { pr.color("alphaBias") }; + let (dbias, abias) = ([dbias[0], dbias[1], dbias[2]], [abias[0], abias[1], abias[2]]); + let s_alpha = neutralise(screen, abias); + let sd = screen_diff(s_alpha, pi, o1, o2, bal).max(1e-3); + let cb = pr.f("clipBlack") as f32 / 100.0; + let cw = pr.f("clipWhite") as f32 / 100.0; + let rollback = pr.f("clipRollback") * b.scale; + let preblur = pr.f("screenPreblur") * b.scale; + let sg = pr.f("screenShrinkGrow") * b.scale; + let soft = pr.f("screenSoftness") * b.scale; + let despot_b = pr.f("screenDespotBlack") * b.scale; + let despot_w = pr.f("screenDespotWhite") * b.scale; + + let src = b.img.clone(); + let blurred; + let key_src: &Image = if preblur > 0.0 { + blurred = gaussian_blur(&src, preblur * 0.5, preblur * 0.5, true); + &blurred + } else { + &src + }; + // Raw screen matte (opacity of the foreground). + let raw = Plane::from_image(key_src, |px| { + let (c, _) = unpremul(px); + let d = screen_diff(neutralise(c, abias), pi, o1, o2, bal); + (1.0 - gain * d / sd).clamp(0.0, 1.0) + }); + let clip = |v: f32| ((v - cb) / (cw - cb).max(1e-4)).clamp(0.0, 1.0); + let mut m = raw.map(clip); + if rollback > 0.0 { + // Restore edge detail lost by clipping within `rollback` pixels of the clipped matte. + let grown = morph_frac(&m, rollback, true); + let shrunk = morph_frac(&m, rollback, false); + m = Plane { + w: m.w, + h: m.h, + data: (0..m.data.len()) + .into_par_iter() + .map(|i| if grown.data[i] > shrunk.data[i] + 1e-4 { m.data[i].max(raw.data[i]).min(grown.data[i]) } else { m.data[i] }) + .collect(), + }; + } + if despot_b > 0.0 { + // Fill small holes: close (dilate then erode). + m = morph_frac(&morph_frac(&m, despot_b, true), despot_b, false); + } + if despot_w > 0.0 { + // Remove small specks: open (erode then dilate). + m = morph_frac(&morph_frac(&m, despot_w, false), despot_w, true); + } + if sg < 0.0 { + m = morph_frac(&m, -sg, false); + } else if sg > 0.0 { + m = morph_frac(&m, sg, true); + } + if soft > 0.0 { + m = gauss_plane(&m, soft * 0.5, soft * 0.5); + } + let screen_matte = m.clone(); + let inside = mask_coverage(ctx, &b, pr.e("insideMask"), pr.f("insideMaskSoftness"), pr.b("invertInsideMask")); + let outside = mask_coverage(ctx, &b, pr.e("outsideMask"), pr.f("outsideMaskSoftness"), pr.b("invertOutsideMask")); + let src_alpha_mode = pr.e("sourceAlpha"); + let n = src.data.len(); + let combined: Vec = (0..n) + .into_par_iter() + .map(|i| { + let mut a = screen_matte.data[i]; + let sa = src.data[i][3].clamp(0.0, 1.0); + let ins = inside.as_ref().map_or(0.0, |p| p.data[i]); + let ins = if src_alpha_mode == 1 { ins.max(sa) } else { ins }; + a = a.max(ins); + if let Some(o) = &outside { + a *= 1.0 - o.data[i]; + } + if src_alpha_mode == 2 { + a *= sa; + } + a.clamp(0.0, 1.0) + }) + .collect(); + let edges: Option> = pr.b("enableEdgeColourCorrection").then(|| { + let grow = pr.f("edgeGrow") * b.scale; + let hard = (pr.f("edgeHardness") / 100.0) as f32; + let esoft = pr.f("edgeSoftness") * b.scale; + // Edge band: where the (grown) combined matte is partial. + let cm = Plane { w: m.w, h: m.h, data: combined.clone() }; + let band = cm.map(|a| if a > 1e-3 && a < 0.999 { 1.0 } else { 0.0 }); + let mut band = if grow > 0.0 { morph_frac(&band, grow, true) } else { band }; + if esoft > 0.0 { + band = gauss_plane(&band, esoft * 0.5, esoft * 0.5); + } + band.data.iter().map(|v| (v * (1.0 + hard * 4.0)).min(1.0)).collect() + }); + let replace = pr.e("replaceMethod"); + let rc = pr.color("replaceColour"); + let in_replace = pr.e("insideReplaceMethod"); + let in_rc = pr.color("insideReplaceColour"); + let fg_cc = pr.b("enableColourCorrection"); + let (fs, fc, fbr) = (pr.f("saturation") as f32 / 100.0, pr.f("contrast") as f32 / 100.0 + 1.0, pr.f("brightness") as f32 / 100.0 + 1.0); + let (es, ec, ebr) = (pr.f("edgeSaturation") as f32 / 100.0, pr.f("edgeContrast") as f32 / 100.0 + 1.0, pr.f("edgeBrightness") as f32 / 100.0 + 1.0); + let unpremultiply = pr.b("unpremultiplyResult"); + let crops = [pr.f("cropLeft"), pr.f("cropRight"), pr.f("cropTop"), pr.f("cropBottom")].map(|v| v / 100.0); + let (lw, lh) = (ctx.layer_size[0], ctx.layer_size[1]); + let w = b.img.width as usize; + let (scale, off) = (b.scale, b.offset); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let (c, _) = unpremul(src.data[i]); + let a = combined[i]; + let sm = screen_matte.data[i]; + // Source crops (layer fractions from each edge). + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let (lx, ly) = ((x - off[0]) / scale / lw.max(1e-9), (y - off[1]) / scale / lh.max(1e-9)); + let cropped = lx < crops[0] || lx > 1.0 - crops[1] || ly < crops[2] || ly > 1.0 - crops[3]; + // Screen removal on the raw (unclipped) transparency, then despill. + let ar = raw.data[i].max(1e-4); + let mut fg = [0, 1, 2].map(|k| ((c[k] - (1.0 - ar) * screen[k]) / ar).max(0.0)); + let nfg = neutralise(fg, dbias); + let spill = screen_diff(nfg, pi, o1, o2, bal).max(0.0); + let l0 = (luminance(dbias[0], dbias[1], dbias[2]) / dbias[pi].max(1e-4)).max(1e-4); + fg[pi] -= spill / l0; + let intermediate = fg; + // Replace colour where the alpha was raised beyond the raw screen matte. + let raised = (sm - raw.data[i]).max(0.0); + let raised_in = inside.as_ref().map_or(0.0, |p| (p.data[i] - sm).max(0.0)); + let apply_replace = |fg: [f32; 3], method: u32, rc: [f32; 4], amt: f32| -> [f32; 3] { + if amt <= 0.0 { + return fg; + } + match method { + 1 => [0, 1, 2].map(|k| fg[k] + (c[k] - fg[k]) * amt), + 2 => [rc[0], rc[1], rc[2]], + 3 => { + let l = luminance(c[0], c[1], c[2]) / luminance(rc[0], rc[1], rc[2]).max(1e-4); + [0, 1, 2].map(|k| fg[k] + (rc[k] * l - fg[k]) * amt) + } + _ => fg, + } + }; + fg = apply_replace(fg, replace, rc, raised); + fg = apply_replace(fg, in_replace, in_rc, raised_in); + if fg_cc { + fg = correct(fg, fs, fc, fbr); + } + let edge = edges.as_ref().map_or(0.0, |e| e[i]); + if edge > 0.0 { + let ce = correct(fg, es, ec, ebr); + fg = [0, 1, 2].map(|k| fg[k] + (ce[k] - fg[k]) * edge); + } + let grey = |v: f32| [v, v, v, 1.0]; + *px = match view { + 1 => grey(src.data[i][3]), + 2 => { + let mut cs = neutralise(c, dbias); + let s2 = screen_diff(cs, pi, o1, o2, bal).max(0.0); + cs[pi] -= s2; + if fg_cc { + cs = correct(cs, fs, fc, fbr); + } + [cs[0], cs[1], cs[2], 1.0] + } + 3 => grey(edge), + 4 => grey(sm), + 5 => grey(inside.as_ref().map_or(0.0, |p| p.data[i])), + 6 => grey(outside.as_ref().map_or(0.0, |p| p.data[i])), + 7 => grey(a), + 8 => { + let v = if a <= 1e-3 { + 0.0 + } else if a >= 0.999 { + 1.0 + } else { + 0.5 + }; + grey(v) + } + 9 => premul(intermediate, sm), + _ => { + if cropped { + [0.0; 4] + } else if unpremultiply { + premul(fg, a) + } else { + // Keylight's premultiplied output read as straight colour. + premul(fg.map(|v| v * a), a) + } + } + }; + }); + b +} + +pub fn specs() -> Vec { + let pp = |id: &'static str, name: &'static str, v: (Value, ParamUi)| p(id, name, v.0, v.1); + let cc = |id: &'static str, name: &'static str| p(id, name, num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 1)); + vec![EffectSpec { + id: "ec.keying.keylight", + name: "Key Light", + category: "Keying", + params: vec![ + p("view", "View", Value::Enum(10), popup(&VIEWS)), + p("unpremultiplyResult", "Unpremultiply Result", Value::Bool(true), ParamUi::Checkbox), + p("screenColour", "Screen Colour", col(0.0, 1.0, 0.0), ParamUi::Color), + p("screenGain", "Screen Gain", num(100.0), slider(0.0, 200.0, 0.0, 200.0, 1)), + p("screenBalance", "Screen Balance", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("despillBias", "Despill Bias", col(0.5, 0.5, 0.5), ParamUi::Color), + p("alphaBias", "Alpha Bias", col(0.5, 0.5, 0.5), ParamUi::Color), + p("lockBiasesTogether", "Lock Biases Together", Value::Bool(true), ParamUi::Checkbox), + p("screenPreblur", "Screen Pre-blur", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 1)), + // Screen Matte + pp("clipBlack", "Clip Black", pct(0.0)), + pp("clipWhite", "Clip White", pct(100.0)), + p("clipRollback", "Clip Rollback", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenShrinkGrow", "Screen Shrink/Grow", num(0.0), slider(-100.0, 100.0, -10.0, 10.0, 1)), + p("screenSoftness", "Screen Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenDespotBlack", "Screen Despot Black", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenDespotWhite", "Screen Despot White", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("replaceMethod", "Replace Method", Value::Enum(3), popup(&REPLACE)), + p("replaceColour", "Replace Colour", col(0.5, 0.5, 0.5), ParamUi::Color), + // Inside Mask + p("insideMask", "Inside Mask", Value::Enum(0), mask_popup()), + p("insideMaskSoftness", "Inside Mask Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("invertInsideMask", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("insideReplaceMethod", "Inside Replace Method", Value::Enum(1), popup(&REPLACE)), + p("insideReplaceColour", "Inside Replace Colour", col(0.5, 0.5, 0.5), ParamUi::Color), + p("sourceAlpha", "Source Alpha", Value::Enum(2), popup(&["Ignore", "Add to Inside Mask", "Normal"])), + // Outside Mask + p("outsideMask", "Outside Mask", Value::Enum(0), mask_popup()), + p("outsideMaskSoftness", "Outside Mask Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("invertOutsideMask", "Invert Outside Mask", Value::Bool(false), ParamUi::Checkbox), + // Foreground Colour Correction + p("enableColourCorrection", "Enable Colour Correction", Value::Bool(false), ParamUi::Checkbox), + p("saturation", "Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + cc("contrast", "Contrast"), + cc("brightness", "Brightness"), + // Edge Colour Correction + p("enableEdgeColourCorrection", "Enable Edge Colour Correction", Value::Bool(false), ParamUi::Checkbox), + pp("edgeHardness", "Edge Hardness", pct(0.0)), + p("edgeSoftness", "Edge Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("edgeGrow", "Edge Grow", num(1.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("edgeSaturation", "Edge Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + cc("edgeContrast", "Edge Contrast"), + cc("edgeBrightness", "Edge Brightness"), + // Source Crops + pp("cropLeft", "Left", pct(0.0)), + pp("cropRight", "Right", pct(0.0)), + pp("cropTop", "Top", pct(0.0)), + pp("cropBottom", "Bottom", pct(0.0)), + ], + render: key_light, + gpu: false, + float: true, + }] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, MaskShape, run_fx}; + + /// Green screen with a red square (fg) and a half-transparent mix strip. + fn plate() -> Image { + crate::util::gen_image(40, 30, |x, y| { + if (10..20).contains(&x) && (10..20).contains(&y) { + [0.8, 0.2, 0.1, 1.0] + } else if (25..30).contains(&x) { + // 50 % red over green. + [0.4, 0.6, 0.05, 1.0] + } else { + [0.1, 0.9, 0.1, 1.0] + } + }) + } + + fn green() -> Value { + col(0.1, 0.9, 0.1) + } + + #[test] + fn keys_the_screen_and_keeps_the_foreground() { + let out = run_fx("ec.keying.keylight", &[("screenColour", green())], plate(), 0.0, EffectEnv::default()); + let bg = out.img.get(2, 2); + let fg = out.img.get(15, 15); + let mix = out.img.get(27, 5); + assert!(bg[3] < 0.02, "{bg:?}"); + assert!(fg[3] > 0.98 && (fg[0] - 0.8).abs() < 0.05, "{fg:?}"); + assert!(mix[3] > 0.3 && mix[3] < 0.7, "{mix:?}"); + // The recovered edge colour has the green removed (no spill). + let (c, _) = unpremul(mix); + assert!(c[1] <= c[0] + 0.05, "{c:?}"); + // Deterministic. + let again = run_fx("ec.keying.keylight", &[("screenColour", green())], plate(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + } + + #[test] + fn clip_gain_views_and_masks() { + // Clip White 60 %: the half-transparent strip becomes solid. + let out = run_fx("ec.keying.keylight", &[("screenColour", green()), ("clipWhite", num(40.0))], plate(), 0.0, EffectEnv::default()); + assert!(out.img.get(27, 5)[3] > 0.99); + // Screen Matte and Status views are grey-scale and opaque. + let sm = run_fx("ec.keying.keylight", &[("screenColour", green()), ("view", Value::Enum(4))], plate(), 0.0, EffectEnv::default()); + assert!(sm.img.get(2, 2)[0] < 0.02 && sm.img.get(15, 15)[0] > 0.98 && sm.img.get(2, 2)[3] == 1.0); + let st = run_fx("ec.keying.keylight", &[("screenColour", green()), ("view", Value::Enum(8))], plate(), 0.0, EffectEnv::default()); + assert_eq!(st.img.get(27, 5)[0], 0.5); + // Source view is the identity. + let src = run_fx("ec.keying.keylight", &[("view", Value::Enum(0))], plate(), 0.0, EffectEnv::default()); + assert_eq!(src.img.data, plate().data); + // Inside mask forces opacity; outside mask forces transparency. + let sq = |x0: f64, y0: f64, x1: f64, y1: f64| MaskShape { + name: "m".into(), + points: vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1]], + closed: true, + inverted: false, + }; + let masks = [sq(0.0, 0.0, 6.0, 6.0), sq(12.0, 12.0, 18.0, 18.0)]; + let env = EffectEnv { masks: &masks, ..Default::default() }; + let m = run_fx("ec.keying.keylight", &[("screenColour", green()), ("insideMask", Value::Enum(1)), ("outsideMask", Value::Enum(2))], plate(), 0.0, env); + assert!(m.img.get(2, 2)[3] > 0.99, "{:?}", m.img.get(2, 2)); + assert!(m.img.get(15, 15)[3] < 0.01); + // Screen gain above 100 % keys more of the strip. + let g = run_fx("ec.keying.keylight", &[("screenColour", green()), ("screenGain", num(150.0))], plate(), 0.0, EffectEnv::default()); + assert!(g.img.get(27, 5)[3] < out.img.get(27, 5)[3]); + assert_eq!(crate::lookup("Keylight (1.2)").map(|s| s.id), Some("ec.keying.keylight")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..46da720e 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,8 +8,10 @@ pub mod audio_fx; mod blur2; +mod blur3; mod channel; mod channel2; +mod channel3d; mod color2; mod color3; mod color_fx; @@ -17,22 +19,27 @@ mod controls; mod distort; mod distort2; mod distort3; +pub mod distort4; mod generate; mod generate2; mod generate3; mod keying; mod keying2; +pub mod keylight; mod matte; mod misc; +pub mod mocha_shape; mod noise; mod noise2; mod obsolete; +mod ocio; pub mod paint; mod perspective; mod perspective2; pub mod puppet; mod sim; mod sim2; +mod sim3; mod stylize2; mod stylize3; mod textfx; @@ -42,6 +49,7 @@ mod transition2; pub mod util; mod utility; mod utility2; +mod vr; pub mod warp_stab; use std::collections::HashMap; @@ -51,7 +59,7 @@ pub use color2::Curve; use effectcraft_keyframe::Value; use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; -pub use effectcraft_raster::Image; +pub use effectcraft_raster::{AuxChannels, Image}; /// Effect categories in Effects & Presets order. pub const CATEGORIES: &[&str] = &[ @@ -232,6 +240,13 @@ pub trait EffectHost: Sync { fn layer_at(&self, _id: u64, _comp_time: f64, _masks_and_effects: bool) -> Option { None } + /// Auxiliary 3D channels (depth, object / material IDs, normals, Cryptomatte, any named + /// EXR channel) of the effect's own layer source at the current time, in layer space (see + /// [`AuxChannels`]). `None` when the source has none (the 3D Channel effects then pass the + /// layer through). + fn aux(&self) -> Option> { + None + } } /// Extra context the renderer may supply (all optional; `Default` is "nothing known"). @@ -246,6 +261,10 @@ pub struct EffectEnv<'a> { pub frame_rate: f64, /// Index of the running effect in its layer's stack (for [`EffectHost::self_at`]). pub effect_index: usize, + /// The project's working colour space (`None` = unmanaged, treated as sRGB). + pub working_space: Option, + /// Working-space pixels are linear light (Linearize Working Space). + pub working_linear: bool, } /// What an effect gets to render with. @@ -349,6 +368,14 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(noise2::specs()); v.extend(color3::specs()); v.extend(obsolete::specs()); + v.extend(ocio::specs()); + v.extend(vr::specs()); + v.extend(blur3::specs()); + v.extend(distort4::specs()); + v.extend(keylight::specs()); + v.extend(mocha_shape::specs()); + v.extend(sim3::specs()); + v.extend(channel3d::specs()); v.extend(time_fx::specs()); v.extend(audio_fx::specs()); v.extend(paint::specs()); @@ -370,7 +397,10 @@ pub fn find(id: &str) -> Option<&'static EffectSpec> { /// Find by id or (case-insensitive) display name. pub fn lookup(name_or_id: &str) -> Option<&'static EffectSpec> { - find(name_or_id).or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) + find(name_or_id).or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))).or_else(|| { + // After Effects' third-party display names we register under a generic name. + keylight::KEYLIGHT_ALIASES.iter().any(|a| a.eq_ignore_ascii_case(name_or_id)).then(|| find("ec.keying.keylight")).flatten() + }) } /// A parameter's default value on a layer of `layer_size` (point defaults are fractions of the @@ -492,6 +522,10 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.ccforcemotionblur", "ec.time.ccwidetime", "ec.time.pixelmotionblur", + "ec.vr.digitalglitch", + "ec.blur.camerashakedeblur", + "ec.distort.rollingshutterrepair", + "ec.obsolete.mochashape", // Each frame gets its own stabilizing warp. warp_stab::ID, ]; diff --git a/crates/effects/src/mocha_shape.rs b/crates/effects/src/mocha_shape.rs new file mode 100644 index 00000000..0fafce9e --- /dev/null +++ b/crates/effects/src/mocha_shape.rs @@ -0,0 +1,243 @@ +//! Obsolete ▸ mocha shape: renders planar-tracked roto shapes as a matte. +//! +//! Mocha's own clipboard / export formats are not publicly specified, so the effect reads shape +//! data in a small documented JSON format of ours (in the hidden `shapeData` parameter, either the +//! JSON itself or a path to a `.json` file), which converters or agents can produce: +//! +//! ```json +//! {"shapes": [{"name": "Shape 1", "closed": true, "inverted": false, "feather": 2.0, +//! "points": [[x, y], …], +//! "frames": [{"time": 0.0, "points": [[x, y], …]}, {"time": 1.0, "points": […]}]}]} +//! ``` +//! +//! Coordinates are layer pixels. `frames` (layer time in seconds, linearly interpolated between +//! keys with the same vertex count, held outside) override the static `points`. Shapes combine +//! with Blend Mode (add / subtract / intersect / difference) into a coverage matte that is used +//! per Render Type: Shape Cutout (layer × matte), Color Composite (shape colour over the layer) +//! or Color Shape Cutout (shape colour inside the matte only). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, point_in_poly}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct MochaShape { + pub name: String, + pub closed: bool, + pub inverted: bool, + pub feather: f64, + pub points: Vec<[f64; 2]>, + /// (layer time, points), sorted by time. + pub frames: Vec<(f64, Vec<[f64; 2]>)>, +} + +impl MochaShape { + /// Outline at layer time `t`. + pub fn at(&self, t: f64) -> Vec<[f64; 2]> { + if self.frames.is_empty() { + return self.points.clone(); + } + let i = self.frames.partition_point(|f| f.0 <= t); + if i == 0 { + return self.frames[0].1.clone(); + } + if i == self.frames.len() { + return self.frames[i - 1].1.clone(); + } + let (a, b) = (&self.frames[i - 1], &self.frames[i]); + if a.1.len() != b.1.len() { + return a.1.clone(); + } + let u = ((t - a.0) / (b.0 - a.0).max(1e-12)).clamp(0.0, 1.0); + a.1.iter().zip(&b.1).map(|(p, q)| [p[0] + (q[0] - p[0]) * u, p[1] + (q[1] - p[1]) * u]).collect() + } +} + +fn pts(v: &serde_json::Value) -> Vec<[f64; 2]> { + v.as_array() + .map(|a| { + a.iter() + .filter_map(|p| { + let p = p.as_array()?; + Some([p.first()?.as_f64()?, p.get(1)?.as_f64()?]) + }) + .collect() + }) + .unwrap_or_default() +} + +/// Parse shape data JSON (see the module docs). `None` when it is not valid. +pub fn parse_shapes(text: &str) -> Option> { + let v: serde_json::Value = serde_json::from_str(text).ok()?; + let list = v.get("shapes").and_then(|s| s.as_array()).or_else(|| v.as_array())?; + let mut out = vec![]; + for s in list { + let mut frames: Vec<(f64, Vec<[f64; 2]>)> = s + .get("frames") + .and_then(|f| f.as_array()) + .map(|a| a.iter().filter_map(|f| Some((f.get("time")?.as_f64()?, pts(f.get("points")?)))).collect()) + .unwrap_or_default(); + frames.sort_by(|a, b| a.0.total_cmp(&b.0)); + out.push(MochaShape { + name: s.get("name").and_then(|n| n.as_str()).unwrap_or("").to_string(), + closed: s.get("closed").and_then(|c| c.as_bool()).unwrap_or(true), + inverted: s.get("inverted").and_then(|c| c.as_bool()).unwrap_or(false), + feather: s.get("feather").and_then(|c| c.as_f64()).unwrap_or(0.0).max(0.0), + points: s.get("points").map(pts).unwrap_or_default(), + frames, + }); + } + Some(out) +} + +type ShapeCache = Mutex>>>>; + +fn load(src: &str) -> Option>> { + let s = src.trim(); + if s.is_empty() { + return None; + } + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(v) = cache.lock().ok().and_then(|c| c.get(src).cloned()) { + return v; + } + let parsed = + if s.starts_with('{') || s.starts_with('[') { parse_shapes(s) } else { std::fs::read_to_string(s).ok().and_then(|t| parse_shapes(&t)) }.map(Arc::new); + if let Ok(mut c) = cache.lock() { + if c.len() > 64 { + c.clear(); + } + c.insert(src.to_string(), parsed.clone()); + } + parsed +} + +fn mocha_shape(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(shapes) = load(ctx.params.s("shapeData")) else { return b }; + let (w, h) = (b.img.width as usize, b.img.height as usize); + let blend = ctx.params.e("blendMode"); + let inv = 1.0 / b.scale.max(1e-9); + let mut matte: Option = None; + for s in shapes.iter() { + let outline = s.at(ctx.time); + if outline.len() < 3 || !s.closed { + continue; + } + let mut cov = Plane::new(w, h); + cov.data.par_chunks_mut(w.max(1)).enumerate().for_each(|(y, row)| { + for (x, v) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - b.offset[0]) * inv; + let ly = (y as f64 + 0.5 - b.offset[1]) * inv; + *v = if point_in_poly(&outline, lx, ly) != s.inverted { 1.0 } else { 0.0 }; + } + }); + if s.feather > 0.0 { + let f = s.feather * b.scale * 0.5; + cov = gauss_plane(&cov, f, f); + } + matte = Some(match matte { + None => cov, + Some(m) => m.zip_map(&cov, |a, c| match blend { + 1 => a * (1.0 - c), + 2 => a * c, + 3 => (a - c).abs(), + _ => a + c - a * c, + }), + }); + } + let Some(mut m) = matte else { return b }; + if ctx.params.b("invert") { + m = m.map(|v| 1.0 - v); + } + let kind = ctx.params.e("renderType"); + let c = ctx.params.color("shapeColour"); + let op = (ctx.params.f("opacity") / 100.0) as f32; + let edge = ctx.params.b("renderEdgeWidth"); + b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| { + let k = if edge { (1.0 - (2.0 * k - 1.0).abs()).clamp(0.0, 1.0) } else { k.clamp(0.0, 1.0) }; + match kind { + 1 => { + let a = k * op; + let s = [c[0] * a, c[1] * a, c[2] * a, a]; + *px = [s[0] + px[0] * (1.0 - a), s[1] + px[1] * (1.0 - a), s[2] + px[2] * (1.0 - a), s[3] + px[3] * (1.0 - a)]; + } + 2 => { + let a = k * op; + *px = [c[0] * a, c[1] * a, c[2] * a, a]; + } + _ => { + let a = 1.0 - (1.0 - k) * op; + *px = px.map(|v| v * a); + } + } + }); + b +} + +pub fn specs() -> Vec { + vec![EffectSpec { + id: "ec.obsolete.mochashape", + name: "mocha shape", + category: "Obsolete", + params: vec![ + p("blendMode", "Blend mode", Value::Enum(0), popup(&["Add", "Subtract", "Intersect", "Difference"])), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("renderEdgeWidth", "Render edge width", Value::Bool(false), ParamUi::Checkbox), + p("renderType", "Render type", Value::Enum(0), popup(&["Shape cutout", "Color composite", "Color shape cutout"])), + p("shapeColour", "Shape colour", col(1.0, 1.0, 1.0), ParamUi::Color), + p("opacity", "Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + // JSON shape data (see the module docs) or a path to a .json file. + p("shapeData", "Shape data", Value::Str(String::new()), ParamUi::Hidden), + ], + render: mocha_shape, + gpu: false, + float: true, + }] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, Image, run_fx}; + + const DATA: &str = r#"{"shapes":[{"name":"S","closed":true,"points":[[2,2],[10,2],[10,10],[2,10]], + "frames":[{"time":0,"points":[[2,2],[10,2],[10,10],[2,10]]},{"time":1,"points":[[12,2],[20,2],[20,10],[12,10]]}]}]}"#; + + #[test] + fn parses_and_interpolates() { + let s = parse_shapes(DATA).unwrap(); + assert_eq!(s.len(), 1); + assert_eq!(s[0].at(0.5)[0], [7.0, 2.0]); + assert_eq!(s[0].at(5.0)[0], [12.0, 2.0]); + assert!(parse_shapes("not json").is_none()); + } + + #[test] + fn cuts_out_the_shape_over_time() { + let img = Image::filled(24, 12, [1.0, 0.0, 0.0, 1.0]); + let a = run_fx("ec.obsolete.mochashape", &[("shapeData", Value::Str(DATA.into()))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(a.img.get(5, 5)[3], 1.0); + assert_eq!(a.img.get(15, 5)[3], 0.0); + let b = run_fx("ec.obsolete.mochashape", &[("shapeData", Value::Str(DATA.into()))], img.clone(), 1.0, EffectEnv::default()); + assert_eq!(b.img.get(5, 5)[3], 0.0); + assert_eq!(b.img.get(15, 5)[3], 1.0); + // Color shape cutout fills with the shape colour; no data = pass-through. + let c = run_fx( + "ec.obsolete.mochashape", + &[("shapeData", Value::Str(DATA.into())), ("renderType", Value::Enum(2)), ("shapeColour", col(0.0, 1.0, 0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(c.img.get(5, 5), [0.0, 1.0, 0.0, 1.0]); + let none = run_fx("ec.obsolete.mochashape", &[], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + } +} diff --git a/crates/effects/src/ocio.rs b/crates/effects/src/ocio.rs new file mode 100644 index 00000000..d868a8f9 --- /dev/null +++ b/crates/effects/src/ocio.rs @@ -0,0 +1,1281 @@ +//! OCIO-style colour transforms (Color Correction ▸ OCIO CDL / Color Space / Display / File / Look +//! Transform), Color Profile Converter (Utility) and Color Stabilizer (Color Correction). +//! +//! The OCIO effects run on a **built-in minimal configuration** written from published +//! standards, not from OpenColorIO's source: +//! +//! * colour spaces are defined by primaries, white point and transfer function: ACES2065-1 (AP0) +//! and ACEScg / ACEScct (AP1) from SMPTE ST 2065-1 and Academy S-2014-004 / S-2016-001, sRGB +//! (IEC 61966-2-1), Rec. 709 / Rec. 2020 with the BT.1886 gamma 2.4 EOTF, Display P3 (SMPTE +//! EG 432-1 primaries, D65, sRGB curve), their linear variants and CIE XYZ D65; +//! * the reference space is CIE XYZ D65; spaces with another white point (the ACES white, +//! x 0.32168 y 0.33767) are adapted with the Bradford chromatic adaptation transform, as in the +//! published ACES configurations; +//! * displays (sRGB, Rec.1886 Rec.709, Display P3, Rec.2100-PQ via SMPTE ST 2084) and views +//! (Standard = clip-free encode, Tone Mapped = extended Reinhard roll-off, Raw); +//! * looks are ASC CDLs applied in ACEScct; +//! * CDL maths follows the ASC CDL v1.2 specification (slope, offset, power, then saturation with +//! Rec. 709 luma weights); the "No Clamp" style mirrors power around zero; +//! * File Transform reads `.cube` (Adobe/Resolve), `.3dl` (Autodesk Lustre/Flame integer +//! lattices), `.csp` (cineSpace, with its per-channel pre-LUT shaper) and ASC `.cc` / `.ccc` / +//! `.cdl` XML, with nearest, trilinear or tetrahedral interpolation and an inverse direction +//! (exact for 1D/CDL, iterative for 3D lattices). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_color::ColorSpace; +use effectcraft_color::space::{invert, mul, mul_vec, rgb_to_xyz}; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use rayon::prelude::*; + +use crate::util::unpremul; +use crate::utility::{Lut, parse_cube}; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +type M3 = [[f64; 3]; 3]; + +const D65: [f64; 2] = [0.3127, 0.3290]; +const ACES_WHITE: [f64; 2] = [0.32168, 0.33767]; +const AP0: [[f64; 2]; 3] = [[0.7347, 0.2653], [0.0, 1.0], [0.0001, -0.0770]]; +const AP1: [[f64; 2]; 3] = [[0.713, 0.293], [0.165, 0.830], [0.128, 0.044]]; +const REC709: [[f64; 2]; 3] = [[0.640, 0.330], [0.300, 0.600], [0.150, 0.060]]; +const REC2020: [[f64; 2]; 3] = [[0.708, 0.292], [0.170, 0.797], [0.131, 0.046]]; +const P3: [[f64; 2]; 3] = [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]]; +const IDENTITY: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +/// Transfer functions (encoded ↔ linear). +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Tf { + Linear, + Srgb, + Gamma(f64), + AcesCct, + /// SMPTE ST 2084 with 1.0 = 100 cd/m². + Pq, +} + +impl Tf { + pub fn decode(self, v: f64) -> f64 { + match self { + Tf::Linear => v, + Tf::Srgb => { + let a = v.abs(); + let l = if a <= 0.04045 { a / 12.92 } else { ((a + 0.055) / 1.055).powf(2.4) }; + l.copysign(v) + } + Tf::Gamma(g) => v.abs().powf(g).copysign(v), + Tf::AcesCct => { + if v <= 0.155_251_141_552_511 { + (v - 0.072_905_534_195_835_5) / 10.540_237_741_654_5 + } else if v < (65504f64.log2() + 9.72) / 17.52 { + 2f64.powf(v * 17.52 - 9.72) + } else { + 65504.0 + } + } + Tf::Pq => { + let (m1, m2, c1, c2, c3) = pq_consts(); + let e = v.max(0.0).powf(1.0 / m2); + let l = ((e - c1).max(0.0) / (c2 - c3 * e)).powf(1.0 / m1); + l * 100.0 + } + } + } + pub fn encode(self, v: f64) -> f64 { + match self { + Tf::Linear => v, + Tf::Srgb => { + let a = v.abs(); + let e = if a <= 0.003_130_8 { a * 12.92 } else { 1.055 * a.powf(1.0 / 2.4) - 0.055 }; + e.copysign(v) + } + Tf::Gamma(g) => v.abs().powf(1.0 / g).copysign(v), + Tf::AcesCct => { + if v <= 0.007_812_5 { + 10.540_237_741_654_5 * v + 0.072_905_534_195_835_5 + } else { + (v.log2() + 9.72) / 17.52 + } + } + Tf::Pq => { + let (m1, m2, c1, c2, c3) = pq_consts(); + let y = (v / 100.0).max(0.0).powf(m1); + ((c1 + c2 * y) / (1.0 + c3 * y)).powf(m2) + } + } + } +} + +fn pq_consts() -> (f64, f64, f64, f64, f64) { + (2610.0 / 16384.0, 2523.0 / 4096.0 * 128.0, 3424.0 / 4096.0, 2413.0 / 4096.0 * 32.0, 2392.0 / 4096.0 * 32.0) +} + +/// A colour space of the built-in config. +#[derive(Clone, Copy, Debug)] +pub struct Space { + pub name: &'static str, + /// Primaries and white (None = CIE XYZ itself). + pub prims: Option<([[f64; 2]; 3], [f64; 2])>, + pub tf: Tf, + /// Passes data through untouched. + pub raw: bool, +} + +const fn sp(name: &'static str, prims: [[f64; 2]; 3], white: [f64; 2], tf: Tf) -> Space { + Space { name, prims: Some((prims, white)), tf, raw: false } +} + +/// The built-in configuration's colour spaces, in popup order. +pub const SPACES: &[Space] = &[ + sp("ACES2065-1", AP0, ACES_WHITE, Tf::Linear), + sp("ACEScg", AP1, ACES_WHITE, Tf::Linear), + sp("ACEScct", AP1, ACES_WHITE, Tf::AcesCct), + sp("Linear Rec.709 (sRGB)", REC709, D65, Tf::Linear), + sp("sRGB", REC709, D65, Tf::Srgb), + sp("Gamma 2.4 Rec.709", REC709, D65, Tf::Gamma(2.4)), + sp("Linear Rec.2020", REC2020, D65, Tf::Linear), + sp("Gamma 2.4 Rec.2020", REC2020, D65, Tf::Gamma(2.4)), + sp("Linear P3-D65", P3, D65, Tf::Linear), + sp("Display P3", P3, D65, Tf::Srgb), + Space { name: "CIE-XYZ-D65", prims: None, tf: Tf::Linear, raw: false }, + Space { name: "Raw", prims: None, tf: Tf::Linear, raw: true }, +]; + +pub fn space_names() -> Vec<&'static str> { + SPACES.iter().map(|s| s.name).collect() +} + +pub fn space_by_name(n: &str) -> Option<&'static Space> { + SPACES.iter().find(|s| s.name.eq_ignore_ascii_case(n)) +} + +/// Bradford chromatic adaptation from white `src` to white `dst` (xy). +pub fn bradford(src: [f64; 2], dst: [f64; 2]) -> M3 { + if src == dst { + return IDENTITY; + } + const MB: M3 = [[0.8951, 0.2664, -0.1614], [-0.7502, 1.7135, 0.0367], [0.0389, -0.0685, 1.0296]]; + let xyz = |c: [f64; 2]| [c[0] / c[1], 1.0, (1.0 - c[0] - c[1]) / c[1]]; + let s = mul_vec(&MB, xyz(src)); + let d = mul_vec(&MB, xyz(dst)); + let diag = [[d[0] / s[0], 0.0, 0.0], [0.0, d[1] / s[1], 0.0], [0.0, 0.0, d[2] / s[2]]]; + mul(&invert(&MB), &mul(&diag, &MB)) +} + +impl Space { + /// Linear RGB → XYZ D65 (Bradford-adapted when `adapt`, else the space's own white). + pub fn to_ref(self, adapt: bool) -> M3 { + match self.prims { + None => IDENTITY, + Some((p, w)) => { + let m = rgb_to_xyz(p, w); + if adapt { mul(&bradford(w, D65), &m) } else { m } + } + } + } +} + +/// Linear-RGB matrix from space `a` to space `b`. +pub fn matrix(a: &Space, b: &Space) -> M3 { + mul(&invert(&b.to_ref(true)), &a.to_ref(true)) +} + +/// A compiled colour-space conversion. +#[derive(Clone, Copy, Debug)] +pub struct Xform { + dec: Tf, + m: [[f32; 3]; 3], + ident: bool, + enc: Tf, +} + +impl Xform { + pub fn new(a: &Space, b: &Space) -> Xform { + if a.raw || b.raw { + return Xform { dec: Tf::Linear, m: IDENTITY.map(|r| r.map(|v| v as f32)), ident: true, enc: Tf::Linear }; + } + let m = matrix(a, b); + let ident = (0..3).all(|i| (0..3).all(|j| (m[i][j] - IDENTITY[i][j]).abs() < 1e-12)); + Xform { dec: a.tf, m: m.map(|r| r.map(|v| v as f32)), ident, enc: b.tf } + } + #[inline] + pub fn apply(&self, c: [f32; 3]) -> [f32; 3] { + let l = c.map(|v| self.dec.decode(v as f64) as f32); + let l = if self.ident { l } else { [0, 1, 2].map(|i| self.m[i][0] * l[0] + self.m[i][1] * l[1] + self.m[i][2] * l[2]) }; + l.map(|v| self.enc.encode(v as f64) as f32) + } +} + +// ---------------------------------------------------------------- ASC CDL + +/// An ASC Colour Decision List correction. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Cdl { + pub slope: [f64; 3], + pub offset: [f64; 3], + pub power: [f64; 3], + pub sat: f64, +} + +impl Default for Cdl { + fn default() -> Self { + Cdl { slope: [1.0; 3], offset: [0.0; 3], power: [1.0; 3], sat: 1.0 } + } +} + +const LUMA: [f64; 3] = [0.2126, 0.7152, 0.0722]; + +impl Cdl { + /// Forward CDL. `clamp` = the v1.2 style (results clamped to 0..1 after SOP and saturation). + pub fn apply(&self, c: [f64; 3], clamp: bool) -> [f64; 3] { + let mut o = [0.0; 3]; + for i in 0..3 { + let v = c[i] * self.slope[i] + self.offset[i]; + o[i] = if clamp { + v.clamp(0.0, 1.0).powf(self.power[i]) + } else if v >= 0.0 { + v.powf(self.power[i]) + } else { + v + }; + } + let l = o[0] * LUMA[0] + o[1] * LUMA[1] + o[2] * LUMA[2]; + let o = o.map(|v| l + self.sat * (v - l)); + if clamp { o.map(|v| v.clamp(0.0, 1.0)) } else { o } + } + /// Exact inverse of [`Cdl::apply`] (inside the clamped range). + pub fn invert(&self, c: [f64; 3], clamp: bool) -> [f64; 3] { + let c = if clamp { c.map(|v| v.clamp(0.0, 1.0)) } else { c }; + let l = c[0] * LUMA[0] + c[1] * LUMA[1] + c[2] * LUMA[2]; + let s = if self.sat.abs() < 1e-9 { 1e-9 } else { self.sat }; + let o = c.map(|v| l + (v - l) / s); + let mut r = [0.0; 3]; + for i in 0..3 { + let v = if clamp { o[i].clamp(0.0, 1.0) } else { o[i] }; + let pw = self.power[i].max(1e-9); + let u = if v >= 0.0 { v.powf(1.0 / pw) } else { v }; + let sl = if self.slope[i].abs() < 1e-9 { 1e-9 } else { self.slope[i] }; + r[i] = (u - self.offset[i]) / sl; + } + if clamp { r.map(|v| v.clamp(0.0, 1.0)) } else { r } + } +} + +fn tag<'a>(xml: &'a str, name: &str) -> Option<&'a str> { + let open = xml.find(&format!("<{name}"))?; + let gt = open + xml[open..].find('>')?; + if xml[..gt].ends_with('/') { + return Some(""); + } + let close = xml[gt..].find(&format!(""))? + gt; + Some(&xml[gt + 1..close]) +} + +fn nums3(s: &str) -> Option<[f64; 3]> { + let v: Vec = s.split_whitespace().filter_map(|t| t.parse().ok()).collect(); + (v.len() >= 3).then(|| [v[0], v[1], v[2]]) +} + +fn parse_cc(block: &str) -> Cdl { + let mut c = Cdl::default(); + if let Some(s) = tag(block, "Slope").and_then(nums3) { + c.slope = s; + } + if let Some(s) = tag(block, "Offset").and_then(nums3) { + c.offset = s; + } + if let Some(s) = tag(block, "Power").and_then(nums3) { + c.power = s; + } + if let Some(s) = tag(block, "Saturation").and_then(|t| t.trim().parse().ok()) { + c.sat = s; + } + c +} + +/// Parse ASC CDL XML (`.cc`, `.ccc`, `.cdl`): every `` with its `id`. +pub fn parse_cdl_xml(xml: &str) -> Vec<(String, Cdl)> { + let mut out = vec![]; + let mut rest = xml; + while let Some(i) = rest.find("').unwrap_or(after.len()); + let head = &after[..head_end]; + let id = head + .find("id=") + .and_then(|k| { + let q = head[k + 3..].chars().next()?; + let s = &head[k + 4..]; + s.find(q).map(|e| s[..e].to_string()) + }) + .unwrap_or_default(); + let end = after.find("").map(|e| e + "".len()).unwrap_or(after.len()); + out.push((id, parse_cc(&after[..end]))); + rest = &after[end..]; + } + out +} + +// ---------------------------------------------------------------- LUT files + +/// A loaded transform file. +#[derive(Clone, Debug)] +pub enum FileXform { + /// A lattice/1D LUT with an optional per-channel piecewise-linear shaper (cineSpace pre-LUT). + Lut { lut: Lut, shaper: Option<[Vec<(f32, f32)>; 3]> }, + /// CDL corrections by id (in file order). + Cdl(Vec<(String, Cdl)>), +} + +/// Parse an Autodesk `.3dl` lattice (integers, blue fastest; an optional input-mesh line). +pub fn parse_3dl(text: &str) -> Option { + let mut mesh: Option> = None; + let mut rows: Vec<[f64; 3]> = vec![]; + let mut out_bits: Option = None; + for line in text.lines() { + let l = line.trim(); + if l.is_empty() || l.starts_with('#') || l.starts_with('<') { + continue; + } + if l.starts_with(|c: char| c.is_ascii_alphabetic()) { + // "Mesh " (Lustre header); other keywords are ignored. + let t: Vec<&str> = l.split_whitespace().collect(); + if t[0].eq_ignore_ascii_case("mesh") && t.len() >= 3 { + out_bits = t[2].parse().ok(); + } + continue; + } + let v: Vec = l.split_whitespace().filter_map(|t| t.parse().ok()).collect(); + if v.len() > 3 && mesh.is_none() && rows.is_empty() { + mesh = Some(v); + } else if v.len() == 3 { + rows.push([v[0], v[1], v[2]]); + } else { + return None; + } + } + let is_cube = |k: usize| { + let n = (k as f64).cbrt().round() as usize; + n * n * n == k + }; + // A 3-entry input mesh (3³ lattice) looks like a data row: tell them apart by the count. + if mesh.is_none() && !is_cube(rows.len()) && rows.len() == 28 { + mesh = Some(rows.remove(0).to_vec()); + } + let n = match &mesh { + Some(m) => m.len(), + None => (rows.len() as f64).cbrt().round() as usize, + }; + if n < 2 || rows.len() != n * n * n { + return None; + } + let maxv = rows.iter().flat_map(|r| r.iter()).fold(0.0f64, |m, &v| m.max(v)); + let out_max = match out_bits { + Some(b) if b > 0 && b < 32 => (1u64 << b) as f64 - 1.0, + _ => [255.0, 1023.0, 4095.0, 65535.0].into_iter().find(|&m| maxv <= m).unwrap_or(maxv.max(1.0)), + }; + // .3dl rows: red slowest, blue fastest. Our lattice is red fastest. + let mut d3 = vec![[0.0f32; 3]; n * n * n]; + for (k, r) in rows.iter().enumerate() { + let (ri, gi, bi) = (k / (n * n), (k / n) % n, k % n); + d3[ri + gi * n + bi * n * n] = r.map(|v| (v / out_max) as f32); + } + Some(Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }) +} + +/// Parse a cineSpace `.csp` LUT (1D or 3D, with its three pre-LUT shapers). +pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { + let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); + if !lines.next()?.starts_with("CSPLUTV100") { + return None; + } + let kind = lines.next()?; + let mut lines = lines.peekable(); + if lines.peek().is_some_and(|l| l.starts_with("BEGIN METADATA")) { + for l in lines.by_ref() { + if l.starts_with("END METADATA") { + break; + } + } + } + let nums = |l: &str| -> Option> { l.split_whitespace().map(|t| t.parse().ok()).collect() }; + let mut shaper: [Vec<(f32, f32)>; 3] = Default::default(); + for ch in shaper.iter_mut() { + let n: usize = lines.next()?.parse().ok()?; + let ins = nums(lines.next()?)?; + let outs = nums(lines.next()?)?; + if ins.len() != n || outs.len() != n || n < 2 { + return None; + } + *ch = ins.into_iter().zip(outs).collect(); + } + if kind.starts_with("3D") { + let sz = nums(lines.next()?)?; + if sz.len() != 3 || sz[0] != sz[1] || sz[1] != sz[2] { + return None; + } + let n = sz[0] as usize; + let mut d3 = Vec::with_capacity(n * n * n); + for _ in 0..n * n * n { + let v = nums(lines.next()?)?; + if v.len() != 3 { + return None; + } + d3.push([v[0], v[1], v[2]]); + } + (n >= 2).then_some((Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + } else { + let n: usize = lines.next()?.parse().ok()?; + let mut d1 = Vec::with_capacity(n); + for _ in 0..n { + let v = nums(lines.next()?)?; + if v.len() != 3 { + return None; + } + d1.push([v[0], v[1], v[2]]); + } + (n >= 2).then_some((Lut { n1: n, d1, n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + } +} + +fn shape(curve: &[(f32, f32)], x: f32) -> f32 { + if curve.len() < 2 { + return x; + } + let i = curve.partition_point(|p| p.0 < x).clamp(1, curve.len() - 1); + let (a, b) = (curve[i - 1], curve[i]); + let t = ((x - a.0) / (b.0 - a.0).max(1e-12)).clamp(0.0, 1.0); + a.1 + (b.1 - a.1) * t +} + +fn shape_inv(curve: &[(f32, f32)], y: f32) -> f32 { + let flipped: Vec<(f32, f32)> = curve.iter().map(|&(a, b)| (b, a)).collect(); + shape(&flipped, y) +} + +/// Parse transform text by its format (detected from `ext` or the content). +pub fn parse_file(text: &str, ext: &str) -> Option { + let ext = ext.to_ascii_lowercase(); + if ext == "cc" || ext == "ccc" || ext == "cdl" || text.contains(">>>; + +/// Load (and cache) a transform from a file path or inline content. +pub fn load_file(src: &str) -> Option> { + let src_t = src.trim(); + if src_t.is_empty() { + return None; + } + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Ok(m) = cache.lock() + && let Some(v) = m.get(src) + { + return v.clone(); + } + let inline = src.contains('\n'); + let parsed = if inline { + parse_file(src, "") + } else { + let ext = std::path::Path::new(src_t).extension().and_then(|e| e.to_str()).unwrap_or(""); + std::fs::read_to_string(src_t).ok().and_then(|t| parse_file(&t, ext)) + } + .map(Arc::new); + if let Ok(mut m) = cache.lock() { + m.insert(src.to_string(), parsed.clone()); + } + parsed +} + +impl FileXform { + pub fn apply(&self, c: [f32; 3], interp: u32, inverse: bool, ccc_id: &str) -> [f32; 3] { + match self { + FileXform::Cdl(list) => { + let cdl = list.iter().find(|(id, _)| !ccc_id.is_empty() && id == ccc_id).or(list.first()).map(|(_, c)| *c).unwrap_or_default(); + let c64 = c.map(|v| v as f64); + let o = if inverse { cdl.invert(c64, true) } else { cdl.apply(c64, true) }; + o.map(|v| v as f32) + } + FileXform::Lut { lut, shaper } => { + let fwd = |c: [f32; 3]| { + let c = match shaper { + Some(s) => [shape(&s[0], c[0]), shape(&s[1], c[1]), shape(&s[2], c[2])], + None => c, + }; + lut.apply_interp(c, interp) + }; + if !inverse { + return fwd(c); + } + if lut.n3 == 0 { + // 1D: invert each channel exactly through the (monotone) tables. + let mut x = c; + if lut.n1 > 1 { + x = [0, 1, 2].map(|i| { + let curve: Vec<(f32, f32)> = + (0..lut.n1).map(|k| (lut.d1[k][i], lut.dmin[i] + (lut.dmax[i] - lut.dmin[i]) * k as f32 / (lut.n1 - 1) as f32)).collect(); + shape(&curve, c[i]) + }); + } + if let Some(s) = shaper { + x = [shape_inv(&s[0], x[0]), shape_inv(&s[1], x[1]), shape_inv(&s[2], x[2])]; + } + return x; + } + // 3D: damped fixed-point (Newton with an identity Jacobian) from the target. + let mut x = c; + for _ in 0..40 { + let y = fwd(x); + let e = [c[0] - y[0], c[1] - y[1], c[2] - y[2]]; + if e.iter().all(|v| v.abs() < 1e-6) { + break; + } + x = [x[0] + e[0], x[1] + e[1], x[2] + e[2]]; + } + x + } + } + } +} + +// ---------------------------------------------------------------- displays, views and looks + +/// Displays of the built-in config: (name, display primaries/white, encoding). +pub const DISPLAYS: &[(&str, [[f64; 2]; 3], Tf)] = + &[("sRGB", REC709, Tf::Srgb), ("Rec.1886 Rec.709", REC709, Tf::Gamma(2.4)), ("Display P3", P3, Tf::Srgb), ("Rec.2100-PQ", REC2020, Tf::Pq)]; + +pub const VIEWS: &[&str] = &["Standard", "Tone Mapped", "Raw"]; + +/// Built-in looks: CDLs applied in ACEScct. +pub fn looks() -> Vec<(&'static str, Cdl)> { + let grey = 0.413_588_402_492_442_1; // ACEScct of 0.18 + let c = |slope: [f64; 3], offset: [f64; 3], sat: f64| Cdl { slope, offset, power: [1.0; 3], sat }; + vec![ + ("Warm", c([1.04, 1.0, 0.94], [0.0; 3], 1.0)), + ("Cool", c([0.95, 1.0, 1.06], [0.0; 3], 1.0)), + ("High Contrast", c([1.25; 3], [grey * (1.0 - 1.25); 3], 1.0)), + ("Low Contrast", c([0.8; 3], [grey * (1.0 - 0.8); 3], 1.0)), + ("Desaturated", c([1.0; 3], [0.0; 3], 0.6)), + ("Vivid", c([1.0; 3], [0.0; 3], 1.3)), + ] +} + +/// Extended Reinhard roll-off (white point 16): maps 0..∞ into 0..1, ≈ identity near 0. +fn tonemap(v: f64) -> f64 { + let w2 = 256.0; + if v <= 0.0 { v } else { v * (1.0 + v / w2) / (1.0 + v) } +} + +fn tonemap_inv(y: f64) -> f64 { + if y <= 0.0 { + return y; + } + // Solve v² / w² + v (1 − y) − y = 0. + let (a, b, c) = (1.0 / 256.0, 1.0 - y, -y); + (-b + (b * b - 4.0 * a * c).sqrt()) / (2.0 * a) +} + +/// Display transform of an input-space colour (`inverse`: display code values → input space). +pub fn display_xform(input: &Space, display: usize, view: usize, inverse: bool) -> impl Fn([f32; 3]) -> [f32; 3] + Sync { + let (_, prims, tf) = DISPLAYS[display.min(DISPLAYS.len() - 1)]; + let disp = Space { name: "display", prims: Some((prims, D65)), tf: Tf::Linear, raw: false }; + let raw = view == 2 || input.raw; + let m_in = Xform::new(&Space { tf: Tf::Linear, ..*input }, &disp); + let m_out = Xform::new(&disp, &Space { tf: Tf::Linear, ..*input }); + let in_tf = input.tf; + move |c: [f32; 3]| { + if raw { + return c; + } + if !inverse { + let lin = c.map(|v| in_tf.decode(v as f64) as f32); + let d = m_in.apply(lin); + d.map(|v| { + let v = v as f64; + let v = if view == 1 { tonemap(v) } else { v }; + tf.encode(v) as f32 + }) + } else { + let d = c.map(|v| { + let l = tf.decode(v as f64); + (if view == 1 { tonemap_inv(l) } else { l }) as f32 + }); + m_out.apply(d).map(|v| in_tf.encode(v as f64) as f32) + } + } +} + +// ---------------------------------------------------------------- effects + +fn spec(id: &'static str, name: &'static str, cat: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: cat, params, render, gpu: false, float: true } +} + +fn space_param(id: &'static str, name: &'static str, default: u32) -> crate::ParamSpec { + p(id, name, Value::Enum(default), popup(&space_names())) +} + +fn direction() -> crate::ParamSpec { + p("direction", "Direction", Value::Enum(0), popup(&["Forward", "Inverse"])) +} + +fn map_px(b: &mut Buf, f: impl Fn([f32; 3]) -> [f32; 3] + Sync) { + b.img.data.par_iter_mut().for_each(|px| { + let (c, a) = unpremul(*px); + if a <= 0.0 { + return; + } + let o = f(c); + *px = [o[0] * a, o[1] * a, o[2] * a, a]; + }); +} + +fn space_at(i: u32) -> &'static Space { + &SPACES[(i as usize).min(SPACES.len() - 1)] +} + +fn ocio_cst(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (a, z) = (space_at(ctx.params.e("source")), space_at(ctx.params.e("destination"))); + let (a, z) = if ctx.params.e("direction") == 1 { (z, a) } else { (a, z) }; + if a.name == z.name { + return b; + } + let x = Xform::new(a, z); + map_px(&mut b, |c| x.apply(c)); + b +} + +fn cdl_from_params(pr: &crate::Params) -> Cdl { + Cdl { + slope: [pr.f("slopeRed"), pr.f("slopeGreen"), pr.f("slopeBlue")], + offset: [pr.f("offsetRed"), pr.f("offsetGreen"), pr.f("offsetBlue")], + power: [pr.f("powerRed").max(0.0), pr.f("powerGreen").max(0.0), pr.f("powerBlue").max(0.0)], + sat: pr.f("saturation").max(0.0), + } +} + +fn ocio_cdl(ctx: &EffectCtx, mut b: Buf) -> Buf { + let cdl = cdl_from_params(ctx.params); + if cdl == Cdl::default() { + return b; + } + let clamp = ctx.params.e("style") == 0; + let inv = ctx.params.e("direction") == 1; + map_px(&mut b, |c| { + let c = c.map(|v| v as f64); + let o = if inv { cdl.invert(c, clamp) } else { cdl.apply(c, clamp) }; + o.map(|v| v as f32) + }); + b +} + +fn ocio_display(ctx: &EffectCtx, mut b: Buf) -> Buf { + let input = space_at(ctx.params.e("source")); + let f = display_xform(input, ctx.params.e("display") as usize, ctx.params.e("view") as usize, ctx.params.e("direction") == 1); + map_px(&mut b, f); + b +} + +fn ocio_file(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(x) = load_file(ctx.params.s("file")) else { return b }; + let interp = match ctx.params.e("interpolation") { + 0 => 0, + 1 => 1, + _ => 2, + }; + let inv = ctx.params.e("direction") == 1; + let id = ctx.params.s("cccId").to_string(); + map_px(&mut b, |c| x.apply(c, interp, inv, &id)); + b +} + +/// Parse a looks string: comma/space separated names, `-Name` = inverse. +pub fn parse_looks(s: &str) -> Vec<(Cdl, bool)> { + let all = looks(); + s.split([',', ':']) + .map(str::trim) + .filter(|t| !t.is_empty()) + .filter_map(|t| { + let (inv, n) = match t.strip_prefix('-') { + Some(r) => (true, r.trim()), + None => (false, t.trim_start_matches('+').trim()), + }; + all.iter().find(|(name, _)| name.eq_ignore_ascii_case(n)).map(|(_, c)| (*c, inv)) + }) + .collect() +} + +fn ocio_look(ctx: &EffectCtx, mut b: Buf) -> Buf { + let mut list = parse_looks(ctx.params.s("looks")); + let pick = ctx.params.e("look"); + if pick > 0 + && let Some((_, c)) = looks().get(pick as usize - 1) + { + list.insert(0, (*c, false)); + } + if list.is_empty() { + return b; + } + let (src, dst) = (space_at(ctx.params.e("source")), space_at(ctx.params.e("destination"))); + let inv = ctx.params.e("direction") == 1; + let cct = space_by_name("ACEScct").expect("built-in"); + let (a, z) = if inv { (dst, src) } else { (src, dst) }; + let to_p = Xform::new(a, cct); + let from_p = Xform::new(cct, z); + if inv { + list.reverse(); + } + map_px(&mut b, |c| { + let mut v = to_p.apply(c).map(|x| x as f64); + for (cdl, li) in &list { + v = if *li != inv { cdl.invert(v, false) } else { cdl.apply(v, false) }; + } + from_p.apply(v.map(|x| x as f32)) + }); + b +} + +// ---- Color Profile Converter (Utility) + +pub const PROFILES: &[&str] = &["Project Working Space", "sRGB IEC61966-2.1", "HDTV (Rec. 709)", "Rec. 2020", "Display P3", "ACEScg", "ACES2065-1"]; + +fn profile(ctx: &EffectCtx, i: u32) -> (Space, bool) { + let ws = |cs: ColorSpace| match cs { + ColorSpace::Srgb => sp("sRGB", REC709, D65, Tf::Srgb), + ColorSpace::Rec709 => sp("Rec.709", REC709, D65, Tf::Gamma(2.4)), + ColorSpace::Rec2020 => sp("Rec.2020", REC2020, D65, Tf::Gamma(2.4)), + ColorSpace::DisplayP3 => sp("Display P3", P3, D65, Tf::Srgb), + }; + match i { + 0 => (ws(ctx.env.working_space.unwrap_or(ColorSpace::Srgb)), ctx.env.working_linear), + 1 => (ws(ColorSpace::Srgb), false), + 2 => (ws(ColorSpace::Rec709), false), + 3 => (ws(ColorSpace::Rec2020), false), + 4 => (ws(ColorSpace::DisplayP3), false), + 5 => (*space_by_name("ACEScg").expect("built-in"), true), + _ => (*space_by_name("ACES2065-1").expect("built-in"), true), + } +} + +/// Gamut compression for the perceptual intent: pull out-of-gamut (negative) colours toward +/// their luminance just enough to bring every channel to ≥ 0, rolling off smoothly. +fn compress_gamut(c: [f64; 3], luma: [f64; 3]) -> [f64; 3] { + let l = c[0] * luma[0] + c[1] * luma[1] + c[2] * luma[2]; + if l <= 0.0 { + return c.map(|v| v.max(0.0)); + } + let mn = c.iter().cloned().fold(f64::INFINITY, f64::min); + if mn >= 0.0 { + return c; + } + // Fraction of the way toward grey that reaches zero on the most negative channel. + let t = (l / (l - mn)).clamp(0.0, 1.0); + c.map(|v| l + (v - l) * t) +} + +fn profile_converter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (inp, in_lin_default) = profile(ctx, ctx.params.e("inputProfile")); + let (out, out_lin_default) = profile(ctx, ctx.params.e("outputProfile")); + let in_lin = ctx.params.b("linearizeInputProfile") || in_lin_default && ctx.params.e("inputProfile") == 0; + let out_lin = ctx.params.b("linearizeOutputProfile") || out_lin_default && ctx.params.e("outputProfile") == 0; + let intent = ctx.params.e("intent"); + let dec = if in_lin { Tf::Linear } else { inp.tf }; + let enc = if out_lin { Tf::Linear } else { out.tf }; + // Absolute colorimetric keeps the source white (no adaptation); the others adapt. + let m = if intent == 3 { mul(&invert(&out.to_ref(false)), &inp.to_ref(false)) } else { matrix(&inp, &out) }; + let identity = dec == enc && (0..3).all(|i| (0..3).all(|j| (m[i][j] - IDENTITY[i][j]).abs() < 1e-9)); + if identity { + return b; + } + let out_luma = out.to_ref(true)[1]; + map_px(&mut b, |c| { + let l = c.map(|v| dec.decode(v as f64)); + let mut o = mul_vec(&m, l); + match intent { + 0 => o = compress_gamut(o, out_luma), + 2 => o = o.map(|v| v.max(0.0)), + _ => {} + } + o.map(|v| enc.encode(v) as f32) + }); + b +} + +// ---- Color Stabilizer + +/// Mean straight colour of a square (half-size `r` layer pixels) around layer point `pt`. +fn sample_mean(b: &Buf, pt: [f64; 2], r: f64) -> [f64; 3] { + let (cx, cy) = b.to_px(pt); + let rr = (r * b.scale).max(0.5); + let (x0, x1) = ((cx - rr).floor().max(0.0) as i64, ((cx + rr).ceil() as i64).min(b.img.width as i64)); + let (y0, y1) = ((cy - rr).floor().max(0.0) as i64, ((cy + rr).ceil() as i64).min(b.img.height as i64)); + let mut acc = [0.0f64; 4]; + for y in y0..y1 { + for x in x0..x1 { + let p = b.img.get(x, y); + for k in 0..4 { + acc[k] += p[k] as f64; + } + } + } + if acc[3] <= 1e-9 { + return [0.0; 3]; + } + [acc[0] / acc[3], acc[1] / acc[3], acc[2] / acc[3]] +} + +/// Monotone piecewise-quadratic curve through (0,0), three samples and (1,1) (linear between +/// them is enough for a stabiliser and stays monotone). +fn curve3(xs: [f64; 3], ys: [f64; 3], v: f64) -> f64 { + let mut pts = vec![(0.0, 0.0)]; + let mut pairs: Vec<(f64, f64)> = xs.iter().cloned().zip(ys.iter().cloned()).collect(); + pairs.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (x, y) in pairs { + if x > pts.last().expect("non-empty").0 + 1e-4 && x < 1.0 - 1e-4 { + pts.push((x, y)); + } + } + pts.push((1.0, 1.0)); + let i = pts.partition_point(|p| p.0 < v).clamp(1, pts.len() - 1); + let (a, c) = (pts[i - 1], pts[i]); + a.1 + (c.1 - a.1) * (v - a.0) / (c.0 - a.0).max(1e-9) +} + +fn color_stabilizer(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let Some(reference) = host.self_at(ctx.params.f("referenceFrame"), ctx.env.effect_index) else { return b }; + let r = ctx.params.f("sampleSize").max(0.5); + let pts = [ctx.params.v2("blackPoint"), ctx.params.v2("midPoint"), ctx.params.v2("whitePoint")]; + let cur = pts.map(|pt| sample_mean(&b, pt, r)); + let refv = pts.map(|pt| sample_mean(&reference, pt, r)); + let mode = ctx.params.e("stabilize"); + map_px(&mut b, |c| { + [0, 1, 2].map(|k| { + let v = c[k] as f64; + let o = match mode { + // Brightness: one sample (black point) — offset each channel. + 0 => v + refv[0][k] - cur[0][k], + // Levels: black and white samples — linear map per channel. + 1 => { + let (c0, c1, r0, r1) = (cur[0][k], cur[2][k], refv[0][k], refv[2][k]); + if (c1 - c0).abs() < 1e-6 { v + r0 - c0 } else { r0 + (v - c0) * (r1 - r0) / (c1 - c0) } + } + // Curves: three samples — a curve through them per channel. + _ => curve3([cur[0][k], cur[1][k], cur[2][k]], [refv[0][k], refv[1][k], refv[2][k]], v), + }; + o as f32 + }) + }); + b +} + +fn sl(lo: f64, hi: f64, smin: f64, smax: f64) -> ParamUi { + slider(lo, hi, smin, smax, 3) +} + +pub fn specs() -> Vec { + let cc = "Color Correction"; + let srgb = SPACES.iter().position(|s| s.name == "sRGB").unwrap_or(0) as u32; + let lin709 = SPACES.iter().position(|s| s.name == "Linear Rec.709 (sRGB)").unwrap_or(0) as u32; + let acescg = SPACES.iter().position(|s| s.name == "ACEScg").unwrap_or(0) as u32; + let mut look_opts = vec!["None"]; + look_opts.extend(looks().iter().map(|(n, _)| *n)); + vec![ + spec( + "ec.color.ociocdl", + "OCIO CDL Transform", + cc, + vec![ + p("slopeRed", "Slope Red", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("slopeGreen", "Slope Green", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("slopeBlue", "Slope Blue", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("offsetRed", "Offset Red", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("offsetGreen", "Offset Green", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("offsetBlue", "Offset Blue", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("powerRed", "Power Red", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("powerGreen", "Power Green", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("powerBlue", "Power Blue", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("saturation", "Saturation", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("style", "Style", Value::Enum(0), popup(&["ASC CDL v1.2", "No Clamp"])), + direction(), + ], + ocio_cdl, + ), + spec( + "ec.color.ociocolorspace", + "OCIO Color Space Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", lin709), + space_param("destination", "Destination", srgb), + direction(), + ], + ocio_cst, + ), + spec( + "ec.color.ociodisplay", + "OCIO Display Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", lin709), + p("display", "Display", Value::Enum(0), popup(&DISPLAYS.iter().map(|d| d.0).collect::>())), + p("view", "View", Value::Enum(0), popup(VIEWS)), + direction(), + ], + ocio_display, + ), + // `file`: a path to a .cube/.3dl/.csp/.cc/.ccc/.cdl file, or the file's text itself. + spec( + "ec.color.ociofile", + "OCIO File Transform", + cc, + vec![ + p("file", "File", Value::Str(String::new()), ParamUi::Text), + p("cccId", "CCC ID", Value::Str(String::new()), ParamUi::Text), + p("interpolation", "Interpolation", Value::Enum(3), popup(&["Nearest", "Linear", "Tetrahedral", "Best"])), + direction(), + ], + ocio_file, + ), + spec( + "ec.color.ociolook", + "OCIO Look Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", acescg), + space_param("destination", "Destination", acescg), + p("look", "Look", Value::Enum(0), popup(&look_opts)), + p("looks", "Looks", Value::Str(String::new()), ParamUi::Text), + direction(), + ], + ocio_look, + ), + spec( + "ec.color.colorstabilizer", + "Color Stabilizer", + cc, + vec![ + p("stabilize", "Stabilize", Value::Enum(0), popup(&["Brightness", "Levels", "Curves"])), + p("referenceFrame", "Reference Frame (s)", num(0.0), slider(-100000.0, 100000.0, 0.0, 60.0, 2)), + p("blackPoint", "Black Point", Value::Vec2([0.25, 0.75]), ParamUi::Point), + p("midPoint", "Mid Point", Value::Vec2([0.5, 0.5]), ParamUi::Point), + p("whitePoint", "White Point", Value::Vec2([0.75, 0.25]), ParamUi::Point), + p("sampleSize", "Sample Size", num(5.0), slider(1.0, 100.0, 1.0, 50.0, 0)), + ], + color_stabilizer, + ), + spec( + "ec.utility.colorprofileconverter", + "Color Profile Converter", + "Utility", + vec![ + p("inputProfile", "Input Profile", Value::Enum(0), popup(PROFILES)), + p("linearizeInputProfile", "Linearize Input Profile", Value::Bool(false), ParamUi::Checkbox), + p("outputProfile", "Output Profile", Value::Enum(1), popup(PROFILES)), + p("linearizeOutputProfile", "Linearize Output Profile", Value::Bool(false), ParamUi::Checkbox), + p("intent", "Intent", Value::Enum(1), popup(&["Perceptual", "Relative Colorimetric", "Saturation", "Absolute Colorimetric"])), + ], + profile_converter, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, Image, LayerPixels, run_fx}; + + fn close(a: M3, b: M3, tol: f64) { + for i in 0..3 { + for j in 0..3 { + assert!((a[i][j] - b[i][j]).abs() < tol, "{a:?} vs {b:?}"); + } + } + } + + #[test] + fn aces_matrices_match_published_values() { + let ap0 = space_by_name("ACES2065-1").unwrap(); + let ap1 = space_by_name("ACEScg").unwrap(); + // Academy S-2014-004 AP0 → AP1. + close( + matrix(ap0, ap1), + [ + [1.451_439_316_1, -0.236_510_746_9, -0.214_928_569_3], + [-0.076_553_773_4, 1.176_229_699_8, -0.099_675_926_4], + [0.008_316_148_4, -0.006_032_449_8, 0.997_716_301_4], + ], + 2e-4, + ); + // ACEScg → linear Rec.709 with Bradford adaptation (published ACES config value). + let lin = space_by_name("Linear Rec.709 (sRGB)").unwrap(); + close(matrix(ap1, lin), [[1.705_05, -0.621_79, -0.083_26], [-0.130_26, 1.140_80, -0.010_55], [-0.024_00, -0.128_97, 1.152_97]], 2e-3); + // White maps to white. + let w = mul_vec(&matrix(ap1, lin), [1.0, 1.0, 1.0]); + assert!(w.iter().all(|v| (v - 1.0).abs() < 1e-3), "{w:?}"); + } + + #[test] + fn transfer_functions_round_trip() { + for tf in [Tf::Srgb, Tf::Gamma(2.4), Tf::AcesCct, Tf::Pq] { + for v in [0.0, 0.001, 0.18, 0.5, 1.0, 4.0] { + let r = tf.decode(tf.encode(v)); + assert!((r - v).abs() < 1e-6 * v.max(1.0), "{tf:?} {v} {r}"); + } + } + // ACEScct of 18 % grey (S-2016-001). + assert!((Tf::AcesCct.encode(0.18) - 0.413_588).abs() < 1e-5); + // PQ: 100 nits (1.0) encodes to ≈ 0.508. + assert!((Tf::Pq.encode(1.0) - 0.508).abs() < 1e-3); + } + + #[test] + fn color_space_transform_effect_round_trips() { + let img = Image::filled(4, 4, [0.2, 0.5, 0.7, 1.0]); + let srgb = SPACES.iter().position(|s| s.name == "sRGB").unwrap() as u32; + let cg = SPACES.iter().position(|s| s.name == "ACEScg").unwrap() as u32; + let a = run_fx("ec.color.ociocolorspace", &[("source", Value::Enum(srgb)), ("destination", Value::Enum(cg))], img.clone(), 0.0, EffectEnv::default()); + assert_ne!(a.img.data, img.data); + let b = run_fx("ec.color.ociocolorspace", &[("source", Value::Enum(cg)), ("destination", Value::Enum(srgb))], a.img.clone(), 0.0, EffectEnv::default()); + let c = run_fx( + "ec.color.ociocolorspace", + &[("source", Value::Enum(srgb)), ("destination", Value::Enum(cg)), ("direction", Value::Enum(1))], + a.img, + 0.0, + EffectEnv::default(), + ); + for (x, y) in b.img.data.iter().zip(&img.data).chain(c.img.data.iter().zip(&img.data)) { + assert!((0..4).all(|k| (x[k] - y[k]).abs() < 1e-4), "{x:?} {y:?}"); + } + } + + #[test] + fn cdl_math() { + let cdl = Cdl { slope: [1.2, 1.0, 0.8], offset: [0.05, 0.0, -0.02], power: [1.1, 1.0, 0.9], sat: 0.8 }; + let c = [0.3, 0.5, 0.7]; + // Hand computation of the v1.2 formula. + let sop = [(0.3f64 * 1.2 + 0.05).powf(1.1), 0.5, (0.7f64 * 0.8 - 0.02).powf(0.9)]; + let l = sop[0] * 0.2126 + sop[1] * 0.7152 + sop[2] * 0.0722; + let want = sop.map(|v| l + 0.8 * (v - l)); + let got = cdl.apply(c, true); + for k in 0..3 { + assert!((got[k] - want[k]).abs() < 1e-12); + } + let back = cdl.invert(got, true); + for k in 0..3 { + assert!((back[k] - c[k]).abs() < 1e-9); + } + // v1.2 clamps, No Clamp keeps negatives. + let neg = Cdl { offset: [-0.5; 3], ..Cdl::default() }; + assert_eq!(neg.apply([0.2; 3], true), [0.0; 3]); + assert!((neg.apply([0.2; 3], false)[0] + 0.3).abs() < 1e-12); + let img = Image::filled(2, 2, [0.3, 0.5, 0.7, 1.0]); + let out = run_fx("ec.color.ociocdl", &[("slopeRed", num(1.2)), ("offsetRed", num(0.05)), ("powerRed", num(1.1))], img, 0.0, EffectEnv::default()); + let r = (0.3f64 * 1.2 + 0.05).powf(1.1); + assert!((out.img.get(0, 0)[0] as f64 - r).abs() < 1e-5); + } + + fn identity_lattice(n: usize, f: impl Fn([f32; 3]) -> [f32; 3]) -> Vec<[f32; 3]> { + let mut v = vec![]; + for b in 0..n { + for g in 0..n { + for r in 0..n { + let s = (n - 1) as f32; + v.push(f([r as f32 / s, g as f32 / s, b as f32 / s])); + } + } + } + v + } + + #[test] + fn lut_files_parse_and_interpolate() { + // .cube with a non-linear lattice: trilinear between lattice points. + let f = |c: [f32; 3]| [c[0] * c[0], c[1], 1.0 - c[2]]; + let mut cube = String::from("LUT_3D_SIZE 3\n"); + for r in identity_lattice(3, f) { + cube += &format!("{} {} {}\n", r[0], r[1], r[2]); + } + let Some(FileXform::Lut { lut, .. }) = parse_file(&cube, "cube") else { panic!() }; + // Trilinear between 0.5 (0.25) and 1.0 (1.0) at 0.75 → 0.625. + let o = lut.apply_interp([0.75, 0.3, 0.2], 1); + assert!((o[0] - 0.625).abs() < 1e-6 && (o[1] - 0.3).abs() < 1e-6 && (o[2] - 0.8).abs() < 1e-6, "{o:?}"); + // Nearest snaps. + assert_eq!(lut.apply_interp([0.74, 0.0, 0.0], 0)[0], 0.25); + // .3dl (blue fastest, 10-bit integers, mesh line). + let mut tdl = String::from("0 511 1023\n"); + for r in 0..3 { + for g in 0..3 { + for b in 0..3 { + let v = [r, g, b].map(|i| i as f64 * 511.5); + tdl += &format!("{} {} {}\n", v[0].round(), v[1].round(), (1023.0 - v[2]).round()); + } + } + } + let Some(FileXform::Lut { lut, .. }) = parse_file(&tdl, "3dl") else { panic!() }; + let o = lut.apply_interp([0.25, 0.5, 0.25], 1); + assert!((o[0] - 0.25).abs() < 2e-3 && (o[1] - 0.5).abs() < 2e-3 && (o[2] - 0.75).abs() < 2e-3, "{o:?}"); + // .csp with a squaring pre-LUT on red and an identity 2³ lattice. + let mut csp = String::from("CSPLUTV100\n3D\n\n3\n0 0.5 1\n0 0.25 1\n2\n0 1\n0 1\n2\n0 1\n0 1\n\n2 2 2\n"); + for r in identity_lattice(2, |c| c) { + csp += &format!("{} {} {}\n", r[0], r[1], r[2]); + } + let x = parse_file(&csp, "csp").unwrap(); + let o = x.apply([0.5, 0.4, 0.6], 2, false, ""); + assert!((o[0] - 0.25).abs() < 1e-6 && (o[1] - 0.4).abs() < 1e-6, "{o:?}"); + let inv = x.apply(o, 2, true, ""); + assert!((inv[0] - 0.5).abs() < 1e-4, "{inv:?}"); + // CDL collection: pick by id. + let ccc = r#"2 2 20 0 01 1 1 + 0.5 0.5 0.51"#; + let x = parse_file(ccc, "ccc").unwrap(); + assert!((x.apply([0.4; 3], 2, false, "b")[0] - 0.2).abs() < 1e-6); + assert!((x.apply([0.4; 3], 2, false, "a")[0] - 0.8).abs() < 1e-6); + assert!((x.apply([0.4; 3], 2, false, "")[0] - 0.8).abs() < 1e-6); + // Effect with inline content; inverse of a 3D lattice. + let img = Image::filled(2, 2, [0.6, 0.3, 0.2, 1.0]); + let out = run_fx("ec.color.ociofile", &[("file", Value::Str(cube.clone()))], img.clone(), 0.0, EffectEnv::default()); + let back = run_fx("ec.color.ociofile", &[("file", Value::Str(cube)), ("direction", Value::Enum(1))], out.img, 0.0, EffectEnv::default()); + assert!((back.img.get(0, 0)[0] - 0.6).abs() < 1e-3, "{:?}", back.img.get(0, 0)); + assert!(parse_file("garbage", "3dl").is_none()); + } + + #[test] + fn display_and_look_transforms() { + // Linear 0.18 through the sRGB display / Standard view = sRGB encoding of 0.18. + let img = Image::filled(2, 2, [0.18, 0.18, 0.18, 1.0]); + let out = run_fx("ec.color.ociodisplay", &[], img.clone(), 0.0, EffectEnv::default()); + assert!((out.img.get(0, 0)[0] as f64 - Tf::Srgb.encode(0.18)).abs() < 1e-5); + let back = run_fx("ec.color.ociodisplay", &[("direction", Value::Enum(1))], out.img, 0.0, EffectEnv::default()); + assert!((back.img.get(0, 0)[0] - 0.18).abs() < 1e-5); + // Tone mapped view keeps highlights below 1. + let hot = Image::filled(1, 1, [8.0, 8.0, 8.0, 1.0]); + let tm = run_fx("ec.color.ociodisplay", &[("view", Value::Enum(1))], hot, 0.0, EffectEnv::default()); + assert!(tm.img.get(0, 0)[0] < 1.0); + assert!((tonemap_inv(tonemap(3.0)) - 3.0).abs() < 1e-9); + // Looks: Warm raises red over blue; its inverse undoes it. + let grey = Image::filled(1, 1, [0.18, 0.18, 0.18, 1.0]); + let w = run_fx("ec.color.ociolook", &[("looks", Value::Str("Warm".into()))], grey.clone(), 0.0, EffectEnv::default()); + let px = w.img.get(0, 0); + assert!(px[0] > px[2]); + let u = run_fx("ec.color.ociolook", &[("looks", Value::Str("-Warm".into()))], w.img, 0.0, EffectEnv::default()); + assert!((u.img.get(0, 0)[0] - 0.18).abs() < 1e-4 && (u.img.get(0, 0)[2] - 0.18).abs() < 1e-4); + assert_eq!(parse_looks("Warm, -Cool, nope").len(), 2); + } + + #[test] + fn profile_converter_converts_and_round_trips() { + let img = Image::filled(2, 2, [0.8, 0.4, 0.1, 1.0]); + // sRGB → Rec. 2020 (relative), then back. + let a = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(1)), ("outputProfile", Value::Enum(3))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_ne!(a.img.data, img.data); + let b = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1))], + a.img, + 0.0, + EffectEnv::default(), + ); + assert!((0..3).all(|k| (b.img.get(0, 0)[k] - img.get(0, 0)[k]).abs() < 1e-4)); + // Same profile both sides: identity. Working space defaults to sRGB. + let c = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(0)), ("outputProfile", Value::Enum(1))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(c.img.data, img.data); + // Perceptual pulls out-of-gamut colours inside: saturated Rec.2020 green into sRGB. + let g = Image::filled(1, 1, [0.0, 1.0, 0.0, 1.0]); + let rel = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1))], + g.clone(), + 0.0, + EffectEnv::default(), + ); + assert!(rel.img.get(0, 0)[0] < 0.0); + let per = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1)), ("intent", Value::Enum(0))], + g, + 0.0, + EffectEnv::default(), + ); + assert!(per.img.get(0, 0).iter().all(|v| *v >= -1e-6)); + } + + struct RefHost(Image); + impl EffectHost for RefHost { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, _: f64, _: usize) -> Option { + Some(Buf { img: self.0.clone(), offset: [0.0; 2], scale: 1.0 }) + } + } + + #[test] + fn color_stabilizer_matches_the_reference_frame() { + let mk = |k: f32, o: f32| { + let mut img = Image::new(40, 40); + for y in 0..40 { + for x in 0..40 { + let v = (x as f32 / 40.0) * k + o; + img.set(x, y, [v, v * 0.9, v * 0.8, 1.0]); + } + } + img + }; + let reference = mk(1.0, 0.0); + let flicker = mk(0.8, 0.1); + let host = RefHost(reference.clone()); + let env = EffectEnv { host: Some(&host), ..Default::default() }; + let pts = [("blackPoint", Value::Vec2([5.0, 20.0])), ("midPoint", Value::Vec2([20.0, 20.0])), ("whitePoint", Value::Vec2([35.0, 20.0]))]; + let mut vals: Vec<(&str, Value)> = pts.to_vec(); + vals.push(("stabilize", Value::Enum(1))); + vals.push(("sampleSize", num(1.0))); + let out = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, env); + for x in [5i64, 20, 35] { + let (o, r) = (out.img.get(x, 20), reference.get(x, 20)); + assert!((o[0] - r[0]).abs() < 0.01, "x {x}: {o:?} vs {r:?}"); + } + // Deterministic, and a no-op without a host. + let again = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, env); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, flicker.data); + // Brightness mode corrects an offset at the black point. + let lifted = mk(1.0, 0.1); + let mut vals2: Vec<(&str, Value)> = pts.to_vec(); + vals2.push(("sampleSize", num(1.0))); + let b = run_fx("ec.color.colorstabilizer", &vals2, lifted, 0.0, env); + assert!((b.img.get(5, 20)[0] - reference.get(5, 20)[0]).abs() < 0.01); + } +} diff --git a/crates/effects/src/sim3.rs b/crates/effects/src/sim3.rs new file mode 100644 index 00000000..66e9e11c --- /dev/null +++ b/crates/effects/src/sim3.rs @@ -0,0 +1,716 @@ +//! Simulation: Particle Playground and CC Hair. +//! +//! **Particle Playground** is a deterministic 2D particle system stepped from layer time 0 at +//! [`SPS`] steps per second through [`SimCache`] (seeking anywhere gives the same frame as +//! playing up to it). Producers: **Cannon** (a stream from a barrel with angle, radius, rate, +//! velocity and random spreads), **Grid** (particles placed on a grid at time 0) and **Layer +//! Exploder** (another layer broken into particles at time 0). Forces: **Gravity** (force, +//! random spread, direction), **Repel** (particles push each other apart within a radius), +//! **Wall** (a mask of the layer the particles bounce off) and the **Persistent Property +//! Mapper** (a map layer's red / green / blue drive particle properties at the particle's +//! position each step). **Layer Map** draws particles with the colours of another layer +//! sampled at each particle's birth position. Map layers are read at the frame being rendered +//! (their pixels are part of the cache key). Particles replace the layer's own pixels, as in +//! After Effects. +//! +//! **CC Hair** grows strands from the layer's opaque pixels (Density per 1 000 px²), each a +//! quadratic curve of Length that droops with Weight, optionally steered and scaled by a hairfall +//! map layer (luminance → length, horizontal/vertical gradient → lean), and shades them with a +//! Kajiya–Kay-style strand model (diffuse ∝ sin(tangent, light), specular from the half vector). + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::sim::{Acc, SPS, Shape, Sprite, combine, splat, steps_at}; +use crate::util::{SimCache, hash1, params_key, point_in_poly, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Simulation", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- Particle Playground + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PgParticle { + pub p: [f32; 2], + pub v: [f32; 2], + /// Layer position where the particle was born (Layer Map lookups). + pub origin: [f32; 2], + pub c: [f32; 4], + pub r: f32, + pub mass: f32, + pub friction: f32, + pub force: [f32; 2], + pub id: u32, +} + +#[derive(Clone, Debug, Default)] +pub struct PgState { + pub parts: Vec, + carry: f64, + next_id: u32, +} + +/// A map layer sampled in layer coordinates. +struct MapImg { + img: Image, + /// Layer point → pixel: (x · k₀ + o₀, y · k₁ + o₁). + k: [f64; 2], + o: [f64; 2], +} + +impl MapImg { + fn from(ctx: &EffectCtx, lp: crate::LayerPixels) -> MapImg { + // Fit the map layer to this layer (stretch), like AE's layer maps. + let (ls, os) = (ctx.layer_size, lp.size); + let sx = if ls[0] > 0.0 { os[0] / ls[0] } else { 1.0 }; + let sy = if ls[1] > 0.0 { os[1] / ls[1] } else { 1.0 }; + MapImg { img: lp.buf.img, k: [lp.buf.scale * sx, lp.buf.scale * sy], o: lp.buf.offset } + } + fn at(&self, x: f32, y: f32) -> Px { + let (c, a) = unpremul(self.img.sample_bilinear_clamped(x as f64 * self.k[0] + self.o[0], y as f64 * self.k[1] + self.o[1])); + [c[0], c[1], c[2], a] + } +} + +pub const MAP_TARGETS: [&str; 12] = + ["None", "Red", "Green", "Blue", "Scale", "Opacity", "X Velocity", "Y Velocity", "X Force", "Y Force", "Kinetic Friction", "Mass"]; + +struct Pg { + // Cannon + cannon: bool, + pos: [f32; 2], + barrel_angle: f32, + barrel_radius: f32, + rate: f64, + dir_spread: f32, + vel: f32, + vel_spread: f32, + color: [f32; 4], + radius: f32, + // Gravity + gravity: [f32; 2], + grav_spread: f32, + // Repel + repel: f32, + repel_radius: f32, + // Wall + wall: Option>, + // Property mapper + mapper: Option, + map_to: [(u32, f32, f32); 3], + seed: u32, + bounds: [f32; 4], +} + +impl Pg { + fn spawn(&self, id: u32) -> PgParticle { + let s = self.seed; + let h = |k: u32| hash1(id, k, s) * 2.0 - 1.0; + // AE angles: 0° = up, clockwise. + let a = (self.barrel_angle + h(1) * self.dir_spread * 0.5).to_radians(); + let d = [a.sin(), -a.cos()]; + let perp = [-d[1], d[0]]; + let off = h(2) * self.barrel_radius; + let speed = (self.vel + h(3) * self.vel_spread).max(0.0); + let p = [self.pos[0] + perp[0] * off, self.pos[1] + perp[1] * off]; + PgParticle { p, v: [d[0] * speed, d[1] * speed], origin: p, c: self.color, r: self.radius, mass: 1.0, friction: 0.0, force: [0.0; 2], id } + } + + fn step(&self, st: &mut PgState) { + let dt = (1.0 / SPS) as f32; + let s = self.seed; + // Property mapper (persistent): drive properties from the map at the particle position. + if let Some(m) = &self.mapper { + st.parts.par_iter_mut().for_each(|q| { + let px = m.at(q.p[0], q.p[1]); + for (ch, &(target, lo, hi)) in self.map_to.iter().enumerate() { + let v = lo + (hi - lo) * px[ch]; + match target { + 1 => q.c[0] = v, + 2 => q.c[1] = v, + 3 => q.c[2] = v, + 4 => q.r = self.radius * v.max(0.0), + 5 => q.c[3] = v.clamp(0.0, 1.0), + 6 => q.v[0] = v, + 7 => q.v[1] = v, + 8 => q.force[0] = v, + 9 => q.force[1] = v, + 10 => q.friction = v.clamp(0.0, 1.0), + 11 => q.mass = v.max(0.01), + _ => {} + } + } + }); + } + // Repel: spatial hash, symmetric pairwise push within the radius. + let mut push = vec![[0.0f32; 2]; st.parts.len()]; + if self.repel != 0.0 && self.repel_radius > 0.0 && st.parts.len() > 1 { + let cell = self.repel_radius; + let mut grid: std::collections::HashMap<(i32, i32), Vec> = std::collections::HashMap::new(); + for (i, q) in st.parts.iter().enumerate() { + grid.entry(((q.p[0] / cell).floor() as i32, (q.p[1] / cell).floor() as i32)).or_default().push(i); + } + let parts = &st.parts; + push.par_iter_mut().enumerate().for_each(|(i, f)| { + let q = parts[i]; + let (cx, cy) = ((q.p[0] / cell).floor() as i32, (q.p[1] / cell).floor() as i32); + for gy in cy - 1..=cy + 1 { + for gx in cx - 1..=cx + 1 { + let Some(list) = grid.get(&(gx, gy)) else { continue }; + for &j in list { + if j == i { + continue; + } + let o = parts[j]; + let (dx, dy) = (q.p[0] - o.p[0], q.p[1] - o.p[1]); + let d = (dx * dx + dy * dy).sqrt(); + if d < self.repel_radius && d > 1e-4 { + let k = self.repel * (1.0 - d / self.repel_radius) / d; + f[0] += dx * k; + f[1] += dy * k; + } + } + } + } + }); + } + let wall = self.wall.as_deref(); + st.parts.par_iter_mut().zip(push.par_iter()).for_each(|(q, f)| { + let g = if self.grav_spread > 0.0 { 1.0 + (hash1(q.id, 7, s) * 2.0 - 1.0) * self.grav_spread } else { 1.0 }; + let ax = self.gravity[0] * g + (q.force[0] + f[0]) / q.mass; + let ay = self.gravity[1] * g + (q.force[1] + f[1]) / q.mass; + let damp = 1.0 - q.friction * 0.1; + q.v = [(q.v[0] + ax * dt) * damp, (q.v[1] + ay * dt) * damp]; + let np = [q.p[0] + q.v[0] * dt, q.p[1] + q.v[1] * dt]; + if let Some(w) = wall { + let was = point_in_poly(w, q.p[0] as f64, q.p[1] as f64); + let now = point_in_poly(w, np[0] as f64, np[1] as f64); + if was != now { + // Bounce: reverse the velocity and stay on the same side. + q.v = [-q.v[0] * 0.9, -q.v[1] * 0.9]; + return; + } + } + q.p = np; + }); + let b = self.bounds; + st.parts.retain(|q| q.p[0] > b[0] && q.p[0] < b[2] && q.p[1] > b[1] && q.p[1] < b[3]); + if self.cannon { + st.carry += self.rate / SPS; + let n = st.carry.floor() as u32; + st.carry -= n as f64; + for _ in 0..n { + if st.parts.len() >= 60_000 { + break; + } + let id = st.next_id; + st.next_id = st.next_id.wrapping_add(1); + let mut q = self.spawn(id); + // Sub-step birth offset keeps streams smooth. + let frac = hash1(id, 9, s) * dt; + q.p = [q.p[0] + q.v[0] * frac, q.p[1] + q.v[1] * frac]; + st.parts.push(q); + } + } + } +} + +static PG_CACHE: SimCache = SimCache::new(6); + +fn image_key(img: &Image) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + img.width.hash(&mut h); + img.height.hash(&mut h); + for p in img.data.iter().step_by(7) { + for c in p { + c.to_bits().hash(&mut h); + } + } + h.finish() +} + +fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + let grav_dir = (pr.f("gravityDirection") as f32).to_radians(); + let gf = pr.f("gravityForce") as f32; + let wall = (pr.f("wallBoundary").round() as usize).checked_sub(1).and_then(|i| ctx.env.masks.get(i)).map(|m| m.points.clone()); + let mapper = if pr.b("mapperEnabled") { ctx.layer_param("useLayerAsMap", true).map(|lp| MapImg::from(ctx, lp)) } else { None }; + let pg = Pg { + cannon: pr.b("cannonEnabled"), + pos: { pr.v2("cannonPosition").map(|v| v as f32) }, + barrel_angle: pr.f("barrelAngle") as f32, + barrel_radius: pr.f("barrelRadius") as f32, + rate: pr.f("particlesPerSecond").max(0.0), + dir_spread: pr.f("directionRandomSpread") as f32, + vel: pr.f("velocity") as f32, + vel_spread: pr.f("velocityRandomSpread") as f32, + color: pr.color("cannonColor"), + radius: pr.f("cannonParticleRadius").max(0.0) as f32, + gravity: [grav_dir.sin() * gf, -grav_dir.cos() * gf], + grav_spread: (pr.f("gravityForceRandomSpread") / 100.0).max(0.0) as f32, + repel: pr.f("repelForce") as f32 * 100.0, + repel_radius: pr.f("repelForceRadius").max(0.0) as f32, + wall, + mapper, + map_to: [ + (pr.e("mapRedTo"), pr.f("redMin") as f32, pr.f("redMax") as f32), + (pr.e("mapGreenTo"), pr.f("greenMin") as f32, pr.f("greenMax") as f32), + (pr.e("mapBlueTo"), pr.f("blueMin") as f32, pr.f("blueMax") as f32), + ], + seed: (pr.f("randomSeed") as u32).wrapping_mul(0x9e37_79b9) ^ ctx.seed, + bounds: [-lw * 2.0, -lh * 2.0, lw * 3.0, lh * 3.0], + }; + // Initial particles: Grid and Layer Exploder. + let grid_on = pr.b("gridEnabled"); + let across = pr.f("particlesAcross").max(0.0).round() as u32; + let down = pr.f("particlesDown").max(0.0).round() as u32; + let gpos = pr.v2("gridPosition"); + let (gw, gh) = (pr.f("gridWidth"), pr.f("gridHeight")); + let gcol = pr.color("gridColor"); + let grad = pr.f("gridParticleRadius").max(0.0) as f32; + let exploder = if pr.b("exploderEnabled") { + let host = ctx.env.host; + let lid = pr.get("explodeLayer").and_then(Value::as_layer); + match (host, lid) { + (Some(h), Some(id)) => h.layer_at(id, ctx.env.comp_time - ctx.time, true).or_else(|| h.layer(id, true)), + _ => None, + } + } else { + None + }; + let ex_r = pr.f("radiusOfNewParticles").max(0.5) as f32; + let ex_disp = pr.f("velocityDispersion") as f32; + let layer_map = pr.b("layerMapEnabled").then(|| ctx.layer_param("layerMapLayer", true).map(|lp| MapImg::from(ctx, lp))).flatten(); + + let mut key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 0x7067); + if let Some(m) = &pg.mapper { + key ^= image_key(&m.img).rotate_left(7); + } + if let Some(e) = &exploder { + key ^= image_key(&e.buf.img).rotate_left(13); + } + if let Some(w) = &pg.wall { + key ^= w.iter().fold(0u64, |a, p| a.rotate_left(5) ^ p[0].to_bits() ^ p[1].to_bits().rotate_left(17)); + } + let seed = pg.seed; + let init = || { + let mut st = PgState::default(); + if grid_on && across > 0 && down > 0 { + for j in 0..down { + for i in 0..across { + let fx = if across > 1 { i as f64 / (across - 1) as f64 - 0.5 } else { 0.0 }; + let fy = if down > 1 { j as f64 / (down - 1) as f64 - 0.5 } else { 0.0 }; + let p = [(gpos[0] + fx * gw) as f32, (gpos[1] + fy * gh) as f32]; + let id = st.next_id; + st.next_id += 1; + st.parts.push(PgParticle { p, v: [0.0; 2], origin: p, c: gcol, r: grad, mass: 1.0, friction: 0.0, force: [0.0; 2], id }); + } + } + } + if let Some(e) = &exploder { + let m = MapImg::from(ctx, e.clone()); + let step = (ex_r * 2.0).max(1.0); + let mut y = step * 0.5; + while y < lh { + let mut x = step * 0.5; + while x < lw { + let c = m.at(x, y); + if c[3] > 0.05 { + let id = st.next_id; + st.next_id += 1; + let h = |k: u32| hash1(id, k, seed) * 2.0 - 1.0; + let (dx, dy) = (x - lw * 0.5, y - lh * 0.5); + let l = (dx * dx + dy * dy).sqrt().max(1.0); + let v = [dx / l * ex_disp + h(1) * ex_disp, dy / l * ex_disp + h(2) * ex_disp]; + st.parts.push(PgParticle { p: [x, y], v, origin: [x, y], c, r: ex_r, mass: 1.0, friction: 0.0, force: [0.0; 2], id }); + } + x += step; + } + y += step; + } + } + st + }; + let st = PG_CACHE.run(key, steps_at(ctx.time), init, |st, _| pg.step(st)); + let s = b.scale as f32; + let sprites: Vec = st + .parts + .iter() + .map(|q| { + let (x, y) = b.to_px([q.p[0] as f64, q.p[1] as f64]); + let c = match &layer_map { + Some(m) => { + let mc = m.at(q.origin[0], q.origin[1]); + [mc[0], mc[1], mc[2], mc[3] * q.c[3]] + } + None => q.c, + }; + Sprite::new(x as f32, y as f32, q.r * s, c, Shape::Disc) + }) + .collect(); + let fx = splat(b.img.width, b.img.height, &sprites, Acc::Over); + // Particles replace the layer's own pixels. + b.img = combine(&b.img, &fx, 5); + b +} + +// ---------------------------------------------------------------- CC Hair + +fn cc_hair(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let len = pr.f("length").max(0.0) as f32; + let density = pr.f("density").max(0.0) as f32 / 100.0; + if len <= 0.0 || density <= 0.0 { + return b; + } + let thick = pr.f("thickness").max(0.05) as f32; + let weight = pr.f("weight") as f32; + let const_mass = pr.b("constantMass"); + let map = ctx.layer_param("mapLayer", true).map(|lp| MapImg::from(ctx, lp)); + let map_strength = (pr.f("mapStrength") / 100.0) as f32; + let map_soft = pr.f("mapSoftness").max(0.0) as f32; + let noise = (pr.f("addNoise") / 100.0) as f32; + let hc = pr.color("hairColor"); + let bright = (pr.f("brightness") / 100.0) as f32; + let opacity = (pr.f("opacity") / 100.0) as f32; + let inherit = (pr.f("colorInheritance") / 100.0) as f32; + let light_i = (pr.f("lightIntensity") / 100.0) as f32; + let ld = (pr.f("lightDirection") as f32).to_radians(); + let light = [ld.sin(), -ld.cos()]; + let (amb, dif, spe) = ((pr.f("ambient") / 100.0) as f32, (pr.f("diffuse") / 100.0) as f32, (pr.f("specular") / 100.0) as f32); + let rough = (pr.f("roughness") / 100.0).clamp(0.01, 1.0) as f32; + let seed = ctx.seed ^ (pr.f("randomSeed") as u32).wrapping_mul(0x85eb_ca6b); + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + // Roots: a jittered grid over the layer, density per 1 000 px². + let count = ((lw * lh / 1000.0) * density * 10.0).min(200_000.0) as u32; + let src = b.img.clone(); + let segs = 6u32; + let strands: Vec> = (0..count) + .into_par_iter() + .filter_map(|i| { + let rx = hash1(i, 1, seed) * lw; + let ry = hash1(i, 2, seed) * lh; + let (bx, by) = b.to_px([rx as f64, ry as f64]); + let root = src.sample_bilinear(bx, by); + if root[3] < 0.5 { + return None; + } + let (rc, _) = unpremul(root); + let mut l = len * (0.75 + 0.5 * hash1(i, 3, seed)); + // Lean: random, plus the map's local gradient (hairfall map). + let mut lean = [(hash1(i, 4, seed) * 2.0 - 1.0) * 0.6, -1.0 + hash1(i, 5, seed) * 0.4]; + if let Some(m) = &map { + let lum = |x: f32, y: f32| { + let c = m.at(x, y); + (0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]) * c[3] + }; + let d = map_soft.max(1.0); + let (gx, gy) = (lum(rx + d, ry) - lum(rx - d, ry), lum(rx, ry + d) - lum(rx, ry - d)); + l *= 1.0 - map_strength + map_strength * lum(rx, ry); + lean = [lean[0] + gx * map_strength * 4.0, lean[1] + gy * map_strength * 4.0]; + if noise > 0.0 { + lean[0] += (hash1(i, 6, seed) * 2.0 - 1.0) * noise; + } + } + let ll = (lean[0] * lean[0] + lean[1] * lean[1]).sqrt().max(1e-4); + let dir = [lean[0] / ll, lean[1] / ll]; + // Droop: heavier (or thinner with Constant Mass off) hair bends more. + let droop = weight * if const_mass { 1.0 } else { 1.0 / thick.max(0.2) } * l; + let base = [rc[0] + (hc[0] - rc[0]) * (1.0 - inherit), rc[1] + (hc[1] - rc[1]) * (1.0 - inherit), rc[2] + (hc[2] - rc[2]) * (1.0 - inherit)]; + let pt = |s: f32| [rx + dir[0] * l * s, ry + dir[1] * l * s + droop * s * s]; + let mut out = Vec::with_capacity(segs as usize); + for k in 0..segs { + let (s0, s1) = (k as f32 / segs as f32, (k + 1) as f32 / segs as f32); + let (a, c) = (pt(s0), pt(s1)); + let t = { + let (tx, ty) = (c[0] - a[0], c[1] - a[1]); + let tl = (tx * tx + ty * ty).sqrt().max(1e-6); + [tx / tl, ty / tl] + }; + // Strand shading (Kajiya–Kay): diffuse ∝ sin(T, L), specular from T·L. + let tl = t[0] * light[0] + t[1] * light[1]; + let diff = (1.0 - tl * tl).max(0.0).sqrt(); + let spec = (1.0 - tl.abs()).powf(1.0 / rough) * spe; + let shade = (amb + dif * diff * light_i) * bright; + let tip = 1.0 - 0.6 * s1; + let c3 = [base[0] * shade + spec * light_i, base[1] * shade + spec * light_i, base[2] * shade + spec * light_i]; + let (pa, pc) = (b.to_px([a[0] as f64, a[1] as f64]), b.to_px([c[0] as f64, c[1] as f64])); + out.push(Sprite { + x: pa.0 as f32, + y: pa.1 as f32, + r: thick * 0.5 * b.scale as f32 * tip, + c: [c3[0], c3[1], c3[2], opacity], + shape: Shape::Line { dx: (pc.0 - pa.0) as f32, dy: (pc.1 - pa.1) as f32 }, + rot: 0.0, + }); + } + Some(out) + }) + .collect(); + let sprites: Vec = strands.into_iter().flatten().collect(); + let fx = splat(b.img.width, b.img.height, &sprites, Acc::Over); + b.img = combine(&b.img, &fx, 0); + b +} + +pub fn specs() -> Vec { + let pt = |x: f64, y: f64| Value::Vec2([x, y]); + let px = |d: f64, max: f64| (num(d), slider(0.0, max * 10.0, 0.0, max, 1)); + let sp = |id: &'static str, name: &'static str, v: (Value, ParamUi)| p(id, name, v.0, v.1); + let mut pg = vec![ + // Cannon + p("cannonEnabled", "Cannon", Value::Bool(true), ParamUi::Checkbox), + p("cannonPosition", "Position", pt(0.5, 0.9), ParamUi::Point), + p("barrelAngle", "Barrel Angle", num(0.0), ParamUi::Angle), + sp("barrelRadius", "Barrel Radius", px(0.0, 100.0)), + sp("particlesPerSecond", "Particles Per Second", px(60.0, 500.0)), + sp("directionRandomSpread", "Direction Random Spread", px(20.0, 360.0)), + sp("velocity", "Velocity", px(130.0, 1000.0)), + sp("velocityRandomSpread", "Velocity Random Spread", px(20.0, 500.0)), + p("cannonColor", "Color", col(1.0, 0.0, 0.0), ParamUi::Color), + sp("cannonParticleRadius", "Particle Radius", px(2.0, 50.0)), + // Grid + p("gridEnabled", "Grid", Value::Bool(true), ParamUi::Checkbox), + p("gridPosition", "Grid Position", pt(0.5, 0.5), ParamUi::Point), + sp("gridWidth", "Width", px(100.0, 2000.0)), + sp("gridHeight", "Height", px(100.0, 2000.0)), + sp("particlesAcross", "Particles Across", px(0.0, 100.0)), + sp("particlesDown", "Particles Down", px(0.0, 100.0)), + p("gridColor", "Grid Color", col(1.0, 1.0, 1.0), ParamUi::Color), + sp("gridParticleRadius", "Grid Particle Radius", px(2.0, 50.0)), + // Layer Exploder + p("exploderEnabled", "Layer Exploder", Value::Bool(true), ParamUi::Checkbox), + p("explodeLayer", "Explode Layer", Value::Layer(None), ParamUi::Layer), + sp("radiusOfNewParticles", "Radius of New Particles", px(2.0, 50.0)), + sp("velocityDispersion", "Velocity Dispersion", px(20.0, 500.0)), + // Layer Map + p("layerMapEnabled", "Layer Map", Value::Bool(true), ParamUi::Checkbox), + p("layerMapLayer", "Use Layer", Value::Layer(None), ParamUi::Layer), + // Gravity + sp("gravityForce", "Force", px(108.0, 1000.0)), + sp("gravityForceRandomSpread", "Force Random Spread", px(0.0, 100.0)), + p("gravityDirection", "Direction", num(180.0), ParamUi::Angle), + // Repel + p("repelForce", "Repel Force", num(0.0), slider(-100.0, 100.0, -10.0, 10.0, 2)), + sp("repelForceRadius", "Force Radius", px(0.0, 100.0)), + // Wall + p("wallBoundary", "Boundary", num(0.0), ParamUi::Mask), + // Persistent Property Mapper + p("mapperEnabled", "Persistent Property Mapper", Value::Bool(true), ParamUi::Checkbox), + p("useLayerAsMap", "Use Layer As Map", Value::Layer(None), ParamUi::Layer), + ]; + for (to, mn, mx, n_to, n_min, n_max) in [ + ("mapRedTo", "redMin", "redMax", "Map Red To", "Red Min", "Red Max"), + ("mapGreenTo", "greenMin", "greenMax", "Map Green To", "Green Min", "Green Max"), + ("mapBlueTo", "blueMin", "blueMax", "Map Blue To", "Blue Min", "Blue Max"), + ] { + pg.push(p(to, n_to, Value::Enum(0), popup(&MAP_TARGETS))); + pg.push(p(mn, n_min, num(0.0), slider(-10000.0, 10000.0, -100.0, 100.0, 2))); + pg.push(p(mx, n_max, num(1.0), slider(-10000.0, 10000.0, -100.0, 100.0, 2))); + } + pg.push(p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0))); + let pc = |d: f64| (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)); + vec![ + spec("ec.sim.particleplayground", "Particle Playground", pg, particle_playground), + spec( + "ec.sim.cchair", + "CC Hair", + vec![ + sp("length", "Length", px(30.0, 200.0)), + p("thickness", "Thickness", num(1.0), slider(0.05, 20.0, 0.1, 5.0, 2)), + p("weight", "Weight", num(0.2), slider(-10.0, 10.0, -2.0, 2.0, 2)), + p("constantMass", "Constant Mass", Value::Bool(false), ParamUi::Checkbox), + sp("density", "Density", px(100.0, 1000.0)), + // Hairfall Map + sp("mapStrength", "Map Strength", pc(0.0)), + p("mapLayer", "Map Layer", Value::Layer(None), ParamUi::Layer), + sp("mapSoftness", "Map Softness", px(0.0, 100.0)), + sp("addNoise", "Add Noise", pc(0.0)), + // Hair Color + p("hairColor", "Color", col(0.35, 0.25, 0.15), ParamUi::Color), + sp("brightness", "Brightness", px(100.0, 400.0)), + sp("opacity", "Opacity", pc(100.0)), + sp("colorInheritance", "Color Inheritance", pc(0.0)), + // Light + sp("lightIntensity", "Light Intensity", px(100.0, 400.0)), + p("lightDirection", "Light Direction", num(-45.0), ParamUi::Angle), + // Shading + sp("ambient", "Ambient", pc(40.0)), + sp("diffuse", "Diffuse", pc(60.0)), + sp("specular", "Specular", pc(30.0)), + sp("roughness", "Roughness", pc(20.0)), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + ], + cc_hair, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, MaskShape, run_fx}; + + fn run(vals: &[(&str, Value)], t: f64) -> Image { + run_fx("ec.sim.particleplayground", vals, Image::filled(80, 60, [0.2, 0.2, 0.2, 1.0]), t, EffectEnv::default()).img + } + + fn cannon(t: f64) -> Image { + run(&[("cannonPosition", Value::Vec2([40.0, 50.0])), ("velocity", num(60.0)), ("gravityForce", num(30.0))], t) + } + + #[test] + fn cannon_stream_is_seek_consistent() { + let a = cannon(1.0); + // Particles replace the layer: some coverage, mostly transparent, red. + let cov: f32 = a.data.iter().map(|p| p[3]).sum(); + assert!(cov > 5.0, "{cov}"); + assert!(a.data.iter().any(|p| p[3] > 0.5 && p[0] > 0.5 && p[1] < 0.1)); + // Seek back and forth: the same frame every time. + let _ = cannon(2.0); + let b = cannon(1.0); + assert_eq!(a.data, b.data); + let c = cannon(0.5); + assert_ne!(a.data, c.data); + // Nothing has been fired at time 0. + let z = cannon(0.0); + assert!(z.data.iter().all(|p| p[3] == 0.0)); + } + + #[test] + fn grid_falls_with_gravity_and_bounces_off_a_wall() { + let grid = |t: f64, env: EffectEnv| { + run_fx( + "ec.sim.particleplayground", + &[ + ("cannonEnabled", Value::Bool(false)), + ("gridPosition", Value::Vec2([40.0, 10.0])), + ("gridWidth", num(40.0)), + ("gridHeight", num(0.0)), + ("particlesAcross", num(5.0)), + ("particlesDown", num(1.0)), + ("gravityForce", num(100.0)), + ("wallBoundary", num(1.0)), + ], + Image::new(80, 60), + t, + env, + ) + .img + }; + // Alpha-weighted mean height of the particles. + let row_of = |img: &Image| { + let (mut s, mut n) = (0.0f64, 0.0f64); + for y in 0..60 { + for x in 0..80 { + let a = img.get(x, y)[3] as f64; + s += (y as f64 + 0.5) * a; + n += a; + } + } + (s / n.max(1e-9)).round() as i64 + }; + let t0 = grid(0.0, EffectEnv::default()); + let t1 = grid(0.5, EffectEnv::default()); + assert_eq!(row_of(&t0), 10); + // ½ g t² = 12.5 px. + assert!((row_of(&t1) - 22).abs() <= 2, "{}", row_of(&t1)); + // A wall (mask 1) whose floor is at y = 15 keeps them above it. + let masks = [MaskShape { name: "w".into(), points: vec![[0.0, 0.0], [80.0, 0.0], [80.0, 15.0], [0.0, 15.0]], closed: true, inverted: false }]; + let env = EffectEnv { masks: &masks, ..Default::default() }; + let w = grid(1.5, env); + assert!(row_of(&w) <= 16, "{}", row_of(&w)); + } + + #[test] + fn repel_spreads_particles() { + let spread = |force: f64| { + let img = run( + &[ + ("cannonEnabled", Value::Bool(false)), + ("gridPosition", Value::Vec2([40.0, 30.0])), + ("gridWidth", num(6.0)), + ("gridHeight", num(6.0)), + ("particlesAcross", num(3.0)), + ("particlesDown", num(3.0)), + ("gravityForce", num(0.0)), + ("repelForce", num(force)), + ("repelForceRadius", num(20.0)), + ], + 0.5, + ); + let (mut x0, mut x1) = (80, 0); + for y in 0..60 { + for x in 0..80 { + if img.get(x, y)[3] > 0.3 { + x0 = x0.min(x); + x1 = x1.max(x); + } + } + } + x1 - x0 + }; + assert!(spread(0.2) > spread(0.0) + 4, "{} vs {}", spread(0.2), spread(0.0)); + } + + #[test] + fn property_mapper_and_layer_map_read_layers() { + // Without a host, the layer-based features are inert (deterministic output). + let a = run(&[("mapRedTo", Value::Enum(6)), ("redMin", num(-50.0)), ("redMax", num(50.0))], 0.7); + let b = run(&[("mapRedTo", Value::Enum(6)), ("redMin", num(-50.0)), ("redMax", num(50.0))], 0.7); + assert_eq!(a.data, b.data); + let st = Pg { + cannon: false, + pos: [0.0; 2], + barrel_angle: 0.0, + barrel_radius: 0.0, + rate: 0.0, + dir_spread: 0.0, + vel: 0.0, + vel_spread: 0.0, + color: [1.0; 4], + radius: 2.0, + gravity: [0.0; 2], + grav_spread: 0.0, + repel: 0.0, + repel_radius: 0.0, + wall: None, + mapper: Some(MapImg { img: Image::filled(4, 4, [1.0, 0.0, 0.5, 1.0]), k: [1.0; 2], o: [0.0; 2] }), + map_to: [(6, 0.0, 120.0), (0, 0.0, 1.0), (4, 0.0, 2.0)], + seed: 1, + bounds: [-1e6, -1e6, 1e6, 1e6], + }; + let mut s = PgState::default(); + s.parts.push(PgParticle { p: [1.0, 1.0], v: [0.0; 2], origin: [1.0, 1.0], c: [1.0; 4], r: 2.0, mass: 1.0, friction: 0.0, force: [0.0; 2], id: 0 }); + st.step(&mut s); + // Red 1 → X velocity 120 px/s; blue 0.5 → scale 1 (radius 2). + assert!((s.parts[0].v[0] - 120.0).abs() < 1e-3); + assert!((s.parts[0].r - 2.0).abs() < 1e-3); + } + + #[test] + fn cc_hair_grows_from_opaque_pixels() { + let mut img = Image::new(60, 60); + for y in 20..40 { + for x in 20..40 { + img.set(x, y, [0.8, 0.6, 0.4, 1.0]); + } + } + let out = run_fx("ec.sim.cchair", &[("length", num(12.0)), ("density", num(300.0))], img.clone(), 0.0, EffectEnv::default()); + // Hair reaches beyond the square (above it: hair leans up, then droops). + let outside: f32 = (0..60).flat_map(|x| (0..20).map(move |y| (x, y))).map(|(x, y)| out.img.get(x, y)[3]).sum(); + assert!(outside > 1.0, "{outside}"); + // Nothing grows from transparent areas far away. + assert_eq!(out.img.get(2, 58)[3], 0.0); + let again = run_fx("ec.sim.cchair", &[("length", num(12.0)), ("density", num(300.0))], img.clone(), 3.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.sim.cchair", &[("length", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + } +} diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index 49b7ed36..7689050d 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -153,7 +153,7 @@ fn cineon(ctx: &EffectCtx, mut b: Buf) -> Buf { // ---- Colour LUTs (.cube) ---- #[derive(Clone, Debug, PartialEq)] -pub(crate) struct Lut { +pub struct Lut { pub n1: usize, pub d1: Vec<[f32; 3]>, pub n3: usize, @@ -163,7 +163,7 @@ pub(crate) struct Lut { } /// Parse `.cube` text (TITLE, LUT_1D_SIZE, LUT_3D_SIZE, DOMAIN_MIN/MAX, data red-fastest). -pub(crate) fn parse_cube(text: &str) -> Option { +pub fn parse_cube(text: &str) -> Option { let mut lut = Lut { n1: 0, d1: vec![], n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }; let mut rows: Vec<[f32; 3]> = Vec::new(); let three = |it: &mut std::str::SplitWhitespace| -> Option<[f32; 3]> { @@ -238,6 +238,57 @@ impl Lut { } c } + /// Apply with a choice of 3D interpolation: 0 nearest, 1 trilinear, otherwise tetrahedral. + pub fn apply_interp(&self, c: [f32; 3], interp: u32) -> [f32; 3] { + let mut c = c; + if self.n1 > 1 { + let t = self.norm(c); + c = [0, 1, 2].map(|i| { + let x = t[i] * (self.n1 - 1) as f32; + if interp == 0 { + return self.d1[(x.round() as usize).min(self.n1 - 1)][i]; + } + let i0 = (x.floor() as usize).min(self.n1 - 2); + let f = x - i0 as f32; + self.d1[i0][i] + (self.d1[i0 + 1][i] - self.d1[i0][i]) * f + }); + } + if self.n3 > 1 { + let t = self.norm(c); + c = match interp { + 0 => self.nearest(t), + 1 => self.trilinear(t), + _ => self.tetra(t), + }; + } + c + } + fn nearest(&self, t: [f32; 3]) -> [f32; 3] { + let n = self.n3; + let s = (n - 1) as f32; + let i = |x: f32| ((x * s).round() as usize).min(n - 1); + self.d3[i(t[0]) + i(t[1]) * n + i(t[2]) * n * n] + } + /// Trilinear interpolation on the lattice (red fastest). + pub fn trilinear(&self, t: [f32; 3]) -> [f32; 3] { + let n = self.n3; + let s = (n - 1) as f32; + let idx = |x: f32| { + let v = x * s; + let i = (v.floor() as usize).min(n - 2); + (i, v - i as f32) + }; + let (r, fr) = idx(t[0]); + let (g, fg) = idx(t[1]); + let (b, fb) = idx(t[2]); + let at = |dr: usize, dg: usize, db: usize| self.d3[(r + dr) + (g + dg) * n + (b + db) * n * n]; + let l = |a: [f32; 3], b: [f32; 3], f: f32| [a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f, a[2] + (b[2] - a[2]) * f]; + let c00 = l(at(0, 0, 0), at(1, 0, 0), fr); + let c10 = l(at(0, 1, 0), at(1, 1, 0), fr); + let c01 = l(at(0, 0, 1), at(1, 0, 1), fr); + let c11 = l(at(0, 1, 1), at(1, 1, 1), fr); + l(l(c00, c10, fg), l(c01, c11, fg), fb) + } fn tetra(&self, t: [f32; 3]) -> [f32; 3] { let n = self.n3; let s = (n - 1) as f32; diff --git a/crates/effects/src/vr.rs b/crates/effects/src/vr.rs new file mode 100644 index 00000000..ba7e6d9d --- /dev/null +++ b/crates/effects/src/vr.rs @@ -0,0 +1,1188 @@ +//! Immersive Video effects (VR Blur, VR Chromatic Aberrations, VR Color Gradients, VR Converter, +//! VR De-Noise, VR Digital Glitch, VR Fractal Noise, VR Glow, VR Plane to Sphere, VR Rotate +//! Sphere, VR Sharpen, VR Sphere To Plane). +//! +//! All of them treat the layer as a 360° equirectangular frame (longitude across, latitude down; +//! the front, +Z, at the centre) and are **seam-aware**: filters wrap around the left/right edge, +//! continue across the poles (a column continues on the opposite meridian), and spatial filters +//! widen their horizontal footprint by 1/cos(latitude) so a blur is the same size on the sphere +//! everywhere. Stereo footage (Frame Layout: over/under or side by side) is processed per eye. +//! +//! VR Converter maps between equirectangular, cube maps (4:3 cross, 3:2 and 6:1 strips), an +//! angular fisheye ("sphere map") and a rectilinear 2D view, through a common view direction. + +use std::f64::consts::PI; + +use effectcraft_color::{BlendMode, blend_pixel, luminance}; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::generate::value_noise; +use crate::util::{hash1, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +type V3 = [f64; 3]; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Immersive Video", params, render, gpu: false, float: true } +} + +pub const FRAME_LAYOUTS: [&str; 3] = ["Monoscopic", "Stereo - Over/Under", "Stereo - Side by Side"]; + +fn layout_param() -> crate::ParamSpec { + p("frameLayout", "Frame Layout", Value::Enum(0), popup(&FRAME_LAYOUTS)) +} + +/// Eye rectangles (x, y, w, h) of a frame. +fn eyes(w: u32, h: u32, layout: u32) -> Vec<(u32, u32, u32, u32)> { + match layout { + 1 if h >= 2 => vec![(0, 0, w, h / 2), (0, h / 2, w, h - h / 2)], + 2 if w >= 2 => vec![(0, 0, w / 2, h), (w / 2, 0, w - w / 2, h)], + _ => vec![(0, 0, w, h)], + } +} + +/// Run `f` on every eye of `img` and reassemble. +fn per_eye(img: &Image, layout: u32, f: impl Fn(&Image, usize) -> Image + Sync) -> Image { + let rects = eyes(img.width, img.height, layout); + if rects.len() == 1 { + return f(img, 0); + } + let outs: Vec = rects.par_iter().enumerate().map(|(i, &(x, y, w, h))| f(&img.crop(x as i64, y as i64, w, h), i)).collect(); + let mut out = Image::new(img.width, img.height); + for (&(x0, y0, w, _), o) in rects.iter().zip(&outs) { + for (yy, row) in o.data.chunks(w.max(1) as usize).enumerate() { + let start = out.idx(x0, y0 + yy as u32); + out.data[start..start + row.len()].copy_from_slice(row); + } + } + out +} + +// ---------------------------------------------------------------- sphere geometry + +#[inline] +fn norm(v: V3) -> V3 { + let l = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2]).sqrt().max(1e-300); + [v[0] / l, v[1] / l, v[2] / l] +} + +#[inline] +fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} + +/// Direction (y up, +z front) of an equirectangular position in pixels. +#[inline] +pub fn equi_dir(x: f64, y: f64, w: f64, h: f64) -> V3 { + let lon = (x / w - 0.5) * 2.0 * PI; + let lat = (0.5 - y / h) * PI; + [lat.cos() * lon.sin(), lat.sin(), lat.cos() * lon.cos()] +} + +/// Equirectangular pixel position of a direction. +#[inline] +pub fn equi_px(d: V3, w: f64, h: f64) -> (f64, f64) { + let lon = d[0].atan2(d[2]); + let lat = d[1].clamp(-1.0, 1.0).asin(); + ((lon / (2.0 * PI) + 0.5) * w, (0.5 - lat / PI) * h) +} + +/// Bilinear sample of an equirectangular image: wraps horizontally, continues over the poles. +pub fn sample_equi(img: &Image, x: f64, y: f64) -> Px { + let (w, h) = (img.width as i64, img.height as i64); + if w == 0 || h == 0 { + return [0.0; 4]; + } + let fx = x - 0.5; + let fy = y - 0.5; + let (x0, y0) = (fx.floor(), fy.floor()); + let (tx, ty) = ((fx - x0) as f32, (fy - y0) as f32); + let at = |xi: i64, yi: i64| -> Px { + let (mut xi, mut yi) = (xi, yi); + if yi < 0 { + yi = -1 - yi; + xi += w / 2; + } else if yi >= h { + yi = 2 * h - 1 - yi; + xi += w / 2; + } + img.data[(yi.clamp(0, h - 1) * w + xi.rem_euclid(w)) as usize] + }; + let (x0, y0) = (x0 as i64, y0 as i64); + let (a, b, c, d) = (at(x0, y0), at(x0 + 1, y0), at(x0, y0 + 1), at(x0 + 1, y0 + 1)); + let mut o = [0.0f32; 4]; + for k in 0..4 { + let top = a[k] + (b[k] - a[k]) * tx; + let bot = c[k] + (d[k] - c[k]) * tx; + o[k] = top + (bot - top) * ty; + } + o +} + +/// Rotation matrix (row-major) for tilt (about X), pan (about Y) and roll (about Z), degrees: +/// R = Ry(pan) · Rx(tilt) · Rz(roll). +pub fn rotation(tilt: f64, pan: f64, roll: f64) -> [[f64; 3]; 3] { + let (a, b, c) = (tilt.to_radians(), pan.to_radians(), roll.to_radians()); + let rx = [[1.0, 0.0, 0.0], [0.0, a.cos(), -a.sin()], [0.0, a.sin(), a.cos()]]; + let ry = [[b.cos(), 0.0, b.sin()], [0.0, 1.0, 0.0], [-b.sin(), 0.0, b.cos()]]; + let rz = [[c.cos(), -c.sin(), 0.0], [c.sin(), c.cos(), 0.0], [0.0, 0.0, 1.0]]; + mm(&ry, &mm(&rx, &rz)) +} + +fn mm(a: &[[f64; 3]; 3], b: &[[f64; 3]; 3]) -> [[f64; 3]; 3] { + let mut o = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + o[i][j] = (0..3).map(|k| a[i][k] * b[k][j]).sum(); + } + } + o +} + +#[inline] +fn mv(m: &[[f64; 3]; 3], v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} + +#[inline] +fn mtv(m: &[[f64; 3]; 3], v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} + +// ---------------------------------------------------------------- projections (VR Converter) + +pub const PROJECTIONS: [&str; 6] = ["Equirectangular", "Cube-map 4:3", "Cube-map 3:2", "Cube-map 6:1", "Sphere Map", "2D Rectilinear"]; + +/// Cube faces: (forward, right, up). +const FACES: [(V3, V3, V3); 6] = [ + ([1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), // +X right + ([-1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), // -X left + ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, -1.0]), // +Y top + ([0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), // -Y bottom + ([0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), // +Z front + ([0.0, 0.0, -1.0], [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), // -Z back +]; + +/// Grid size and the (column, row) of each face (+X, −X, +Y, −Y, +Z, −Z) for a cube layout. +fn cube_grid(kind: u32) -> ((u32, u32), [(u32, u32); 6]) { + match kind { + // Horizontal cross: top over the front; left, front, right, back across the middle. + 1 => ((4, 3), [(2, 1), (0, 1), (1, 0), (1, 2), (1, 1), (3, 1)]), + // 3 × 2: right, left, top / bottom, front, back. + 2 => ((3, 2), [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1)]), + // 6 × 1 strip in face order. + _ => ((6, 1), [(0, 0), (1, 0), (2, 0), (3, 0), (4, 0), (5, 0)]), + } +} + +/// A projection: frame position (pixels) ↔ view direction. +#[derive(Clone, Copy, Debug)] +pub struct Proj { + pub kind: u32, + /// Horizontal field of view in degrees (sphere map and 2D only). + pub fov: f64, +} + +impl Proj { + /// Direction seen at frame position (x, y) of a `w`×`h` frame (None outside the image). + pub fn dir(&self, x: f64, y: f64, w: f64, h: f64) -> Option { + match self.kind { + 0 => Some(equi_dir(x, y, w, h)), + 1..=3 => { + let ((gw, gh), cells) = cube_grid(self.kind); + let (cw, ch) = (w / gw as f64, h / gh as f64); + let (cx, cy) = ((x / cw).floor(), (y / ch).floor()); + let f = cells.iter().position(|&(c, r)| c as f64 == cx && r as f64 == cy)?; + let a = (x - cx * cw) / cw * 2.0 - 1.0; + let b = (y - cy * ch) / ch * 2.0 - 1.0; + let (fw, r, u) = FACES[f]; + Some(norm([fw[0] + a * r[0] - b * u[0], fw[1] + a * r[1] - b * u[1], fw[2] + a * r[2] - b * u[2]])) + } + 4 => { + // Angular fisheye centred on +Z; radius ∝ angle, `fov` across the image width. + let half = (self.fov.clamp(1.0, 360.0) / 2.0).to_radians(); + let s = w.min(h) / 2.0; + let (u, v) = ((x - w / 2.0) / s, (h / 2.0 - y) / s); + let r = (u * u + v * v).sqrt(); + if r > 1.0 { + return None; + } + let th = r * half; + let (cu, cv) = if r > 1e-12 { (u / r, v / r) } else { (0.0, 0.0) }; + Some([th.sin() * cu, th.sin() * cv, th.cos()]) + } + _ => { + let t = (self.fov.clamp(1.0, 179.0) / 2.0).to_radians().tan(); + let u = (x / w * 2.0 - 1.0) * t; + let v = (1.0 - y / h * 2.0) * t * h / w; + Some(norm([u, v, 1.0])) + } + } + } + + /// Frame position (pixels) of direction `d`, and for cube maps the face rectangle to clamp + /// samples into. None when the direction is not visible. + pub fn pos(&self, d: V3, w: f64, h: f64) -> Option<(f64, f64, Option<[f64; 4]>)> { + match self.kind { + 0 => { + let (x, y) = equi_px(d, w, h); + Some((x, y, None)) + } + 1..=3 => { + let f = (0..6).max_by(|&i, &j| dot(d, FACES[i].0).total_cmp(&dot(d, FACES[j].0)))?; + let (fw, r, u) = FACES[f]; + let k = dot(d, fw); + let (a, b) = (dot(d, r) / k, -dot(d, u) / k); + let ((gw, gh), cells) = cube_grid(self.kind); + let (cw, ch) = (w / gw as f64, h / gh as f64); + let (c, rr) = cells[f]; + let (x0, y0) = (c as f64 * cw, rr as f64 * ch); + Some((x0 + (a + 1.0) * 0.5 * cw, y0 + (b + 1.0) * 0.5 * ch, Some([x0, y0, x0 + cw, y0 + ch]))) + } + 4 => { + let half = (self.fov.clamp(1.0, 360.0) / 2.0).to_radians(); + let th = d[2].clamp(-1.0, 1.0).acos(); + if th > half + 1e-9 { + return None; + } + let s = w.min(h) / 2.0; + let r = th / half; + let l = (d[0] * d[0] + d[1] * d[1]).sqrt(); + let (cu, cv) = if l > 1e-12 { (d[0] / l, d[1] / l) } else { (0.0, 0.0) }; + Some((w / 2.0 + cu * r * s, h / 2.0 - cv * r * s, None)) + } + _ => { + if d[2] <= 1e-9 { + return None; + } + let t = (self.fov.clamp(1.0, 179.0) / 2.0).to_radians().tan(); + let u = d[0] / d[2] / t; + let v = d[1] / d[2] / (t * h / w); + if u.abs() > 1.0 || v.abs() > 1.0 { + return None; + } + Some(((u + 1.0) * 0.5 * w, (1.0 - v) * 0.5 * h, None)) + } + } + } + + /// Sample image `img` (in this projection) in direction `d`. + pub fn sample(&self, img: &Image, d: V3) -> Px { + let (w, h) = (img.width as f64, img.height as f64); + match self.pos(d, w, h) { + None => [0.0; 4], + Some((x, y, None)) => { + if self.kind == 0 { + sample_equi(img, x, y) + } else { + img.sample_bilinear_clamped(x, y) + } + } + Some((x, y, Some(r))) => { + // Stay inside the face so neighbouring faces don't bleed in. + let x = x.clamp(r[0] + 0.5, r[2] - 0.5); + let y = y.clamp(r[1] + 0.5, r[3] - 0.5); + img.sample_bilinear_clamped(x, y) + } + } + } +} + +/// Re-project `src` (in `from`) into a `w`×`h` frame in `to`, rotating the view by `rot` +/// (output direction → source direction = rotᵀ · d). +pub fn convert(src: &Image, from: Proj, to: Proj, w: u32, h: u32, rot: Option<&[[f64; 3]; 3]>) -> Image { + let mut out = Image::new(w, h); + let (fw, fh) = (w as f64, h as f64); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = match to.dir(x as f64 + 0.5, y as f64 + 0.5, fw, fh) { + Some(d) => { + let d = match rot { + Some(m) => mtv(m, d), + None => d, + }; + from.sample(src, d) + } + None => [0.0; 4], + }; + } + }); + out +} + +// ---------------------------------------------------------------- seam-aware filters + +/// Horizontal box mean with wrap-around, radius `r` (pixels, ≤ w/2) on one row. +fn box_row_wrap(src: &[Px], dst: &mut [Px], r: usize) { + let n = src.len(); + if n == 0 { + return; + } + if r == 0 { + dst.copy_from_slice(src); + return; + } + let r = r.min((n - 1) / 2); + if r == 0 { + dst.copy_from_slice(src); + return; + } + let k = (2 * r + 1) as f64; + let mut acc = [0.0f64; 4]; + for i in 0..=2 * r { + let p = src[(i + n - r) % n]; + for c in 0..4 { + acc[c] += p[c] as f64; + } + } + for x in 0..n { + for c in 0..4 { + dst[x][c] = (acc[c] / k) as f32; + } + let add = src[(x + r + 1) % n]; + let sub = src[(x + n - r) % n]; + for c in 0..4 { + acc[c] += add[c] as f64 - sub[c] as f64; + } + } +} + +/// Vertical box mean with pole continuation, radius `r`, over all columns. +fn box_cols_pole(img: &Image, r: usize) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + if r == 0 || h == 0 { + return img.clone(); + } + let at = |x: usize, y: i64| -> Px { + let (mut x, mut y) = (x, y); + if y < 0 { + y = -1 - y; + x = (x + w / 2) % w; + } else if y >= h as i64 { + y = 2 * h as i64 - 1 - y; + x = (x + w / 2) % w; + } + img.data[y.clamp(0, h as i64 - 1) as usize * w + x] + }; + let k = (2 * r + 1) as f64; + let cols: Vec> = (0..w) + .into_par_iter() + .map(|x| { + let mut out = vec![[0.0f32; 4]; h]; + let mut acc = [0.0f64; 4]; + for j in -(r as i64)..=(r as i64) { + let p = at(x, j); + for c in 0..4 { + acc[c] += p[c] as f64; + } + } + for y in 0..h { + for c in 0..4 { + out[y][c] = (acc[c] / k) as f32; + } + let add = at(x, y as i64 + r as i64 + 1); + let sub = at(x, y as i64 - r as i64); + for c in 0..4 { + acc[c] += add[c] as f64 - sub[c] as f64; + } + } + out + }) + .collect(); + let mut out = Image::new(img.width, img.height); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = cols[x][y]; + } + }); + out +} + +/// Gaussian-like blur of an equirectangular image with standard deviation `sigma` pixels at +/// the equator (3 box passes per axis), wrapping at the seam and continuing over the poles. +pub fn sphere_blur(img: &Image, sigma: f64) -> Image { + if sigma < 0.1 || img.is_empty() { + return img.clone(); + } + let (w, h) = (img.width as usize, img.height as usize); + let radii = crate::util::box_radii(sigma, 3); + let mut cur = img.clone(); + let mut tmp = Image::new(img.width, img.height); + for &r in &radii { + tmp.rows_mut().for_each(|(y, row)| { + let lat = (0.5 - (y as f64 + 0.5) / h as f64) * PI; + let rr = ((r as f64) / lat.cos().max(1e-3)).round().min((w / 2) as f64) as usize; + box_row_wrap(&cur.data[y * w..(y + 1) * w], row, rr); + }); + std::mem::swap(&mut cur, &mut tmp); + } + for &r in &radii { + cur = box_cols_pole(&cur, r); + } + cur +} + +// ---------------------------------------------------------------- effects + +fn layout(ctx: &EffectCtx) -> u32 { + ctx.params.e("frameLayout") +} + +fn vr_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let s = ctx.params.f("blurriness").max(0.0) * b.scale * 0.5; + if s < 0.1 { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| sphere_blur(e, s)); + b +} + +fn vr_sharpen(ctx: &EffectCtx, mut b: Buf) -> Buf { + let amt = ctx.params.f("sharpenAmount") as f32 / 100.0; + if amt == 0.0 { + return b; + } + let s = (1.5 * b.scale).max(0.5); + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let bl = sphere_blur(e, s); + let mut o = e.clone(); + o.data.par_iter_mut().zip(bl.data.par_iter()).for_each(|(p, q)| { + for c in 0..3 { + p[c] = (p[c] + (p[c] - q[c]) * amt).clamp(0.0, p[3].max(0.0) * 4.0); + } + }); + o + }); + b +} + +fn vr_denoise(ctx: &EffectCtx, mut b: Buf) -> Buf { + let level = ctx.params.f("noiseLevel").max(0.0) / 100.0; + if level <= 0.0 { + return b; + } + let r = ((1.0 + level * 4.0) * b.scale).max(1.0) as usize; + let eps = (level * level * 0.02) as f32; + let detail = (ctx.params.f("detail") / 100.0).clamp(0.0, 1.0) as f32; + b.img = per_eye(&b.img, layout(ctx), |e, _| { + // Guided filter (self-guided, per channel) on a frame padded with wrapped / pole-continued + // pixels so the seam and poles are filtered like any other place. + let pad = r * 2 + 1; + let padded = wrap_pad(e, pad); + let guide = crate::util::Plane::luma(&padded); + let ch = crate::util::split(&padded); + let f: Vec = ch.par_iter().map(|c| crate::util::guided_filter(&guide, c, r, eps.max(1e-6))).collect(); + let joined = crate::util::join(&[f[0].clone(), f[1].clone(), f[2].clone(), f[3].clone()]); + let mut o = joined.crop(pad as i64, pad as i64, e.width, e.height); + o.data.par_iter_mut().zip(e.data.par_iter()).for_each(|(p, s)| { + for c in 0..4 { + p[c] = p[c] + (s[c] - p[c]) * detail; + } + p[3] = p[3].clamp(0.0, 1.0); + for c in 0..3 { + p[c] = p[c].max(0.0); + } + }); + o + }); + b +} + +/// Pad an equirectangular image by `pad` pixels: wrapped columns, pole-continued rows. +pub fn wrap_pad(img: &Image, pad: usize) -> Image { + let (w, h) = (img.width as i64, img.height as i64); + let p = pad as i64; + let mut out = Image::new((w + 2 * p) as u32, (h + 2 * p) as u32); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let (mut sx, mut sy) = (x as i64 - p, y as i64 - p); + if sy < 0 { + sy = -1 - sy; + sx += w / 2; + } else if sy >= h { + sy = 2 * h - 1 - sy; + sx += w / 2; + } + *px = img.data[(sy.clamp(0, h - 1) * w + sx.rem_euclid(w.max(1))) as usize]; + } + }); + out +} + +fn vr_glow(ctx: &EffectCtx, mut b: Buf) -> Buf { + let th = ctx.params.f("luminanceThreshold") as f32 / 100.0; + let radius = ctx.params.f("glowRadius").max(0.0) * b.scale; + let bright = ctx.params.f("glowBrightness") as f32 / 100.0; + let sat = ctx.params.f("glowSaturation") as f32 / 100.0; + let tint = ctx.params.b("useTintColor"); + let tc = ctx.params.color("tintColor"); + if bright <= 0.0 { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let mut hi = e.clone(); + hi.data.par_iter_mut().for_each(|p| { + let (c, a) = unpremul(*p); + let l = luminance(c[0], c[1], c[2]); + let k = if l > th { ((l - th) / (1.0 - th).max(1e-3)).min(4.0) } else { 0.0 }; + let mut c = c.map(|v| v * k); + let lc = luminance(c[0], c[1], c[2]); + c = c.map(|v| lc + (v - lc) * sat); + if tint { + c = [lc * tc[0], lc * tc[1], lc * tc[2]]; + } + *p = [c[0] * a, c[1] * a, c[2] * a, a * k.min(1.0)]; + }); + let g = sphere_blur(&hi, (radius * 0.5).max(0.5)); + let mut o = e.clone(); + o.data.par_iter_mut().zip(g.data.par_iter()).for_each(|(p, q)| { + for c in 0..3 { + p[c] += q[c] * bright; + } + p[3] = (p[3] + q[3] * bright).min(1.0); + }); + o + }); + b +} + +fn rot_params(ctx: &EffectCtx) -> [[f64; 3]; 3] { + rotation(ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")) +} + +fn vr_rotate(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (t, pn, r) = (ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")); + if t == 0.0 && pn == 0.0 && r == 0.0 { + return b; + } + let m = rot_params(ctx); + let eq = Proj { kind: 0, fov: 360.0 }; + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, eq, eq, e.width, e.height, Some(&m))); + b +} + +fn vr_converter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let from = Proj { kind: ctx.params.e("sourceProjection"), fov: ctx.params.f("sourceHorizontalFov") }; + let to = Proj { kind: ctx.params.e("targetProjection"), fov: ctx.params.f("targetHorizontalFov") }; + let (t, pn, r) = (ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")); + if from.kind == to.kind && (from.kind <= 3 || from.fov == to.fov) && t == 0.0 && pn == 0.0 && r == 0.0 { + return b; + } + let m = rot_params(ctx); + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, from, to, e.width, e.height, Some(&m))); + b +} + +fn vr_sphere_to_plane(ctx: &EffectCtx, mut b: Buf) -> Buf { + let fov = ctx.params.f("horizontalFov"); + let m = rot_params(ctx); + let eq = Proj { kind: 0, fov: 360.0 }; + let flat = Proj { kind: 5, fov }; + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, eq, flat, e.width, e.height, Some(&m))); + b +} + +fn vr_plane_to_sphere(ctx: &EffectCtx, mut b: Buf) -> Buf { + // The layer is a flat picture placed on the sphere: horizontal FOV = 90° × scale. + let fov = (90.0 * ctx.params.f("scale") / 100.0).clamp(1.0, 179.0); + let m = rot_params(ctx); + let feather = (ctx.params.f("feather") / 100.0).clamp(0.0, 1.0); + let flat = Proj { kind: 5, fov }; + let eq = Proj { kind: 0, fov: 360.0 }; + let src = b.img.clone(); + let mut out = convert(&src, flat, eq, src.width, src.height, Some(&m)); + if feather > 0.0 { + let (w, h) = (src.width as f64, src.height as f64); + let t = (fov / 2.0).to_radians().tan(); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = mtv(&m, equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h)); + if d[2] <= 0.0 { + continue; + } + let u = (d[0] / d[2] / t).abs(); + let v = (d[1] / d[2] / (t * h / w)).abs(); + let edge = 1.0 - u.max(v); + let k = (edge / feather).clamp(0.0, 1.0) as f32; + for c in px.iter_mut() { + *c *= k; + } + } + }); + } + b.img = out; + b +} + +/// Angle (radians) between two directions. +#[inline] +fn angle(a: V3, b: V3) -> f64 { + dot(a, b).clamp(-1.0, 1.0).acos() +} + +fn vr_chromatic(ctx: &EffectCtx, mut b: Buf) -> Buf { + let k = [ctx.params.f("aberrationRed"), ctx.params.f("aberrationGreen"), ctx.params.f("aberrationBlue")].map(|v| v / 100.0 * 0.1); + if k.iter().all(|v| *v == 0.0) { + return b; + } + let center = mv(&rotation(ctx.params.f("centerTilt"), ctx.params.f("centerPan"), 0.0), [0.0, 0.0, 1.0]); + let falloff = (ctx.params.f("falloff") / 100.0).clamp(0.0, 1.0); + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut o = Image::new(e.width, e.height); + o.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + let th = angle(d, center); + // Axis perpendicular to centre and d: rotate d about it toward/away from centre. + let ax = [center[1] * d[2] - center[2] * d[1], center[2] * d[0] - center[0] * d[2], center[0] * d[1] - center[1] * d[0]]; + let al = dot(ax, ax).sqrt(); + let weight = (th / PI).powf(1.0 + falloff * 3.0); + let mut out = [0.0f32; 4]; + for c in 0..3 { + let s = if al < 1e-9 { + e.data[(y * e.width as usize) + x] + } else { + let new_th = th * (1.0 - k[c] * weight); + // Direction at angle new_th from the centre in the plane of (centre, d). + let perp = norm([d[0] - center[0] * th.cos(), d[1] - center[1] * th.cos(), d[2] - center[2] * th.cos()]); + let nd = [ + center[0] * new_th.cos() + perp[0] * new_th.sin(), + center[1] * new_th.cos() + perp[1] * new_th.sin(), + center[2] * new_th.cos() + perp[2] * new_th.sin(), + ]; + let (sx, sy) = equi_px(nd, w, h); + sample_equi(e, sx, sy) + }; + out[c] = s[c]; + out[3] = out[3].max(s[3]); + } + *px = out; + } + }); + o + }); + b +} + +const BLEND_OPTS: [&str; 6] = ["None", "Normal", "Add", "Multiply", "Screen", "Overlay"]; + +fn blend_mode(i: u32) -> Option { + match i { + 0 => None, + 2 => Some(BlendMode::Add), + 3 => Some(BlendMode::Multiply), + 4 => Some(BlendMode::Screen), + 5 => Some(BlendMode::Overlay), + _ => Some(BlendMode::Normal), + } +} + +/// Composite a generated eye image over the original with a blend mode and opacity (mode None +/// replaces the layer). +fn composite(orig: &Image, layer: &Image, mode: u32, opacity: f32) -> Image { + let mut o = orig.clone(); + o.data.par_iter_mut().zip(layer.data.par_iter()).for_each(|(d, s)| { + let s = [s[0] * opacity, s[1] * opacity, s[2] * opacity, s[3] * opacity]; + *d = match blend_mode(mode) { + None => s, + Some(m) => { + let mut r = blend_pixel(m, *d, s, 0.5); + // Keep the layer's own coverage: generated pixels show only where it is opaque. + let a = d[3]; + if a < 1.0 { + let k = a / r[3].max(1e-6); + if k < 1.0 { + r = r.map(|v| v * k); + } + } + r + } + }; + }); + o +} + +fn vr_gradients(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let blend = pr.f("blend").max(0.1); + let opacity = (pr.f("opacity") / 100.0) as f32; + let mode = pr.e("blendingMode"); + let (lw, lh) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + let pts: Vec<(V3, [f32; 4])> = (1..=5) + .filter(|i| pr.b(&format!("enablePoint{i}"))) + .map(|i| { + let p = pr.v2(&format!("point{i}")); + (equi_dir(p[0], p[1], lw, lh), pr.color(&format!("color{i}"))) + }) + .collect(); + if pts.is_empty() { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut g = Image::new(e.width, e.height); + g.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + // Inverse-distance weighting on great-circle distance. + let mut acc = [0.0f64; 3]; + let mut ws = 0.0; + let mut exact = None; + for (pd, c) in &pts { + let a = angle(d, *pd); + if a < 1e-6 { + exact = Some(*c); + break; + } + let wgt = 1.0 / a.powf(blend); + for k in 0..3 { + acc[k] += c[k] as f64 * wgt; + } + ws += wgt; + } + let c = exact.map(|c| [c[0], c[1], c[2]]).unwrap_or([(acc[0] / ws) as f32, (acc[1] / ws) as f32, (acc[2] / ws) as f32]); + *px = [c[0], c[1], c[2], 1.0]; + } + }); + composite(e, &g, mode, opacity) + }); + b +} + +fn vr_fractal(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let kind = pr.e("fractalType"); + let invert = pr.b("invert"); + let contrast = pr.f("contrast") as f32 / 100.0; + let brightness = pr.f("brightness") as f32 / 100.0; + let scale = (pr.f("scale") / 100.0).max(0.01) as f32; + let octaves = pr.f("complexity").clamp(1.0, 20.0); + let evo = (pr.f("evolution") / 360.0) as f32; + let opacity = (pr.f("opacity") / 100.0) as f32; + let mode = pr.e("blendingMode"); + let seed = pr.f("randomSeed") as u32; + let base = 4.0 / scale; + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut g = Image::new(e.width, e.height); + g.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + let mut v = 0.0f32; + let mut amp = 1.0f32; + let mut freq = base; + let mut norm = 0.0f32; + let n_oct = octaves.ceil() as usize; + for o in 0..n_oct { + let wgt = if o + 1 == n_oct && octaves.fract() > 0.0 { octaves.fract() as f32 } else { 1.0 }; + let q = [d[0] as f32 * freq + evo * 0.37, d[1] as f32 * freq + evo * 0.71, d[2] as f32 * freq + evo * 0.59]; + let n = value_noise(q[0] + o as f32 * 17.3, q[1], q[2], seed.wrapping_add(o as u32)); + let n = match kind { + 1 => (n * 2.0 - 1.0).abs(), + 2 => 1.0 - (n * 2.0 - 1.0).abs(), + _ => n, + }; + v += n * amp * wgt; + norm += amp * wgt; + amp *= 0.5; + freq *= 2.0; + } + let mut v = v / norm.max(1e-6); + v = (v - 0.5) * contrast + 0.5 + brightness; + if invert { + v = 1.0 - v; + } + let v = v.clamp(0.0, 1.0); + *px = [v, v, v, 1.0]; + } + }); + composite(e, &g, mode, opacity) + }); + b +} + +fn vr_glitch(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let amp = (pr.f("masterAmplitude") / 100.0).max(0.0); + if amp <= 0.0 { + return b; + } + let rate = pr.f("distortionRate").max(0.0); + let geo = pr.f("geometricDistortion") / 100.0 * amp; + let complexity = pr.f("distortionComplexity").clamp(1.0, 100.0); + let rgb = pr.f("colorDistortion") / 100.0 * amp; + let scan = (pr.f("scanlines") / 100.0 * amp) as f32; + let seed = (pr.f("randomSeed") as u32) ^ ctx.seed; + // The glitch pattern changes `rate` times a second (held between changes). + let slot = if rate > 0.0 { (ctx.time * rate).floor() as i64 } else { 0 } as u32; + b.img = per_eye(&b.img, layout(ctx), |e, eye| { + let (w, h) = (e.width as f64, e.height as f64); + let bands = complexity.round() as u32; + let mut o = Image::new(e.width, e.height); + o.rows_mut().for_each(|(y, row)| { + let band = ((y as f64 / h) * bands as f64) as u32; + let r0 = hash1(band, slot, seed ^ (eye as u32 * 977)); + let on = r0 < 0.35; + let shift = if on { (hash1(band, slot + 1, seed) as f64 * 2.0 - 1.0) * geo * w * 0.15 } else { 0.0 }; + let split = if on { rgb * w * 0.01 * (1.0 + hash1(band, slot + 2, seed) as f64) } else { 0.0 }; + let dark = if y % 2 == 0 { 1.0 - scan * 0.5 } else { 1.0 }; + for (x, px) in row.iter_mut().enumerate() { + let sx = x as f64 + 0.5 - shift; + let sy = y as f64 + 0.5; + // Horizontal shifts wrap around the seam (the frame is a full turn). + let g = sample_equi(e, sx, sy); + let r = sample_equi(e, sx - split, sy); + let bb = sample_equi(e, sx + split, sy); + *px = [r[0] * dark, g[1] * dark, bb[2] * dark, g[3].max(r[3]).max(bb[3])]; + } + }); + o + }); + b +} + +fn ang(id: &'static str, name: &'static str) -> crate::ParamSpec { + p(id, name, num(0.0), ParamUi::Angle) +} + +pub fn specs() -> Vec { + let pct = |d: f64| (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)); + let mut grad = vec![layout_param()]; + let defaults = [ + ([0.25, 0.5], [1.0, 0.0, 0.0]), + ([0.5, 0.25], [1.0, 1.0, 0.0]), + ([0.75, 0.5], [0.0, 0.0, 1.0]), + ([0.5, 0.75], [0.0, 1.0, 0.0]), + ([0.0, 0.5], [1.0, 0.0, 1.0]), + ]; + const IDS: [[&str; 6]; 5] = [ + ["enablePoint1", "Enable Point 1", "point1", "Point 1", "color1", "Color 1"], + ["enablePoint2", "Enable Point 2", "point2", "Point 2", "color2", "Color 2"], + ["enablePoint3", "Enable Point 3", "point3", "Point 3", "color3", "Color 3"], + ["enablePoint4", "Enable Point 4", "point4", "Point 4", "color4", "Color 4"], + ["enablePoint5", "Enable Point 5", "point5", "Point 5", "color5", "Color 5"], + ]; + for (i, (ids, (pt, c))) in IDS.iter().zip(defaults).enumerate() { + grad.push(p(ids[0], ids[1], Value::Bool(i < 4), ParamUi::Checkbox)); + grad.push(p(ids[2], ids[3], Value::Vec2(pt), ParamUi::Point)); + grad.push(p(ids[4], ids[5], col(c[0], c[1], c[2]), ParamUi::Color)); + } + grad.push(p("blend", "Blend", num(2.0), slider(0.1, 10.0, 0.5, 6.0, 2))); + let (v, u) = pct(100.0); + grad.push(p("opacity", "Opacity", v, u)); + grad.push(p("blendingMode", "Blending Mode", Value::Enum(0), popup(&BLEND_OPTS))); + vec![ + spec("ec.vr.blur", "VR Blur", vec![layout_param(), p("blurriness", "Blurriness", num(10.0), slider(0.0, 1000.0, 0.0, 100.0, 1))], vr_blur), + spec( + "ec.vr.chromaticaberrations", + "VR Chromatic Aberrations", + vec![ + layout_param(), + ang("centerTilt", "Center Tilt"), + ang("centerPan", "Center Pan"), + p("aberrationRed", "Aberration (Red)", num(10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("aberrationGreen", "Aberration (Green)", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("aberrationBlue", "Aberration (Blue)", num(-10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("falloff", "Falloff", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_chromatic, + ), + spec("ec.vr.colorgradients", "VR Color Gradients", grad, vr_gradients), + spec( + "ec.vr.converter", + "VR Converter", + vec![ + layout_param(), + p("sourceProjection", "Source Projection", Value::Enum(0), popup(&PROJECTIONS)), + p("sourceHorizontalFov", "Source Horizontal FOV", num(90.0), slider(1.0, 360.0, 1.0, 360.0, 1)), + p("targetProjection", "Target Projection", Value::Enum(2), popup(&PROJECTIONS)), + p("targetHorizontalFov", "Target Horizontal FOV", num(90.0), slider(1.0, 360.0, 1.0, 360.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + ], + vr_converter, + ), + spec( + "ec.vr.denoise", + "VR De-Noise", + vec![ + layout_param(), + p("noiseLevel", "Noise Level", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("detail", "Detail", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_denoise, + ), + spec( + "ec.vr.digitalglitch", + "VR Digital Glitch", + vec![ + layout_param(), + p("masterAmplitude", "Master Amplitude", num(100.0), slider(0.0, 500.0, 0.0, 200.0, 1)), + p("distortionRate", "Distortion Rate", num(5.0), slider(0.0, 60.0, 0.0, 30.0, 2)), + p("distortionComplexity", "Distortion Complexity", num(16.0), slider(1.0, 100.0, 1.0, 64.0, 0)), + p("geometricDistortion", "Geometric Distortion", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("colorDistortion", "Color Distortion", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scanlines", "Scanlines", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + ], + vr_glitch, + ), + spec( + "ec.vr.fractalnoise", + "VR Fractal Noise", + vec![ + layout_param(), + p("fractalType", "Fractal Type", Value::Enum(0), popup(&["Basic", "Turbulent Smooth", "Turbulent Sharp"])), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("contrast", "Contrast", num(100.0), slider(0.0, 10000.0, 0.0, 400.0, 1)), + p("brightness", "Brightness", num(0.0), slider(-10000.0, 10000.0, -200.0, 200.0, 1)), + p("scale", "Scale", num(100.0), slider(1.0, 10000.0, 10.0, 600.0, 1)), + p("complexity", "Complexity", num(6.0), slider(1.0, 20.0, 1.0, 20.0, 1)), + ang("evolution", "Evolution"), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + p("opacity", "Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("blendingMode", "Blending Mode", Value::Enum(0), popup(&BLEND_OPTS)), + ], + vr_fractal, + ), + spec( + "ec.vr.glow", + "VR Glow", + vec![ + layout_param(), + p("luminanceThreshold", "Luminance Threshold", num(60.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("glowRadius", "Glow Radius", num(20.0), slider(0.0, 1000.0, 0.0, 200.0, 1)), + p("glowBrightness", "Glow Brightness", num(100.0), slider(0.0, 1000.0, 0.0, 400.0, 1)), + p("glowSaturation", "Glow Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + p("useTintColor", "Use Tint Color", Value::Bool(false), ParamUi::Checkbox), + p("tintColor", "Tint Color", col(1.0, 0.8, 0.5), ParamUi::Color), + ], + vr_glow, + ), + spec( + "ec.vr.planetosphere", + "VR Plane to Sphere", + vec![ + layout_param(), + p("scale", "Scale", num(100.0), slider(1.0, 198.0, 1.0, 198.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_plane_to_sphere, + ), + spec( + "ec.vr.rotatesphere", + "VR Rotate Sphere", + vec![layout_param(), ang("tilt", "Tilt (X Axis)"), ang("pan", "Pan (Y Axis)"), ang("roll", "Roll (Z Axis)")], + vr_rotate, + ), + spec( + "ec.vr.sharpen", + "VR Sharpen", + vec![layout_param(), p("sharpenAmount", "Sharpen Amount", num(20.0), slider(0.0, 500.0, 0.0, 100.0, 1))], + vr_sharpen, + ), + spec( + "ec.vr.spheretoplane", + "VR Sphere To Plane", + vec![ + layout_param(), + p("horizontalFov", "Horizontal FOV", num(90.0), slider(1.0, 179.0, 10.0, 170.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + ], + vr_sphere_to_plane, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, run_fx}; + + /// A smooth function of the view direction rendered as an equirectangular image. + fn sphere_img(w: u32, h: u32) -> Image { + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w as f64, h as f64); + img.set(x, y, [(0.5 + 0.4 * d[0]) as f32, (0.5 + 0.4 * d[1]) as f32, (0.5 + 0.4 * d[2]) as f32, 1.0]); + } + } + img + } + + #[test] + fn projections_are_inverse() { + for kind in 0..6u32 { + let pr = Proj { kind, fov: if kind == 4 { 360.0 } else { 100.0 } }; + let (w, h) = (240.0, 120.0); + for (x, y) in [(10.5, 20.5), (100.5, 60.5), (230.5, 110.5), (130.5, 33.5)] { + let Some(d) = pr.dir(x, y, w, h) else { continue }; + let (px, py, _) = pr.pos(d, w, h).unwrap(); + assert!((px - x).abs() < 1e-6 && (py - y).abs() < 1e-6, "kind {kind}: ({x},{y}) → ({px},{py})"); + } + } + } + + #[test] + fn converter_round_trips_equirect_through_cube_maps() { + let img = sphere_img(128, 64); + for target in [1u32, 2, 3] { + let cube = run_fx("ec.vr.converter", &[("targetProjection", Value::Enum(target))], img.clone(), 0.0, EffectEnv::default()); + let back = run_fx( + "ec.vr.converter", + &[("sourceProjection", Value::Enum(target)), ("targetProjection", Value::Enum(0))], + cube.img.clone(), + 0.0, + EffectEnv::default(), + ); + let mut worst = 0.0f32; + let mut sum = 0.0f32; + for (a, b) in back.img.data.iter().zip(&img.data) { + let e = (0..3).map(|k| (a[k] - b[k]).abs()).fold(0.0, f32::max); + worst = worst.max(e); + sum += e; + } + let mean = sum / img.data.len() as f32; + // The 6:1 strip squeezes each face into a narrow cell, so its seams are coarser. + let tol = if target == 3 { 0.25 } else { 0.08 }; + assert!(mean < 0.01 && worst < tol, "target {target}: mean {mean} worst {worst}"); + // Cube faces carry the expected directions: the front face centre looks at +Z. + let pr = Proj { kind: target, fov: 90.0 }; + let (x, y, _) = pr.pos([0.0, 0.0, 1.0], 128.0, 64.0).unwrap(); + let px = cube.img.sample_bilinear_clamped(x, y); + assert!((px[2] - 0.9).abs() < 0.02, "{px:?}"); + } + } + + #[test] + fn rotate_sphere_pans_and_round_trips() { + let img = sphere_img(96, 48); + let a = run_fx("ec.vr.rotatesphere", &[("pan", num(90.0))], img.clone(), 0.0, EffectEnv::default()); + let b = run_fx("ec.vr.rotatesphere", &[("pan", num(-90.0))], a.img.clone(), 0.0, EffectEnv::default()); + let err: f32 = b.img.data.iter().zip(&img.data).map(|(p, q)| (p[0] - q[0]).abs()).sum::() / img.data.len() as f32; + assert!(err < 0.01, "{err}"); + // A 90° pan is a horizontal shift by a quarter turn. + let (x, y) = (10u32, 24u32); + let shifted = img.get(((x + 24) % 96) as i64, y as i64); + let got = a.img.get(x as i64, y as i64); + let alt = img.get(((x + 72) % 96) as i64, y as i64); + assert!((got[0] - shifted[0]).abs() < 0.02 || (got[0] - alt[0]).abs() < 0.02); + } + + #[test] + fn blur_is_seamless() { + // A frame whose left and right edges differ: after blurring, the two edge columns agree + // (they are neighbours on the sphere). + let mut img = Image::new(64, 32); + for y in 0..32 { + for x in 0..64 { + let v = if x < 4 { + 1.0 + } else if x >= 60 { + 0.0 + } else { + 0.5 + }; + img.set(x, y, [v, v, v, 1.0]); + } + } + let out = run_fx("ec.vr.blur", &[("blurriness", num(8.0))], img.clone(), 0.0, EffectEnv::default()); + let (l, r) = (out.img.get(0, 16)[0], out.img.get(63, 16)[0]); + assert!((l - r).abs() < 0.2, "{l} {r}"); + assert!(r > 0.2 && l < 0.85); + // Pole rows are uniform (every pixel of the top row is the same point). + let top: Vec = (0..64).map(|x| out.img.get(x, 0)[0]).collect(); + let spread = top.iter().cloned().fold(f32::MIN, f32::max) - top.iter().cloned().fold(f32::MAX, f32::min); + assert!(spread < 0.1, "{spread}"); + // Constant images stay constant. + let flat = Image::filled(32, 16, [0.3, 0.3, 0.3, 1.0]); + let o = run_fx("ec.vr.blur", &[], flat, 0.0, EffectEnv::default()); + assert!(o.img.data.iter().all(|p| (p[0] - 0.3).abs() < 1e-4)); + } + + #[test] + fn stereo_layouts_process_each_eye() { + let mut img = Image::new(32, 32); + for y in 0..32 { + for x in 0..32 { + let v = if y < 16 { 0.2 } else { 0.8 }; + img.set(x, y, [v, v, v, 1.0]); + } + } + // Over/under: blurring each eye keeps the eyes separate. + let o = run_fx("ec.vr.blur", &[("frameLayout", Value::Enum(1)), ("blurriness", num(10.0))], img.clone(), 0.0, EffectEnv::default()); + assert!((o.img.get(5, 15)[0] - 0.2).abs() < 1e-4 && (o.img.get(5, 16)[0] - 0.8).abs() < 1e-4); + let mono = run_fx("ec.vr.blur", &[("blurriness", num(10.0))], img, 0.0, EffectEnv::default()); + assert!((mono.img.get(5, 15)[0] - 0.2).abs() > 0.01); + } + + #[test] + fn sphere_to_plane_and_back() { + let img = sphere_img(128, 64); + // The centre of a 2D view looking forward sees +Z (blue 0.9). + let v = run_fx("ec.vr.spheretoplane", &[], img.clone(), 0.0, EffectEnv::default()); + let c = v.img.get(64, 32); + assert!((c[2] - 0.9).abs() < 0.02, "{c:?}"); + // Looking up (tilt 90°): +Y (green 0.9). + let up = run_fx("ec.vr.spheretoplane", &[("tilt", num(-90.0))], img.clone(), 0.0, EffectEnv::default()); + let u = up.img.get(64, 32); + let dn = run_fx("ec.vr.spheretoplane", &[("tilt", num(90.0))], img, 0.0, EffectEnv::default()); + let d = dn.img.get(64, 32); + assert!((u[1] - 0.9).abs() < 0.03 || (d[1] - 0.9).abs() < 0.03, "{u:?} {d:?}"); + // Plane to sphere puts the picture in front, transparent behind. + let pic = Image::filled(64, 32, [1.0, 0.0, 0.0, 1.0]); + let s = run_fx("ec.vr.planetosphere", &[], pic, 0.0, EffectEnv::default()); + assert!(s.img.get(32, 16)[3] > 0.99 && s.img.get(1, 16)[3] < 0.01); + } + + #[test] + fn generators_and_filters_are_deterministic_and_seamless() { + let img = sphere_img(64, 32); + for (id, vals) in [ + ("ec.vr.fractalnoise", vec![]), + ("ec.vr.colorgradients", vec![]), + ("ec.vr.glow", vec![("luminanceThreshold", num(50.0))]), + ("ec.vr.sharpen", vec![]), + ("ec.vr.denoise", vec![]), + ("ec.vr.chromaticaberrations", vec![]), + ("ec.vr.digitalglitch", vec![]), + ] { + let a = run_fx(id, &vals, img.clone(), 0.5, EffectEnv::default()); + let b = run_fx(id, &vals, img.clone(), 0.5, EffectEnv::default()); + assert_eq!(a.img.data, b.img.data, "{id}"); + assert!(a.img.data.iter().all(|p| p.iter().all(|v| v.is_finite())), "{id}"); + } + // Fractal noise is continuous across the seam: the first and last columns are close. + let n = run_fx("ec.vr.fractalnoise", &[], img.clone(), 0.0, EffectEnv::default()); + let mut worst = 0.0f32; + for y in 0..32 { + worst = worst.max((n.img.get(0, y)[0] - n.img.get(63, y)[0]).abs()); + } + assert!(worst < 0.15, "{worst}"); + // Gradients reach the point colour at the point. + let pts = [ + ("point1", Value::Vec2([16.0, 16.0])), + ("point2", Value::Vec2([32.0, 8.0])), + ("point3", Value::Vec2([48.0, 16.0])), + ("point4", Value::Vec2([32.0, 24.0])), + ]; + let g = run_fx("ec.vr.colorgradients", &pts, img.clone(), 0.0, EffectEnv::default()); + let px = g.img.get(16, 16); + assert!(px[0] > 0.8 && px[1] < 0.3, "{px:?}"); + // Glitch changes over time. + let g0 = run_fx("ec.vr.digitalglitch", &[], img.clone(), 0.0, EffectEnv::default()); + let g1 = run_fx("ec.vr.digitalglitch", &[], img.clone(), 3.3, EffectEnv::default()); + assert_ne!(g0.img.data, g1.img.data); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 132bdcac..a60242bb 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,7 @@ effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } +effectcraft-model = { workspace = true } effectcraft-lottie = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/engine/src/commands/comp.rs b/crates/engine/src/commands/comp.rs index 5ebd491e..a33fd574 100644 --- a/crates/engine/src/commands/comp.rs +++ b/crates/engine/src/commands/comp.rs @@ -125,6 +125,7 @@ fn settings(s: &mut Session, p: &Value) -> Result { } let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; apply_settings(c, p); + super::model3d::sync_geometry_options(proj, cid); Ok(()) })?; let c = s.project.comp(cid).ok_or(EngineError::NoComp)?; diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index a85a391f..11e8a387 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -110,7 +110,7 @@ fn import(s: &mut Session, p: &Value) -> Result { let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, - FootageKind::Video => Label::Aqua, + FootageKind::Video | FootageKind::Model => Label::Aqua, }; let id = proj.add_item(&name, label, None, ItemKind::Footage(f)); ids.push(id.0); diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index 3dea6880..ac728200 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -24,7 +24,11 @@ pub(crate) fn color_p(p: &Value, k: &str) -> Option<[f32; 3]> { } /// Insert a new layer above the selection (or at the top) and select it. -pub(crate) fn insert_layer(proj: &mut Project, st: &mut crate::EditorState, cid: ItemId, layer: Layer) -> Result { +pub(crate) fn insert_layer(proj: &mut Project, st: &mut crate::EditorState, cid: ItemId, mut layer: Layer) -> Result { + // Text and shape layers in Advanced 3D comps get Geometry Options (extrusion, bevels). + if proj.comp(cid).is_some_and(|c| c.renderer == effectcraft_project::Renderer::Advanced3D) { + super::model3d::add_geometry_options(&mut proj.next_id, &mut layer); + } let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; let at = st.selected_layers.first().and_then(|id| comp.layers.iter().position(|l| l.id == *id)).unwrap_or(0); let id = layer.id; @@ -235,6 +239,9 @@ fn add_item(s: &mut Session, p: &Value) -> Result { let d = (s.prefs.import.still_footage == "seconds").then(|| Tick::from_seconds_f64(s.prefs.import.still_seconds)); (LayerSource::Footage { item }, (f.width, f.height), d) } + ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Model => { + return super::model3d::new_model(s, &serde_json::json!({"comp": cid.0, "item": item.0, "time": f_p(p, "time")})); + } ItemKind::Footage(f) => (LayerSource::Footage { item }, (f.width, f.height), Some(f.duration)), ItemKind::Solid(so) => (LayerSource::Solid { item }, (so.width, so.height), None), ItemKind::Folder => return Err(bad("layer.addItem", "folders can't be layers")), diff --git a/crates/engine/src/commands/liquify.rs b/crates/engine/src/commands/liquify.rs new file mode 100644 index 00000000..748c664c --- /dev/null +++ b/crates/engine/src/commands/liquify.rs @@ -0,0 +1,156 @@ +//! Liquify brush strokes (`liquify.stroke`, `liquify.clear`). +//! +//! A stroke is appended to the layer's Liquify effect (added when the layer has none) as one +//! line of its hidden Distortion Mesh data (see `effectcraft_effects::distort4`). Brush size, +//! pressure, turbulent jitter and clone offset default to the effect's own tool options. When +//! the Distortion Mesh is animated, the stroke goes into a keyframe at the current time (copying +//! the mesh in effect there), as After Effects records mesh keyframes. + +use effectcraft_effects::distort4::{LIQUIFY_TOOLS, LiquifyStroke, parse_strokes}; +use effectcraft_keyframe::{Keyframe, Value as KV}; +use effectcraft_project::build::Ids; +use effectcraft_project::{GroupKind, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const ID: &str = "ec.distort.liquify"; + +fn is_liquify(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ID) +} + +fn liquify_effect(fx: &mut PropGroup, ids: &mut Ids, layer_size: [f64; 2], want: Option) -> Result { + if let Some(u) = want { + return fx.groups().find(|g| g.uid == u && is_liquify(g)).map(|g| g.uid).ok_or_else(|| bad("liquify.stroke", "no such Liquify effect")); + } + if let Some(g) = fx.groups().filter(|g| is_liquify(g)).last() { + return Ok(g.uid); + } + let spec = effectcraft_effects::find(ID).ok_or_else(|| EngineError::Other("Liquify effect missing".into()))?; + let g = effectcraft_effects::instantiate(spec, ids, "Liquify", layer_size); + let uid = g.uid; + fx.children.push(g.into()); + Ok(uid) +} + +fn num_of(g: &PropGroup, id: &str, default: f64) -> f64 { + g.get(id).map(|p| p.value.as_f64()).unwrap_or(default) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.stroke"; + let (cid, lid) = layer_p(s, p, cmd)?; + let tool_name = str_p(p, "tool").unwrap_or("warp"); + let tool = LIQUIFY_TOOLS + .iter() + .position(|t| t.eq_ignore_ascii_case(tool_name)) + .ok_or_else(|| bad(cmd, format!("unknown tool `{tool_name}` ({})", LIQUIFY_TOOLS.join(" | "))))?; + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer space)"))?; + let mut points = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + points.push([x, y]); + } + if points.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = comp.frame_rate.snap_nearest(super::time_p(s, p, Some(comp))); + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if layer.effects().is_none() || !layer.source.is_av() { + return Err(bad(cmd, "Liquify works on footage, solid, text, shape and precomp layers")); + } + let lt = layer.layer_time(t); + let (w, h) = effectcraft_render::source_size(&s.project, layer); + let size = if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] }; + let want = p.get("effect").and_then(Value::as_u64); + let (fx_uid, count) = s.edit("Liquify Stroke", None, |proj, _st| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let mut ids = Ids(&mut next); + let fx_uid = liquify_effect(fx, &mut ids, size, want)?; + proj.next_id = next; + let l = layer_mut(proj, cid, lid)?; + let g = l.props.sub_mut("effects").and_then(|fx| fx.find_group_mut(fx_uid)).ok_or_else(|| bad(cmd, "Liquify effect gone"))?; + let co = g.get("cloneOffset").map(|p| p.value.as_vec2()).unwrap_or([0.0; 2]); + let st = LiquifyStroke { + tool, + size: f_p(p, "size").unwrap_or_else(|| num_of(g, "brushSize", 64.0)).max(1.0), + pressure: f_p(p, "pressure").unwrap_or_else(|| num_of(g, "brushPressure", 50.0)).clamp(1.0, 100.0), + jitter: f_p(p, "jitter").unwrap_or_else(|| num_of(g, "turbulentJitter", 70.0)).clamp(1.0, 100.0), + clone_offset: match p.get("cloneOffset").and_then(Value::as_array) { + Some(a) => [a.first().and_then(Value::as_f64).unwrap_or(0.0), a.get(1).and_then(Value::as_f64).unwrap_or(0.0)], + None => co, + }, + points: points.clone(), + }; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "Liquify effect has no Distortion Mesh"))?; + let append = |cur: &str| if cur.trim().is_empty() { st.to_line() } else { format!("{}\n{}", cur.trim_end(), st.to_line()) }; + let text = if prop.keys.is_empty() { + let cur = match &prop.value { + KV::Str(x) => x.clone(), + _ => String::new(), + }; + let v = append(&cur); + prop.value = KV::Str(v.clone()); + v + } else { + // Hold keyframes: the mesh in effect at `lt` is the last key at or before it. + let cur = prop.keys.iter().rfind(|k| k.time <= lt).or(prop.keys.first()).map(|k| k.value.clone()); + let cur = match cur { + Some(KV::Str(x)) => x, + _ => String::new(), + }; + let v = append(&cur); + match prop.keys.iter_mut().find(|k| k.time == lt) { + Some(k) => k.value = KV::Str(v.clone()), + None => { + prop.keys.push(Keyframe::new(lt, KV::Str(v.clone()))); + prop.keys.sort_by_key(|k| k.time); + } + } + v + }; + Ok((fx_uid, parse_strokes(&text).len())) + })?; + Ok(json!({"effect": fx_uid, "strokes": count})) +} + +fn clear(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.clear"; + let (cid, lid) = layer_p(s, p, cmd)?; + let want = p.get("effect").and_then(Value::as_u64); + s.edit("Clear Liquify Mesh", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "no effects"))?; + let uid = + fx.groups().filter(|g| is_liquify(g) && want.is_none_or(|u| u == g.uid)).last().map(|g| g.uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let g = fx.find_group_mut(uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "no Distortion Mesh"))?; + prop.keys.clear(); + prop.value = KV::Str(String::new()); + Ok(()) + })?; + Ok(Value::Null) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "liquify.stroke", + "Liquify Stroke", + [], + None, + "{layer?, effect?: uid, tool?: warp|turbulence|twirlClockwise|twirlCounterclockwise|pucker|bloat|shiftPixels|reflection|clone|reconstruction|freeze|thaw, points: [[x,y],…] (layer space), size?, pressure? (1-100), jitter? (1-100), cloneOffset?: [dx,dy], time? (s) | frame?}", + has_comp, + stroke + ), + cmd!("liquify.clear", "Clear Liquify Mesh", [], None, "{layer?, effect?: uid}", has_comp, clear), + ] +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 2e40eb63..18e155f2 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -17,10 +17,12 @@ mod layer; mod layer_menu; mod layer_time; mod link; +mod liquify; mod lottie; mod markers; mod mask; pub mod mask_interp; +pub(crate) mod model3d; pub mod paint; mod paths; mod project_items; @@ -102,6 +104,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(layer::specs()); v.extend(styles::specs()); v.extend(three_d::specs()); + v.extend(model3d::specs()); v.extend(text_anim::specs()); v.extend(text_edit::specs()); v.extend(layer_time::specs()); @@ -133,6 +136,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(warp_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); + v.extend(liquify::specs()); v.extend(settings::specs()); v.extend(stubs::specs()); v diff --git a/crates/engine/src/commands/model3d.rs b/crates/engine/src/commands/model3d.rs new file mode 100644 index 00000000..5de5a2c2 --- /dev/null +++ b/crates/engine/src/commands/model3d.rs @@ -0,0 +1,397 @@ +//! Advanced 3D: model layers (File ▸ Import of glTF/OBJ, Layer ▸ New ▸ 3D primitives), the +//! comp renderer switch, Layer ▸ Environment Layer, Geometry Options for extruded text and +//! shapes, and Layer ▸ Material commands. + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{FootageKind, ItemId, ItemKind, Layer, LayerSource, Node, PrimitiveKind, PropGroup, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::layer::{color_p, insert_layer}; +use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, has_layers, layer_mut, layer_p, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +// ---------------------------------------------------------------- geometry options + +/// Add Geometry Options (Bevel Style, Bevel Depth, Hole Bevel Depth, Extrusion Depth) to a text +/// or shape layer that has none. +pub(crate) fn add_geometry_options(next_id: &mut u64, l: &mut Layer) { + if !matches!(l.source, LayerSource::Text | LayerSource::Shape) || l.props.sub("geometryOptions").is_some() { + return; + } + let g = build::extrusion_geometry_options(&mut Ids(next_id)); + // After Transform, like After Effects. + let at = l.props.children.iter().position(|c| c.match_id() == "transform").map_or(l.props.children.len(), |i| i + 1); + l.props.children.insert(at, g.into()); +} + +/// Advanced 3D comps: give every text and shape layer Geometry Options. +pub(crate) fn sync_geometry_options(proj: &mut effectcraft_project::Project, cid: ItemId) { + if proj.comp(cid).is_none_or(|c| c.renderer != Renderer::Advanced3D) { + return; + } + let mut next = proj.next_id; + if let Some(c) = proj.comp_mut(cid) { + for l in &mut c.layers { + add_geometry_options(&mut next, l); + } + } + proj.next_id = next; +} + +fn renderer_p(p: &Value, cmd: &str) -> Result { + let v = str_p(p, "renderer").or_else(|| str_p(p, "value")).ok_or_else(|| bad(cmd, "missing `renderer` (advanced3d|classic3d)"))?; + match v.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "advanced3d" | "advanced" => Ok(Renderer::Advanced3D), + "classic3d" | "classic" => Ok(Renderer::Classic3D), + _ => Err(bad(cmd, format!("renderer: advanced3d|classic3d, not `{v}`"))), + } +} + +fn comp_renderer(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + if p.get("renderer").is_none() && p.get("value").is_none() { + let r = s.project.comp(cid).ok_or(EngineError::NoComp)?.renderer; + return Ok(json!({"renderer": renderer_id(r)})); + } + let r = renderer_p(p, "comp.renderer")?; + s.edit("Change Renderer", None, |proj, _| { + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.renderer = r; + sync_geometry_options(proj, cid); + Ok(()) + })?; + Ok(json!({"renderer": renderer_id(r)})) +} + +fn renderer_id(r: Renderer) -> &'static str { + match r { + Renderer::Advanced3D => "advanced3d", + Renderer::Classic3D => "classic3d", + } +} + +// ---------------------------------------------------------------- model layers + +/// Set a property's value at comp time `t` (adds a key when it is animated). +fn set_at(l: &mut Layer, path: &str, t: Tick, v: KV) -> bool { + let lt = l.layer_time(t); + match l.props.prop_mut(path) { + Some(pr) => { + pr.set_value_at(lt, v); + true + } + None => false, + } +} + +/// New 3D model layer from a model footage item (`item`) or a file (`path`, imported first). +pub(crate) fn new_model(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let item = match (p.get("item"), str_p(p, "path")) { + (Some(Value::Number(n)), _) => ItemId(n.as_u64().unwrap_or(0)), + (Some(Value::String(name)), _) => s.project.find_by_name(name).map(|i| i.id).ok_or_else(|| bad("layer.newModel", format!("no item `{name}`")))?, + (_, Some(path)) => { + let r = s.execute("file.import", json!({"paths": [path]}))?; + let id = r.get("items").and_then(|a| a.get(0)).and_then(Value::as_u64); + let err = r.get("errors").and_then(|e| e.get(0)).and_then(Value::as_str).unwrap_or("import failed").to_string(); + ItemId(id.ok_or_else(|| bad("layer.newModel", err))?) + } + _ => *s.state.project_selection.first().ok_or_else(|| bad("layer.newModel", "missing `item` or `path`"))?, + }; + let it = s.project.item(item).ok_or_else(|| bad("layer.newModel", "no such item"))?.clone(); + let ItemKind::Footage(f) = &it.kind else { return Err(bad("layer.newModel", format!("`{}` is not a 3D model", it.name))) }; + if f.kind != FootageKind::Model { + return Err(bad("layer.newModel", format!("`{}` is not a 3D model", it.name))); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + // Fit the model to half the comp height, and list its animation clips. + let model = s.footage.model(item, f); + let (scale, clips) = match &model { + Some(m) => { + let ext = m.bounds().map_or(1.0, |(lo, hi)| (0..3).map(|k| hi[k] - lo[k]).fold(0.0f64, f64::max)); + let scale = if ext > 1e-9 { comp.height as f64 * 0.5 / ext } else { 100.0 }; + let clips = m.animations.iter().enumerate().map(|(i, a)| if a.name.is_empty() { format!("Animation {}", i + 1) } else { a.name.clone() }).collect(); + (scale, clips) + } + None => (100.0, vec![]), + }; + let name = str_p(p, "name").unwrap_or(&it.name).to_string(); + let fr = comp.frame_rate; + let start = fr.snap_nearest(f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(Tick::ZERO)); + let id = s.edit("New 3D Model Layer", None, |proj, st| { + let mut l = build::layer(proj, &comp, &name, LayerSource::Model { item }, (comp.width, comp.height), None); + let g = build::model_geometry_options(&mut Ids(&mut proj.next_id), scale, &clips); + if let Some(slot) = l.props.children.iter_mut().find(|c| c.match_id() == "geometryOptions") { + *slot = g.into(); + } + l.start_time = start; + l.in_point = start; + l.out_point = comp.duration.max(start + fr.frame_duration()); + insert_layer(proj, st, cid, l) + })?; + Ok(json!({"layer": id.0, "item": item.0})) +} + +fn kind_p(p: &Value, cmd: &str) -> Result { + let k = str_p(p, "kind").or_else(|| str_p(p, "type")).ok_or_else(|| bad(cmd, "missing `kind` (cube|sphere|plane|torus|cone|cylinder)"))?; + PrimitiveKind::parse(k).ok_or_else(|| bad(cmd, format!("kind: cube|sphere|plane|torus|cone|cylinder, not `{k}`"))) +} + +/// Parse Material Options parameters onto a layer at comp time `t`. +fn apply_material(l: &mut Layer, p: &Value, t: Tick, cmd: &str) -> Result { + let mut n = 0; + let tri = |v: &Value| -> Option { + match v { + Value::Bool(b) => Some(*b as u32), + Value::Number(x) => x.as_u64().map(|x| x.min(2) as u32), + Value::String(s) => match s.to_ascii_lowercase().as_str() { + "off" | "false" => Some(0), + "on" | "true" => Some(1), + "only" => Some(2), + _ => None, + }, + _ => None, + } + }; + for k in ["castsShadows", "acceptsShadows"] { + if let Some(v) = p.get(k) { + let e = tri(v).ok_or_else(|| bad(cmd, format!("{k}: off|on|only")))?; + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Enum(e)) as usize; + } + } + for k in ["acceptsLights", "appearsInReflections"] { + if let Some(v) = b_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Bool(v)) as usize; + } + } + for k in ["lightTransmission", "ambient", "diffuse", "specularIntensity", "specularShininess", "metal", "metallic", "roughness"] { + if let Some(v) = f_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Scalar(v)) as usize; + } + } + for k in ["baseColor", "emissive"] { + if let Some(c) = color_p(p, k) { + n += set_at(l, &format!("materialOptions/{k}"), t, KV::Color([c[0] as f64, c[1] as f64, c[2] as f64, 1.0])) as usize; + } + } + Ok(n) +} + +fn new_primitive(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let kind = kind_p(p, "layer.newPrimitive")?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| kind.label().to_string()); + let t = super::time_p(s, p, Some(&comp)); + let id = s.edit(&format!("New {}", kind.label()), None, |proj, st| { + let mut l = build::layer(proj, &comp, &name, LayerSource::Primitive { kind }, (comp.width, comp.height), None); + for k in ["width", "height", "depth", "radius", "tubeRadius", "segments", "rings"] { + if let Some(v) = f_p(p, k) { + set_at(&mut l, &format!("geometryOptions/{k}"), t, KV::Scalar(v.max(0.0))); + } + } + if let Some(pos) = p.get("position").and_then(Value::as_array) { + let g = |i: usize| pos.get(i).and_then(Value::as_f64).unwrap_or(0.0); + set_at(&mut l, "transform/position", t, KV::Vec3([g(0), g(1), g(2)])); + } + apply_material(&mut l, p, t, "layer.newPrimitive")?; + insert_layer(proj, st, cid, l) + })?; + // Primitives are drawn by the Advanced 3D renderer. + let advanced = s.project.comp(cid).is_some_and(|c| c.renderer == Renderer::Advanced3D); + Ok(json!({"layer": id.0, "kind": kind.label(), "advanced3d": advanced})) +} + +// ---------------------------------------------------------------- environment layer + +fn environment(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("layer.environment", "select a footage or composition layer")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + for id in &ids { + let l = comp.layer(*id).ok_or(EngineError::NoComp)?; + if !l.source.is_av() || l.source.is_model() { + return Err(bad("layer.environment", format!("`{}` can't be an environment layer (use a footage, comp or solid layer)", l.name))); + } + } + let on = b_p(p, "on").unwrap_or_else(|| !comp.layer(ids[0]).is_some_and(|l| l.environment)); + s.edit("Environment Layer", None, |proj, _| { + for id in &ids { + let l = layer_mut(proj, cid, *id)?; + l.environment = on; + if on { + l.switches.three_d = true; + } + } + Ok(()) + })?; + Ok(json!({"environment": on, "layers": ids.iter().map(|l| l.0).collect::>()})) +} + +// ---------------------------------------------------------------- materials + +fn material_set(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("material.set", "no `layers` given and no layer selected")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let t = super::time_p(s, p, Some(&comp)); + let n = s.edit("Material Options", None, |proj, _| { + let mut n = 0; + for id in &ids { + n += apply_material(layer_mut(proj, cid, *id)?, p, t, "material.set")?; + } + Ok(n) + })?; + if n == 0 { + return Err(bad( + "material.set", + "no Material Options property matched (castsShadows, acceptsShadows, acceptsLights, baseColor, metallic, roughness, …)", + )); + } + Ok(json!({"set": n})) +} + +/// Fresh Material Options for a layer. +fn default_material(next: &mut u64, l: &Layer) -> Option { + let ids = &mut Ids(next); + match l.source { + LayerSource::Primitive { .. } => Some(build::model_material_options(ids, true)), + LayerSource::Model { .. } => Some(build::model_material_options(ids, false)), + _ if l.source.is_av() => Some(build::material_options(ids)), + _ => None, + } +} + +fn replace_material(l: &mut Layer, g: PropGroup) { + match l.props.children.iter_mut().find(|c| c.match_id() == "materialOptions") { + Some(slot) => *slot = Node::Group(g), + None => l.props.children.push(g.into()), + } +} + +fn material_reset(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.is_empty() { + return Err(bad("material.reset", "no `layers` given and no layer selected")); + } + s.edit("Reset Material", None, |proj, _| { + let mut next = proj.next_id; + for id in &ids { + let l = layer_mut(proj, cid, *id)?; + if let Some(g) = default_material(&mut next, l) { + replace_material(l, g); + } + } + proj.next_id = next; + Ok(()) + })?; + Ok(json!({"layers": ids.iter().map(|l| l.0).collect::>()})) +} + +/// Duplicate and Assign Material: the first layer's Material Options are copied onto the other +/// selected layers of the same kind. +fn material_duplicate(s: &mut Session, p: &Value) -> Result { + let (cid, ids) = layers_p(s, p)?; + if ids.len() < 2 { + return Err(bad("material.duplicateAssign", "select the source layer first, then the layers to assign its material to")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let src = comp + .layer(ids[0]) + .and_then(|l| l.props.sub("materialOptions")) + .cloned() + .ok_or_else(|| bad("material.duplicateAssign", "the first layer has no material"))?; + let n = s.edit("Duplicate and Assign Material", None, |proj, _| { + let mut n = 0; + for id in &ids[1..] { + let l = layer_mut(proj, cid, *id)?; + let Some(dst) = l.props.sub("materialOptions") else { continue }; + // Property sets differ (models, primitives, cards): copy the values they share. + let mut g = dst.clone(); + for c in g.children.iter_mut() { + if let Node::Prop(d) = c + && let Some(sv) = src.get(&d.match_id) + { + d.value = sv.value.clone(); + d.keys = sv.keys.clone(); + d.expr = sv.expr.clone(); + } + } + replace_material(l, g); + n += 1; + } + Ok(n) + })?; + Ok(json!({"assigned": n})) +} + +fn material_reveal(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "material.revealSource")?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let LayerSource::Model { item } = l.source else { return Err(bad("material.revealSource", "the layer's material is its own (not from a model file)")) }; + s.state.project_selection = vec![item]; + Ok(json!({"item": item.0})) +} + +fn has_model_or_layers(s: &Session) -> std::result::Result<(), String> { + has_layers(s) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "layer.newModel", + "3D Model Layer", + [], + None, + "{item?: id or name of a 3D model footage item, path?: a .gltf, .glb or .obj file to import, name?, time? (s)}", + has_comp, + new_model + ), + cmd!( + "layer.new3dPrimitive", + "3D Primitive", + [], + None, + "{kind: cube|sphere|plane|torus|cone|cylinder, name?, width?, height?, depth?, radius?, tubeRadius?, segments?, rings?, position? [x,y,z], baseColor?, metallic?, roughness?, emissive?, castsShadows?, acceptsShadows?, acceptsLights?}", + has_comp, + new_primitive + ), + cmd!( + "layer.newPrimitive", + "New 3D Primitive", + [], + None, + "{kind: cube|sphere|plane|torus|cone|cylinder, name?, width?, height?, depth?, radius?, tubeRadius?, segments?, rings?, position? [x,y,z], baseColor?, metallic?, roughness?, emissive?, castsShadows?, acceptsShadows?, acceptsLights?}", + has_comp, + new_primitive + ), + cmd!("comp.renderer", "Renderer", [], None, "{renderer?: advanced3d|classic3d (omit to read), value?}", has_comp, comp_renderer), + cmd!("layer.environment", "Environment Layer", ["Layer"], None, "{layers?, on?}", has_layers, environment), + cmd!( + "material.set", + "Material Options", + [], + None, + "{layers?, castsShadows?: off|on|only, acceptsShadows?: off|on|only, acceptsLights?, appearsInReflections?, lightTransmission?, ambient?, diffuse?, specularIntensity?, specularShininess?, metal?, baseColor?, metallic?, roughness?, emissive?, time?}", + has_layers, + material_set + ), + cmd!("material.revealSource", "Reveal Material Source in Project", ["Layer", "Material"], None, "{layer?}", has_layers, material_reveal), + cmd!("material.reset", "Reset Material", ["Layer", "Material"], None, "{layers?}", has_model_or_layers, material_reset), + cmd!( + "material.duplicateAssign", + "Duplicate and Assign Material", + ["Layer", "Material"], + None, + "{layers? (first = source)}", + has_layers, + material_duplicate + ), + ] +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..8b244bf6 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -29,11 +29,9 @@ pub fn specs() -> Vec { stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), - stub!("layer.new3dPrimitive", "3D Primitive", [], "{kind: cube|sphere|plane|torus|cone|cylinder}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), - stub!("layer.environment", "Environment Layer", ["Layer"], "{}"), stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), @@ -47,9 +45,6 @@ pub fn specs() -> Vec { stub!("light.fromModel", "Create Lights from 3D Model", ["Layer", "Light"], "{}"), stub!("light.controlWithCamera", "Control Light with Camera", ["Layer", "Light"], "{}"), stub!("light.environmentBackground", "Create Environment Light Background Layer", ["Layer", "Light"], "{}"), - stub!("material.revealSource", "Reveal Material Source in Project", ["Layer", "Material"], "{}"), - stub!("material.reset", "Reset Material", ["Layer", "Material"], "{}"), - stub!("material.duplicateAssign", "Duplicate and Assign Material", ["Layer", "Material"], "{}"), stub!("view.3d.default", "Default", ["View", "Switch 3D View"], "{}"), stub!("view.splitLockedViewer", "Split with New Locked Viewer", ["View"], "{}"), // Color management. diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 063b9bdf..39524c37 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -604,6 +604,8 @@ mod tests_mask_warp; #[cfg(test)] mod tests_menu_cmds; #[cfg(test)] +mod tests_model3d; +#[cfg(test)] mod tests_project_items; #[cfg(test)] mod tests_settings; diff --git a/crates/engine/src/tests_model3d.rs b/crates/engine/src/tests_model3d.rs new file mode 100644 index 00000000..51f9173d --- /dev/null +++ b/crates/engine/src/tests_model3d.rs @@ -0,0 +1,194 @@ +//! Advanced 3D commands: model import → model layers, primitives, the renderer switch with +//! Geometry Options, Environment Layer and Material commands (undo/redo, serde). + +use std::sync::Arc; + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{Footage, FootageKind, ItemId, LayerId, LayerSource, PrimitiveKind, Project, Renderer}; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::{Importer, Session}; + +fn fixture(name: &str) -> String { + format!("{}/../model/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) +} + +/// Loads models from disk (what `effectcraft-media` does in the app). +struct DiskModels; + +impl FootageSource for DiskModels { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn model(&self, _: ItemId, f: &Footage) -> Option> { + let dir = std::path::Path::new(&f.path).parent()?.to_path_buf(); + let bytes = std::fs::read(&f.path).ok()?; + effectcraft_model::load(&f.path, &bytes, &|u| std::fs::read(dir.join(u)).ok()).ok().map(Arc::new) + } +} + +struct ModelImporter; + +impl Importer for ModelImporter { + fn probe(&self, path: &str) -> Result { + effectcraft_model::format_of(path).ok_or("not a model")?; + Ok(Footage { + path: path.into(), + kind: FootageKind::Model, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + native_rate: None, + duration: Tick::ZERO, + has_video: false, + has_audio: false, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: "glTF".into(), + missing: false, + sequence: vec![], + color_profile: None, + }) + } +} + +fn session() -> Session { + let mut s = Session { footage: Arc::new(DiskModels), importer: Some(Arc::new(ModelImporter)), ..Default::default() }; + s.execute("comp.new", json!({"name": "C", "width": 640, "height": 360, "duration": 2})).unwrap(); + s +} + +fn layer(s: &Session, id: u64) -> effectcraft_project::Layer { + s.active_comp().unwrap().layer(LayerId(id)).unwrap().clone() +} + +#[test] +fn import_gltf_and_add_model_layer() { + let mut s = session(); + let r = s.execute("file.import", json!({"paths": [fixture("quad.gltf")]})).unwrap(); + let item = r["items"][0].as_u64().unwrap(); + assert_eq!(s.project.item(ItemId(item)).unwrap().type_name(), "3D Model"); + // Adding the item to the comp makes a model layer (3D, fitted, with the clip popup). + let id = s.execute("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let l = layer(&s, id); + assert_eq!(l.source, LayerSource::Model { item: ItemId(item) }); + assert!(l.switches.three_d); + assert!(l.props.sub("masks").is_none() && l.props.sub("materialOptions").is_some()); + // Bounds 2 units tall → half the comp height. + let scale = l.props.prop("geometryOptions/unitScale").unwrap().value.as_f64(); + assert!((scale - 90.0).abs() < 1e-6, "{scale}"); + let anim = l.props.prop("geometryOptions/animation").unwrap(); + assert_eq!(anim.value, KV::Enum(1)); + match &anim.ui { + effectcraft_project::ParamUi::Popup { options } => assert_eq!(options, &vec!["None".to_string(), "Move".to_string()]), + u => panic!("{u:?}"), + } + s.undo(); + assert!(s.active_comp().unwrap().layers.is_empty()); + s.redo(); + assert_eq!(s.active_comp().unwrap().layers.len(), 1); + // layer.newModel imports and places in one step. + let r = s.execute("layer.newModel", json!({"path": fixture("cube.obj")})).unwrap(); + assert!(r["layer"].as_u64().is_some()); + assert!(s.execute("layer.newModel", json!({"path": "/nonexistent/thing.png"})).is_err()); +} + +#[test] +fn primitives_commands() { + let mut s = session(); + for k in PrimitiveKind::ALL { + let r = s.execute("layer.new3dPrimitive", json!({"kind": k.label().to_lowercase()})).unwrap(); + let l = layer(&s, r["layer"].as_u64().unwrap()); + assert_eq!(l.source, LayerSource::Primitive { kind: k }); + assert_eq!(l.name, k.label()); + assert!(l.switches.three_d); + assert!(l.props.sub("geometryOptions").is_some()); + } + let r = s + .execute("layer.newPrimitive", json!({"kind": "sphere", "radius": 80, "segments": 12, "baseColor": [1, 0, 0], "metallic": 100, "castsShadows": "on"})) + .unwrap(); + let l = layer(&s, r["layer"].as_u64().unwrap()); + assert_eq!(l.props.prop("geometryOptions/radius").unwrap().value.as_f64(), 80.0); + assert_eq!(l.props.prop("materialOptions/metallic").unwrap().value.as_f64(), 100.0); + assert_eq!(l.props.prop("materialOptions/castsShadows").unwrap().value, KV::Enum(1)); + assert!(s.execute("layer.newPrimitive", json!({"kind": "teapot"})).is_err()); + let n = s.active_comp().unwrap().layers.len(); + s.undo(); + assert_eq!(s.active_comp().unwrap().layers.len(), n - 1); + // Menu entries point at the real command now. + assert!(s.is_enabled("layer.new3dPrimitive")); +} + +#[test] +fn renderer_switch_adds_geometry_options() { + let mut s = session(); + let t = s.execute("layer.newText", json!({"text": "Hi"})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, t).props.sub("geometryOptions").is_none()); + s.execute("comp.renderer", json!({"renderer": "advanced3d"})).unwrap(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Advanced3D); + let g = layer(&s, t); + let g = g.props.sub("geometryOptions").unwrap(); + let ids: Vec<&str> = g.children.iter().map(|c| c.match_id()).collect(); + assert_eq!(ids, vec!["bevelStyle", "bevelDepth", "holeBevelDepth", "extrusionDepth"]); + // New shape layers in the Advanced 3D comp get them too. + let sh = s.execute("layer.newShape", json!({})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, sh).props.sub("geometryOptions").is_some()); + assert_eq!(s.execute("comp.renderer", json!({})).unwrap()["renderer"], "advanced3d"); + s.undo(); + s.undo(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Classic3D); + assert!(layer(&s, t).props.sub("geometryOptions").is_none()); + s.redo(); + assert_eq!(s.active_comp().unwrap().renderer, Renderer::Advanced3D); + assert!(s.execute("comp.renderer", json!({"renderer": "raytraced"})).is_err()); + // Extrusion values are ordinary properties. + s.execute("prop.set", json!({"layer": t, "path": "geometryOptions/extrusionDepth", "value": 40})).unwrap(); + assert_eq!(layer(&s, t).props.prop("geometryOptions/extrusionDepth").unwrap().value.as_f64(), 40.0); +} + +#[test] +fn environment_layer_toggle() { + let mut s = session(); + let id = s.execute("layer.newSolid", json!({"width": 64, "height": 32})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.environment", json!({"layer": id})).unwrap(); + assert!(layer(&s, id).environment && layer(&s, id).switches.three_d); + s.execute("layer.environment", json!({"layer": id})).unwrap(); + assert!(!layer(&s, id).environment); + s.undo(); + assert!(layer(&s, id).environment); + let cam = s.execute("layer.newCamera", json!({})).unwrap()["layer"].as_u64().unwrap(); + assert!(s.execute("layer.environment", json!({"layer": cam})).is_err()); + // An Environment light sources a layer. + let li = s.execute("layer.newLight", json!({"kind": "Environment"})).unwrap()["layer"].as_u64().unwrap(); + assert!(layer(&s, li).props.prop("lightOptions/source").is_some()); +} + +#[test] +fn material_commands_and_serde() { + let mut s = session(); + let a = s.execute("layer.newPrimitive", json!({"kind": "cube"})).unwrap()["layer"].as_u64().unwrap(); + let b = s.execute("layer.newPrimitive", json!({"kind": "torus"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("material.set", json!({"layers": [a], "baseColor": "#3366ff", "roughness": 20, "acceptsLights": false})).unwrap(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert!(s.execute("material.set", json!({"layers": [a], "castsShadows": "sometimes"})).is_err()); + s.execute("material.duplicateAssign", json!({"layers": [a, b]})).unwrap(); + assert_eq!(layer(&s, b).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert_eq!(layer(&s, b).props.prop("materialOptions/acceptsLights").unwrap().value, KV::Bool(false)); + s.execute("material.reset", json!({"layers": [a]})).unwrap(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 50.0); + s.undo(); + assert_eq!(layer(&s, a).props.prop("materialOptions/roughness").unwrap().value.as_f64(), 20.0); + assert!(s.execute("material.revealSource", json!({"layer": a})).is_err()); + // The project (model and primitive sources, environment flag) round-trips through JSON. + let r = s.execute("file.import", json!({"paths": [fixture("quad.glb")]})).unwrap(); + let item = r["items"][0].as_u64().unwrap(); + let m = s.execute("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + assert_eq!(s.execute("material.revealSource", json!({"layer": m})).unwrap()["item"], item); + let json = s.project.to_json(); + let back = Project::from_json(&json).unwrap(); + assert_eq!(&back, &*s.project); +} diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index e274f725..9926e04c 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -227,3 +227,26 @@ fn project_with_paint_and_puppet_round_trips() { let v: Value = serde_json::from_str(&j).unwrap(); assert!(v.to_string().contains("ec.paint.paint") && v.to_string().contains("ec.distort.puppet")); } + +#[test] +fn liquify_stroke_adds_the_effect_and_warps() { + let (mut s, id) = setup(); + // Layer (5, 30) is comp (55, 50). Before: opaque blue. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + let r = s.execute("liquify.stroke", json!({"layer": id, "tool": "warp", "points": [[-20, 30], [10, 30]], "size": 40, "pressure": 100})).unwrap(); + assert_eq!(r["strokes"], 1); + let l = layer(&s, id); + let fx = l.effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(&fx.kind, effectcraft_project::GroupKind::Effect { effect } if effect == "ec.distort.liquify")); + // The warp pulls transparent pixels from outside the layer over its left edge. + assert!(px(&s, 0.0, 55, 50)[3] < 0.9, "{:?}", px(&s, 0.0, 55, 50)); + // A second stroke goes into the same effect. + let r2 = s.execute("liquify.stroke", json!({"layer": id, "tool": "reconstruction", "points": [[0, 30]], "size": 60, "pressure": 100})).unwrap(); + assert_eq!(r2["strokes"], 2); + assert_eq!(r2["effect"], r["effect"]); + assert!(s.execute("liquify.stroke", json!({"layer": id, "tool": "smudge", "points": [[0, 0]]})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); +} diff --git a/crates/gpu/src/adv3d.rs b/crates/gpu/src/adv3d.rs new file mode 100644 index 00000000..6e428c84 --- /dev/null +++ b/crates/gpu/src/adv3d.rs @@ -0,0 +1,318 @@ +//! The Advanced 3D rasteriser on wgpu: a render pipeline with a reversed-Z depth buffer +//! (`Depth32Float`, Greater), the scene's materials, textures, lights and shadow maps in +//! storage buffers, and `advanced3d.wgsl`'s physically based fragment shader. Opaque triangles +//! are drawn with depth writes, then the sorted transparent ones blended over (premultiplied). +//! Colour (`Rgba16Float`) and camera depth (`R32Float`) are read back for the CPU's resolve, +//! depth of field and encoding. + +use effectcraft_render::three_d::adv::{Scene, Target}; +use wgpu::util::DeviceExt; + +use crate::context::{Enc, GpuContext}; + +/// Pipelines (built on first use). +pub(crate) struct Pipes { + bgl: wgpu::BindGroupLayout, + opaque: wgpu::RenderPipeline, + transparent: wgpu::RenderPipeline, +} + +const COLOR: wgpu::TextureFormat = wgpu::TextureFormat::Rgba16Float; +const DEPTH_OUT: wgpu::TextureFormat = wgpu::TextureFormat::R32Float; +const DEPTH: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float; +const VERTEX_SIZE: u64 = 52; + +fn storage(binding: u32) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: true }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +impl Pipes { + fn new(device: &wgpu::Device) -> Pipes { + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effectcraft advanced 3d"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/advanced3d.wgsl").into()), + }); + let mut entries = vec![wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }]; + entries.extend((1..=6).map(storage)); + let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("advanced 3d"), entries: &entries }); + let layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("advanced 3d"), bind_group_layouts: &[Some(&bgl)], immediate_size: 0 }); + let attrs = [ + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x3, offset: 0, shader_location: 0 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x3, offset: 12, shader_location: 1 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x2, offset: 24, shader_location: 2 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Float32x4, offset: 32, shader_location: 3 }, + wgpu::VertexAttribute { format: wgpu::VertexFormat::Uint32, offset: 48, shader_location: 4 }, + ]; + let make = |transparent: bool| { + let blend = transparent.then_some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(if transparent { "advanced 3d transparent" } else { "advanced 3d opaque" }), + layout: Some(&layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs"), + compilation_options: Default::default(), + buffers: &[Some(wgpu::VertexBufferLayout { array_stride: VERTEX_SIZE, step_mode: wgpu::VertexStepMode::Vertex, attributes: &attrs })], + }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + ..Default::default() + }, + depth_stencil: Some(wgpu::DepthStencilState { + format: DEPTH, + depth_write_enabled: Some(!transparent), + depth_compare: Some(wgpu::CompareFunction::Greater), + stencil: Default::default(), + bias: Default::default(), + }), + multisample: Default::default(), + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs"), + compilation_options: Default::default(), + targets: &[ + Some(wgpu::ColorTargetState { format: COLOR, blend, write_mask: wgpu::ColorWrites::ALL }), + Some(wgpu::ColorTargetState { + format: DEPTH_OUT, + blend: None, + write_mask: if transparent { wgpu::ColorWrites::empty() } else { wgpu::ColorWrites::ALL }, + }), + ], + }), + multiview_mask: None, + cache: None, + }) + }; + Pipes { opaque: make(false), transparent: make(true), bgl } + } +} + +fn f32s(v: impl IntoIterator) -> Vec { + let mut b: Vec = v.into_iter().flat_map(f32::to_le_bytes).collect(); + while b.len() < 16 { + b.push(0); + } + b +} + +/// Pack the scene's arrays as the shader's storage buffers. +fn pack(s: &Scene) -> [Vec; 7] { + let mut globals: Vec = vec![]; + // Column-major clip matrix. + for c in 0..4 { + for r in 0..4 { + globals.push(s.clip[r][c]); + } + } + globals.extend(s.view[2]); + globals.extend([s.eye[0], s.eye[1], s.eye[2], s.ortho as u32 as f32]); + globals.extend([s.cam_fwd[0], s.cam_fwd[1], s.cam_fwd[2], s.lights.len() as f32]); + globals.extend([s.ambient[0], s.ambient[1], s.ambient[2], s.env.is_some() as u32 as f32]); + let e = s.env.unwrap_or(effectcraft_render::three_d::adv::EnvInfo { radiance: 0, mips: 1, irradiance: 0, intensity: 0.0, rotation: 0.0 }); + globals.extend([e.radiance as f32, e.mips as f32, e.irradiance as f32, e.intensity]); + let lit = !(s.lights.is_empty() && s.env.is_none() && s.ambient == [0.0; 3]); + globals.extend([e.rotation, lit as u32 as f32, 0.0, 0.0]); + let mats = s.materials.iter().flat_map(|m| { + let flags = (m.double_sided as u32) | (m.unlit as u32) << 1 | (m.accepts_lights as u32) << 2 | (m.receives_shadows as u32) << 3 | (m.alpha_mode << 8); + [ + m.base[0], + m.base[1], + m.base[2], + m.base[3], + m.emissive[0], + m.emissive[1], + m.emissive[2], + m.metallic, + m.roughness, + m.normal_scale, + m.occlusion_strength, + m.alpha_cutoff, + m.tex_base as f32, + m.tex_mr as f32, + m.tex_normal as f32, + m.tex_occlusion as f32, + m.tex_emissive as f32, + flags as f32, + m.diffuse_k, + m.specular_k, + m.ambient_k, + m.opacity, + 0.0, + 0.0, + ] + }); + let mut tex: Vec = s.textures.iter().flat_map(|t| [t.offset, t.width, t.height, t.wrap_u | t.wrap_v << 8]).flat_map(u32::to_le_bytes).collect(); + while tex.len() < 16 { + tex.push(0); + } + let texels = f32s(s.texels.iter().flatten().copied()); + let lights = s.lights.iter().flat_map(|l| { + [ + l.pos[0], + l.pos[1], + l.pos[2], + l.kind as f32, + l.dir[0], + l.dir[1], + l.dir[2], + l.cos_outer, + l.color[0], + l.color[1], + l.color[2], + l.cos_inner, + l.falloff as f32, + l.radius, + l.falloff_distance, + l.shadow as f32, + l.shadow_darkness, + 0.0, + 0.0, + 0.0, + ] + }); + let shadows = s.shadows.iter().flat_map(|m| { + let mut v: Vec = m.view.iter().flatten().copied().collect(); + v.extend([m.focal, m.ortho as u32 as f32, f32::from_bits(m.size), f32::from_bits(m.offset), m.bias, m.radius, 0.0, 0.0]); + v + }); + [f32s(globals), f32s(mats), tex, texels, f32s(lights), f32s(shadows), f32s(s.shadow_texels.iter().copied())] +} + +fn half_to_f32(h: u16) -> f32 { + let s = if h & 0x8000 != 0 { -1.0 } else { 1.0 }; + let e = ((h >> 10) & 0x1f) as i32; + let m = (h & 0x3ff) as f32; + match e { + 0 => s * m * 2f32.powi(-24), + 31 => { + if m == 0.0 { + s * f32::INFINITY + } else { + f32::NAN + } + } + _ => s * (1.0 + m / 1024.0) * 2f32.powi(e - 15), + } +} + +/// Rasterise a scene. `None` when the device can't (no readback, size or buffer limits). +pub(crate) fn render(g: &GpuContext, s: &Scene) -> Option { + if !g.can_readback() || !g.fits(s.width, s.height) || s.indices.is_empty() { + return None; + } + let limits = g.device.limits(); + let bufs = pack(s); + if bufs.iter().any(|b| b.len() as u64 > limits.max_storage_buffer_binding_size) { + return None; + } + let p = g.adv3d.get_or_init(|| Pipes::new(&g.device)); + let dev = &g.device; + let vb: Vec = s + .vertices + .iter() + .flat_map(|v| { + let mut b = Vec::with_capacity(VERTEX_SIZE as usize); + for x in v.pos.iter().chain(&v.normal).chain(&v.uv).chain(&v.tangent) { + b.extend_from_slice(&x.to_le_bytes()); + } + b.extend_from_slice(&v.material.to_le_bytes()); + b + }) + .collect(); + let vbuf = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d vertices"), contents: &vb, usage: wgpu::BufferUsages::VERTEX }); + let ib: Vec = s.indices.iter().flat_map(|i| i.to_le_bytes()).collect(); + let ibuf = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d indices"), contents: &ib, usage: wgpu::BufferUsages::INDEX }); + let ub = dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d globals"), contents: &bufs[0], usage: wgpu::BufferUsages::UNIFORM }); + let sbs: Vec = bufs[1..] + .iter() + .map(|b| dev.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("adv3d data"), contents: b, usage: wgpu::BufferUsages::STORAGE })) + .collect(); + let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: ub.as_entire_binding() }]; + for (i, b) in sbs.iter().enumerate() { + entries.push(wgpu::BindGroupEntry { binding: i as u32 + 1, resource: b.as_entire_binding() }); + } + let bg = dev.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("adv3d"), layout: &p.bgl, entries: &entries }); + let tex = |format: wgpu::TextureFormat, usage: wgpu::TextureUsages| { + dev.create_texture(&wgpu::TextureDescriptor { + label: Some("adv3d target"), + size: wgpu::Extent3d { width: s.width, height: s.height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format, + usage, + view_formats: &[], + }) + }; + let rt = wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC; + let color = tex(COLOR, rt); + let zout = tex(DEPTH_OUT, rt); + let depth = tex(DEPTH, wgpu::TextureUsages::RENDER_ATTACHMENT); + let (cv, zv, dv) = (color.create_view(&Default::default()), zout.create_view(&Default::default()), depth.create_view(&Default::default())); + let mut enc = dev.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("adv3d") }); + { + let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("adv3d"), + color_attachments: &[ + Some(wgpu::RenderPassColorAttachment { + view: &cv, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store }, + }), + Some(wgpu::RenderPassColorAttachment { + view: &zv, + depth_slice: None, + resolve_target: None, + ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color { r: -1.0, g: 0.0, b: 0.0, a: 0.0 }), store: wgpu::StoreOp::Store }, + }), + ], + depth_stencil_attachment: Some(wgpu::RenderPassDepthStencilAttachment { + view: &dv, + depth_ops: Some(wgpu::Operations { load: wgpu::LoadOp::Clear(0.0), store: wgpu::StoreOp::Store }), + stencil_ops: None, + }), + timestamp_writes: None, + occlusion_query_set: None, + multiview_mask: None, + }); + pass.set_bind_group(0, &bg, &[]); + pass.set_vertex_buffer(0, vbuf.slice(..)); + pass.set_index_buffer(ibuf.slice(..), wgpu::IndexFormat::Uint32); + let n = s.indices.len() as u32; + if s.opaque_count > 0 { + pass.set_pipeline(&p.opaque); + pass.draw_indexed(0..s.opaque_count, 0, 0..1); + } + if n > s.opaque_count { + pass.set_pipeline(&p.transparent); + pass.draw_indexed(s.opaque_count..n, 0, 0..1); + } + } + g.queue.submit([enc.finish()]); + let mut e = Enc::new(g); + let cb = e.read_texture(&color, s.width, s.height, 8)?; + let zb = e.read_texture(&zout, s.width, s.height, 4)?; + let color = cb.chunks_exact(8).map(|c| [0, 1, 2, 3].map(|k| half_to_f32(u16::from_le_bytes([c[2 * k], c[2 * k + 1]])))).collect(); + let depth = zb + .chunks_exact(4) + .map(|c| { + let z = f32::from_le_bytes([c[0], c[1], c[2], c[3]]); + if z < 0.0 { f32::INFINITY } else { z } + }) + .collect(); + Some(Target { width: s.width, height: s.height, color, depth }) +} diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index 9f814377..1b141301 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -94,6 +94,8 @@ pub struct GpuContext { dummy_tex: wgpu::TextureView, dummy_buf: wgpu::Buffer, uploads: Mutex, + /// Advanced 3D render pipelines (built on first use). + pub(crate) adv3d: std::sync::OnceLock, } fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry { @@ -225,6 +227,7 @@ impl GpuContext { dummy_tex: dummy.create_view(&Default::default()), dummy_buf, uploads: Mutex::new(Uploads::default()), + adv3d: std::sync::OnceLock::new(), }) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index e7b54ab2..c299cf5d 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -9,6 +9,9 @@ //! clamping and quantisation, and converts colour spaces. 3D runs, adjustment layers and //! wireframes run on the CPU between GPU steps (read back, draw, upload). //! +//! Advanced 3D comps rasterise on a render pipeline (`advanced3d.wgsl`: depth buffer, PBR, +//! image-based light, shadow maps), see [`Accelerator::raster_3d`]. +//! //! GPU effects ([`effectcraft_effects::GPU_EFFECTS`]) run as compute kernels with the CPU //! effect's exact steps (padding, box-blur radii, parameter conversions); chains of them are //! uploaded and read back once. @@ -18,6 +21,7 @@ //! skip readback entirely with [`Gpu::render_display`], which leaves an RGBA8 texture for //! egui-wgpu to draw. +mod adv3d; mod context; mod effects; mod ops; @@ -121,7 +125,13 @@ impl Accelerator for Gpu { fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option { effects::run_chain(&mut Enc::new(&self.ctx), chain, buf, levels) } + + fn raster_3d(&self, scene: &effectcraft_render::three_d::adv::Scene) -> Option { + adv3d::render(&self.ctx, scene) + } } #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_adv3d; diff --git a/crates/gpu/src/shaders/advanced3d.wgsl b/crates/gpu/src/shaders/advanced3d.wgsl new file mode 100644 index 00000000..12100cc3 --- /dev/null +++ b/crates/gpu/src/shaders/advanced3d.wgsl @@ -0,0 +1,430 @@ +// Advanced 3D: rasterised meshes with physically based shading. A step-for-step port of +// `effectcraft_render::three_d::adv::shade` (Cook–Torrance GGX + Lambert, image-based light +// from an equirectangular environment, PCF shadow maps); the CPU rasteriser is the reference. + +struct Globals { + clip: mat4x4, + // Camera-space depth row (x, y, z, w). + view_z: vec4, + // eye.xyz, ortho (0/1) + eye: vec4, + // cam_fwd.xyz, light count + fwd: vec4, + // ambient.rgb, environment present (0/1) + ambient: vec4, + // radiance first texture, mip count, irradiance texture, intensity + env: vec4, + // rotation, any lighting (0/1), -, - + env2: vec4, +}; + +@group(0) @binding(0) var g: Globals; +@group(0) @binding(1) var materials: array>; +@group(0) @binding(2) var texinfo: array>; +@group(0) @binding(3) var texels: array>; +@group(0) @binding(4) var lights: array>; +@group(0) @binding(5) var shadows: array>; +@group(0) @binding(6) var shadow_texels: array; + +const PI: f32 = 3.14159265358979; +const MAT_STRIDE: u32 = 6u; +const LIGHT_STRIDE: u32 = 5u; +const SHADOW_STRIDE: u32 = 5u; + +struct VIn { + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) uv: vec2, + @location(3) tangent: vec4, + @location(4) material: u32, +}; + +struct VOut { + @builtin(position) clip: vec4, + @location(0) pos: vec3, + @location(1) normal: vec3, + @location(2) uv: vec2, + @location(3) tangent: vec4, + @location(4) @interpolate(flat) material: u32, +}; + +@vertex +fn vs(v: VIn) -> VOut { + var o: VOut; + o.clip = g.clip * vec4(v.pos, 1.0); + o.pos = v.pos; + o.normal = v.normal; + o.uv = v.uv; + o.tangent = v.tangent; + o.material = v.material; + return o; +} + +fn norm3(a: vec3) -> vec3 { + let l = sqrt(dot(a, a)); + if (l > 1e-20) { + return a / l; + } + return vec3(0.0, 0.0, -1.0); +} + +fn srgb_to_linear(c: f32) -> f32 { + if (c <= 0.04045) { + return c / 12.92; + } + return pow((c + 0.055) / 1.055, 2.4); +} + +fn wrap_i(i: i32, n: i32, mode: u32) -> i32 { + if (mode == 1u) { + return clamp(i, 0, n - 1); + } + if (mode == 2u) { + let p = 2 * n; + var m = i % p; + if (m < 0) { + m = m + p; + } + if (m >= n) { + return p - 1 - m; + } + return m; + } + var m = i % n; + if (m < 0) { + m = m + n; + } + return m; +} + +fn texel(t: vec4, x: i32, y: i32) -> vec4 { + let w = i32(t.y); + let h = i32(t.z); + let xi = wrap_i(x, w, t.w & 255u); + let yi = wrap_i(y, h, (t.w >> 8u) & 255u); + return texels[t.x + u32(yi * w + xi)]; +} + +fn sample_tex(i: i32, uv: vec2) -> vec4 { + let t = texinfo[u32(i)]; + if (t.y == 0u || t.z == 0u) { + return vec4(1.0); + } + let x = uv.x * f32(t.y) - 0.5; + let y = uv.y * f32(t.z) - 0.5; + let x0 = floor(x); + let y0 = floor(y); + let fx = x - x0; + let fy = y - y0; + let xi = i32(x0); + let yi = i32(y0); + let a = texel(t, xi, yi); + let b = texel(t, xi + 1, yi); + let c = texel(t, xi, yi + 1); + let d = texel(t, xi + 1, yi + 1); + let top = a + (b - a) * fx; + let bot = c + (d - c) * fx; + return top + (bot - top) * fy; +} + +fn d_ggx(n_h: f32, a: f32) -> f32 { + let a2 = a * a; + let d = n_h * n_h * (a2 - 1.0) + 1.0; + return a2 / max(PI * d * d, 1e-12); +} + +fn g1(x: f32, k: f32) -> f32 { + return x / (x * (1.0 - k) + k); +} + +fn brdf(base: vec3, metallic: f32, rough_in: f32, spec_k: f32, diff_k: f32, n: vec3, v: vec3, l: vec3) -> vec3 { + let n_l = dot(n, l); + if (n_l <= 0.0) { + return vec3(0.0); + } + let n_v = max(dot(n, v), 1e-4); + let h = norm3(v + l); + let n_h = max(dot(n, h), 0.0); + let v_h = max(dot(v, h), 0.0); + let r = clamp(rough_in, 0.03, 1.0); + let f0 = mix(vec3(0.04 * spec_k), base, metallic); + let fk = pow(clamp(1.0 - v_h, 0.0, 1.0), 5.0); + let f = f0 + (vec3(1.0) - f0) * fk; + let d = d_ggx(n_h, r * r); + let k = (r + 1.0) * (r + 1.0) / 8.0; + let gg = g1(n_v, k) * g1(n_l, k); + let spec = d * gg / max(4.0 * n_v * n_l, 1e-4); + let kd = (vec3(1.0) - f) * (1.0 - metallic); + return (kd * base / PI * diff_k + f * spec) * n_l * PI; +} + +fn env_brdf(n_v: f32, rough: f32) -> vec2 { + let c0 = vec4(-1.0, -0.0275, -0.572, 0.022); + let c1 = vec4(1.0, 0.0425, 1.04, -0.04); + let r = rough * c0 + c1; + let a004 = min(r.x * r.x, exp2(-9.28 * n_v)) * r.x + r.y; + return vec2(-1.04 * a004 + r.z, 1.04 * a004 + r.w); +} + +fn equirect_uv(d: vec3, rot: f32) -> vec2 { + let u = 0.5 + (atan2(d.x, d.z) + rot) / (2.0 * PI); + let v = acos(clamp(-d.y, -1.0, 1.0)) / PI; + return vec2(u - floor(u), v); +} + +fn shadow_factor(m: u32, p: vec3, n_l: f32) -> f32 { + let r0 = shadows[m]; + let r1 = shadows[m + 1u]; + let r2 = shadows[m + 2u]; + let info = shadows[m + 3u]; + let extra = shadows[m + 4u]; + let q = vec3(dot(r0.xyz, p) + r0.w, dot(r1.xyz, p) + r1.w, dot(r2.xyz, p) + r2.w); + let focal = info.x; + let ortho = info.y > 0.5; + let size = bitcast(info.z); + let offset = bitcast(info.w); + let c = f32(size) * 0.5; + var u: f32; + var v: f32; + if (ortho) { + u = q.x * focal + c; + v = q.y * focal + c; + } else { + if (q.z <= 1e-3) { + return 1.0; + } + u = q.x * focal / q.z + c; + v = q.y * focal / q.z + c; + } + let z = q.z; + let texel = select(abs(z) / focal, 1.0 / focal, ortho); + let bias = extra.x * texel * (1.0 + extra.y) * (1.0 + 3.0 * (1.0 - clamp(n_l, 0.0, 1.0))); + let step = extra.y / 2.0; + let sz = i32(size); + var lit = 0.0; + for (var j = -2; j <= 2; j = j + 1) { + for (var i = -2; i <= 2; i = i + 1) { + let x = i32(floor(u + f32(i) * step)); + let y = i32(floor(v + f32(j) * step)); + if (x < 0 || y < 0 || x >= sz || y >= sz) { + lit = lit + 1.0; + continue; + } + let d = shadow_texels[offset + u32(y * sz + x)]; + if (z - bias <= d) { + lit = lit + 1.0; + } + } + } + return lit / 25.0; +} + +fn light_shadow(li: u32, kind: u32, lpos: vec3, shadow: i32, darkness: f32, p: vec3, n_l: f32) -> f32 { + if (shadow < 0) { + return 1.0; + } + var face = 0u; + if (kind == 2u) { + let d = p - lpos; + let a = abs(d); + if (a.x >= a.y && a.x >= a.z) { + face = select(1u, 0u, d.x > 0.0); + } else if (a.y >= a.z) { + face = select(3u, 2u, d.y > 0.0); + } else { + face = select(5u, 4u, d.z > 0.0); + } + } + let f = shadow_factor((u32(shadow) + face) * SHADOW_STRIDE, p, n_l); + return 1.0 - darkness * (1.0 - f); +} + +fn attenuation(kind: u32, falloff: u32, radius: f32, falloff_distance: f32, cos_inner: f32, cos_outer: f32, dir: vec3, to_l: vec3, dist: f32) -> f32 { + var k = 1.0; + if (falloff == 1u && kind != 0u) { + if (dist <= radius) { + k = 1.0; + } else if (falloff_distance <= 0.0) { + k = 0.0; + } else { + let t = clamp((dist - radius) / falloff_distance, 0.0, 1.0); + k = 1.0 - t * t * (3.0 - 2.0 * t); + } + } else if (falloff == 2u && kind != 0u) { + if (dist <= radius || (radius <= 0.0 && dist <= 1.0)) { + k = 1.0; + } else { + let q = max(radius, 1.0) / dist; + k = q * q; + } + } + if (kind == 1u) { + let cos_a = clamp(dot(-to_l, dir), -1.0, 1.0); + if (cos_a >= cos_inner) { + k = k * 1.0; + } else if (cos_a <= cos_outer) { + k = 0.0; + } else { + let a = acos(cos_a); + let ai = acos(cos_inner); + let ao = acos(cos_outer); + let t = clamp((a - ai) / max(ao - ai, 1e-6), 0.0, 1.0); + k = k * (1.0 - t * t * (3.0 - 2.0 * t)); + } + } + return k; +} + +struct FOut { + @location(0) color: vec4, + @location(1) depth: f32, +}; + +@fragment +fn fs(i: VOut, @builtin(front_facing) front: bool) -> FOut { + let mb = i.material * MAT_STRIDE; + let m0 = materials[mb]; + let m1 = materials[mb + 1u]; + let m2 = materials[mb + 2u]; + let m3 = materials[mb + 3u]; + let m4 = materials[mb + 4u]; + let m5 = materials[mb + 5u]; + let flags = u32(m4.y); + let double_sided = (flags & 1u) != 0u; + let unlit = (flags & 2u) != 0u; + let accepts_lights = (flags & 4u) != 0u; + let receives_shadows = (flags & 8u) != 0u; + let alpha_mode = (flags >> 8u) & 3u; + if (!front && !double_sided) { + discard; + } + var base = m0; + let tex_base = i32(m3.x); + if (tex_base >= 0) { + base = base * sample_tex(tex_base, i.uv); + } + let opacity = m5.y; + var alpha = base.a * opacity; + if (alpha_mode == 1u && alpha < m2.w) { + discard; + } + if (alpha <= 1e-5 && alpha_mode != 0u) { + discard; + } + if (alpha_mode == 0u) { + alpha = opacity; + } + let rgb = base.rgb; + var emissive = m1.xyz; + let tex_em = i32(m4.x); + if (tex_em >= 0) { + emissive = emissive * sample_tex(tex_em, i.uv).rgb; + } + var out: FOut; + out.depth = dot(g.view_z.xyz, i.pos) + g.view_z.w; + if (unlit || !accepts_lights || g.env2.y < 0.5) { + let c = rgb + emissive; + out.color = vec4(c * alpha, alpha); + return out; + } + var metallic = m1.w; + var rough = m2.x; + let tex_mr = i32(m3.y); + if (tex_mr >= 0) { + let t = sample_tex(tex_mr, i.uv); + rough = rough * t.y; + metallic = metallic * t.z; + } + var n = norm3(i.normal); + if (!front) { + n = -n; + } + let tex_n = i32(m3.z); + if (tex_n >= 0) { + let t = sample_tex(tex_n, i.uv); + let ns = m2.y; + let tn = vec3((t.x * 2.0 - 1.0) * ns, (t.y * 2.0 - 1.0) * ns, t.z * 2.0 - 1.0); + let tg = norm3(i.tangent.xyz - n * dot(n, i.tangent.xyz)); + let sign = select(-i.tangent.w, i.tangent.w, front); + let bt = cross(n, tg) * sign; + n = norm3(tg * tn.x + bt * tn.y + n * tn.z); + } + var ao = 1.0; + let tex_ao = i32(m3.w); + if (tex_ao >= 0) { + ao = 1.0 + m2.z * (sample_tex(tex_ao, i.uv).x - 1.0); + } + var v: vec3; + if (g.eye.w > 0.5) { + v = -g.fwd.xyz; + } else { + v = norm3(g.eye.xyz - i.pos); + } + let diff_k = m4.z; + let spec_k = m4.w; + let amb_k = m5.x; + var acc = vec3(0.0); + let nl_count = u32(g.fwd.w); + for (var li = 0u; li < nl_count; li = li + 1u) { + let lb = li * LIGHT_STRIDE; + let l0 = lights[lb]; + let l1 = lights[lb + 1u]; + let l2 = lights[lb + 2u]; + let l3 = lights[lb + 3u]; + let l4 = lights[lb + 4u]; + let kind = u32(l0.w); + var to_l: vec3; + var dist: f32; + if (kind == 0u) { + to_l = -l1.xyz; + dist = 1e30; + } else { + let d = l0.xyz - i.pos; + let len = sqrt(dot(d, d)); + if (len < 1e-6) { + to_l = vec3(0.0, 0.0, -1.0); + dist = 0.0; + } else { + to_l = d / len; + dist = len; + } + } + let k = attenuation(kind, u32(l3.x), l3.y, l3.z, l2.w, l1.w, l1.xyz, to_l, dist); + if (k <= 0.0) { + continue; + } + let n_l = dot(n, to_l); + if (n_l <= 0.0) { + continue; + } + var sh = 1.0; + if (receives_shadows) { + sh = light_shadow(li, kind, l0.xyz, i32(l3.w), l4.x, i.pos, n_l); + } + let b = brdf(rgb, metallic, rough, spec_k, diff_k, n, v, to_l); + acc = acc + b * l2.xyz * k * sh; + } + acc = acc + g.ambient.xyz * rgb * amb_k * ao; + if (g.ambient.w > 0.5) { + let n_v = clamp(dot(n, v), 1e-4, 1.0); + let r = n * (2.0 * dot(n, v)) - v; + let rot = g.env2.x; + let irr = sample_tex(i32(g.env.z), equirect_uv(n, rot)).rgb * g.env.w; + let mips = u32(g.env.y); + let lv = clamp(rough, 0.0, 1.0) * f32(max(mips, 1u) - 1u); + let lo = u32(floor(lv)); + let hi = min(lo + 1u, max(mips, 1u) - 1u); + let f = lv - f32(lo); + let ruv = equirect_uv(r, rot); + let sa = sample_tex(i32(g.env.x) + i32(lo), ruv).rgb; + let sb = sample_tex(i32(g.env.x) + i32(hi), ruv).rgb; + let spec = (sa + (sb - sa) * f) * g.env.w; + let f0 = mix(vec3(0.04 * spec_k), rgb, metallic); + let ab = env_brdf(n_v, rough); + let fs = f0 * ab.x + vec3(ab.y); + acc = acc + (irr * rgb * (vec3(1.0) - fs) * (1.0 - metallic) * diff_k + spec * fs) * ao; + } + let c = acc + emissive; + out.color = vec4(c * alpha, alpha); + return out; +} diff --git a/crates/gpu/src/tests_adv3d.rs b/crates/gpu/src/tests_adv3d.rs new file mode 100644 index 00000000..067e4c8a --- /dev/null +++ b/crates/gpu/src/tests_adv3d.rs @@ -0,0 +1,167 @@ +//! Advanced 3D: the GPU rasteriser against the CPU reference on the same scenes. +//! +//! Tolerances: the GPU's colour target is half float (relative error ≤ 1e-3) and its +//! rasteriser and interpolators differ from the CPU's in the last bits, so pixels on triangle +//! edges may be covered by the other neighbour. Interior pixels agree within 2/255; at most +//! 1.5 % of pixels (silhouettes) may differ more. Skipped without an adapter. + +use effectcraft_keyframe::Value; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, LightKind, PrimitiveKind, Project, Renderer as R3, Solid, build}; +use effectcraft_render::three_d::adv::{self, Scene}; +use effectcraft_render::{Accelerator, EvalCtx, NoFootage, RenderOpts, Renderer}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::Gpu; + +fn gpu() -> Option { + let g = Gpu::headless(); + if g.is_none() { + eprintln!("effectcraft-gpu adv3d tests: no GPU adapter, skipping"); + } + g +} + +fn add(p: &mut Project, cid: ItemId, src: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let comp = p.comp(cid).unwrap().clone(); + let mut l = build::layer(p, &comp, "L", src, (comp.width, comp.height), None); + edit(&mut l); + p.comp_mut(cid).unwrap().layers.insert(0, l); +} + +fn set(l: &mut effectcraft_project::Layer, path: &str, v: Value) { + l.props.prop_mut(path).unwrap_or_else(|| panic!("{path}")).value = v; +} + +/// Primitives with metal/dielectric materials, a shadow-casting spot light and a point light, +/// an environment light, a semi-transparent textured card. +fn scene_project() -> (Project, ItemId) { + let mut p = Project::default(); + let mut c = Comp::new(240, 160, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + c.renderer = R3::Advanced3D; + let cid = p.add_item("C", Default::default(), None, ItemKind::Comp(c.into())); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Plane }, |l| { + set(l, "geometryOptions/width", Value::Scalar(900.0)); + set(l, "geometryOptions/height", Value::Scalar(900.0)); + set(l, "transform/position", Value::Vec3([120.0, 130.0, 0.0])); + set(l, "transform/rotationX", Value::Scalar(90.0)); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Sphere }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(35.0)); + set(l, "transform/position", Value::Vec3([80.0, 95.0, 0.0])); + set(l, "materialOptions/baseColor", Value::Color([0.9, 0.2, 0.2, 1.0])); + set(l, "materialOptions/roughness", Value::Scalar(35.0)); + set(l, "materialOptions/castsShadows", Value::Enum(1)); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Torus }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(35.0)); + set(l, "geometryOptions/tubeRadius", Value::Scalar(12.0)); + set(l, "transform/position", Value::Vec3([165.0, 90.0, 0.0])); + set(l, "transform/rotationX", Value::Scalar(60.0)); + set(l, "materialOptions/metallic", Value::Scalar(100.0)); + set(l, "materialOptions/roughness", Value::Scalar(20.0)); + set(l, "materialOptions/castsShadows", Value::Enum(1)); + }); + let sid = p.add_item("S", Default::default(), None, ItemKind::Solid(Solid { color: [0.2, 0.6, 0.9], width: 60, height: 40, pixel_aspect: 1.0 })); + add(&mut p, cid, LayerSource::Solid { item: sid }, |l| { + l.switches.three_d = true; + set(l, "transform/opacity", Value::Scalar(60.0)); + set(l, "transform/position", Value::Vec3([120.0, 80.0, -60.0])); + set(l, "transform/rotationY", Value::Scalar(30.0)); + }); + let env = p.add_item("E", Default::default(), None, ItemKind::Solid(Solid { color: [0.6, 0.7, 1.0], width: 32, height: 16, pixel_aspect: 1.0 })); + add(&mut p, cid, LayerSource::Solid { item: env }, |l| { + l.environment = true; + l.switches.three_d = true; + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Environment }, |l| set(l, "lightOptions/intensity", Value::Scalar(50.0))); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Spot }, |l| { + set(l, "transform/position", Value::Vec3([40.0, -120.0, -200.0])); + set(l, "transform/poi", Value::Vec3([120.0, 100.0, 0.0])); + set(l, "lightOptions/castsShadows", Value::Bool(true)); + set(l, "lightOptions/coneAngle", Value::Scalar(110.0)); + set(l, "lightOptions/shadowDiffusion", Value::Scalar(4.0)); + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Point }, |l| { + set(l, "transform/position", Value::Vec3([200.0, 20.0, -100.0])); + set(l, "lightOptions/intensity", Value::Scalar(40.0)); + set(l, "lightOptions/falloff", Value::Enum(1)); + }); + (p, cid) +} + +fn scene(p: &Project, cid: ItemId) -> Scene { + let r = Renderer::new(p, &NoFootage, RenderOpts::default()); + let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); + adv::scene_of(&r, &ctx, &run, (ctx.comp.width, ctx.comp.height)) +} + +fn compare(s: &Scene, g: &Gpu) { + let cpu = adv::raster::render(s); + let gt = g.raster_3d(s).expect("gpu raster"); + assert_eq!((gt.width, gt.height), (cpu.width, cpu.height)); + let n = cpu.color.len(); + let mut bad = 0; + let mut sum = 0.0f64; + for (a, b) in cpu.color.iter().zip(>.color) { + let d = (0..4).map(|k| (a[k] - b[k]).abs() / a[k].abs().max(1.0)).fold(0.0f32, f32::max); + sum += d as f64; + if d > 2.0 / 255.0 { + bad += 1; + } + } + let frac = bad as f64 / n as f64; + eprintln!("adv3d gpu vs cpu: {bad}/{n} pixels beyond 2/255 ({:.3} %), mean {:.5}", frac * 100.0, sum / n as f64); + assert!(frac < 0.015, "{:.3} % of pixels differ", frac * 100.0); + assert!(sum / (n as f64) < 0.002); + // Depth agrees where both see a surface. + let mut dz = 0; + for (a, b) in cpu.depth.iter().zip(>.depth) { + if a.is_finite() != b.is_finite() || (a.is_finite() && (a - b).abs() > 0.01 * a.abs().max(1.0)) { + dz += 1; + } + } + assert!((dz as f64) < 0.015 * n as f64, "{dz} depth mismatches"); +} + +#[test] +fn gpu_matches_cpu_lit_scene() { + let Some(g) = gpu() else { return }; + let (p, cid) = scene_project(); + let s = scene(&p, cid); + assert!(!s.shadows.is_empty() && s.env.is_some() && s.opaque_count < s.indices.len() as u32); + compare(&s, &g); +} + +#[test] +fn gpu_matches_cpu_unlit_and_full_frames() { + let Some(g) = gpu() else { return }; + let (mut p, cid) = scene_project(); + // Unlit: remove the lights. + p.comp_mut(cid).unwrap().layers.retain(|l| !l.is_light()); + compare(&scene(&p, cid), &g); + // Whole frames through the renderer with the GPU attached. + let (p, cid) = scene_project(); + let cpu = Renderer::new(&p, &NoFootage, RenderOpts::default()).comp_frame(cid, Tick::ZERO); + let mut r = Renderer::new(&p, &NoFootage, RenderOpts { backend: effectcraft_render::Backend::Gpu, ..Default::default() }); + r.accel = Some(&g); + let gimg = r.comp_frame(cid, Tick::ZERO); + let diff = cpu.data.iter().zip(&gimg.data).filter(|(a, b)| (0..4).any(|k| (a[k] - b[k]).abs() > 3.0 / 255.0)).count(); + assert!((diff as f64) < 0.02 * cpu.data.len() as f64, "{diff} pixels differ"); +} + +#[test] +fn gpu_raster_perf_smoke() { + let Some(g) = gpu() else { return }; + let (p, cid) = scene_project(); + let s = scene(&p, cid); + let t0 = std::time::Instant::now(); + let _ = adv::raster::render(&s); + let cpu = t0.elapsed(); + // First use compiles the pipelines. + let _ = g.raster_3d(&s); + let t0 = std::time::Instant::now(); + let _ = g.raster_3d(&s); + let gpu = t0.elapsed(); + eprintln!("adv3d 480x320 (2x SSAA of 240x160), {} triangles: cpu {:?}, gpu {:?}", s.indices.len() / 3, cpu, gpu); +} diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index 45e850ff..aa94b558 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -77,6 +77,39 @@ mod tests { assert!(img.data.iter().any(|p| p[3] > 0.5)); } + /// File ▸ Import of glTF/GLB/OBJ models through the media layer, placed as model layers and + /// drawn by the Advanced 3D renderer. + #[test] + fn imported_models_render_in_advanced_3d() { + let fx = |n: &str| format!("{}/../model/tests/fixtures/{n}", env!("CARGO_MANIFEST_DIR")); + let mut s = super::session(); + s.execute("comp.new", json!({"name": "M", "width": 160, "height": 120, "renderer": "advanced3d"})).unwrap(); + let r = s.execute("file.import", json!({"paths": [fx("cube.obj"), fx("quad.glb"), fx("quad.gltf")]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let items: Vec = r["items"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect(); + assert_eq!(items.len(), 3); + let cube = s.execute("layer.addItem", json!({"item": items[0]})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.set", json!({"layer": cube, "path": "transform/rotationY", "value": 30})).unwrap(); + s.execute("prop.set", json!({"layer": cube, "path": "transform/rotationX", "value": 20})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let opts = effectcraft_engine::render::RenderOpts::default(); + let img = s.render(cid, s.time(), opts); + let covered = img.data.iter().filter(|p| p[3] > 0.99).count(); + assert!(covered > 1000, "the cube covers part of the frame: {covered}"); + // Red faces (material Red) are visible. + assert!(img.data.iter().any(|p| p[3] > 0.99 && p[0] > 0.5 && p[1] < 0.2)); + // The textured binary glTF quad. + for _ in 0..3 { + s.execute("edit.undo", json!({})).unwrap(); + } + s.execute("layer.addItem", json!({"item": items[1]})).unwrap(); + let img = s.render(cid, s.time(), opts); + assert!(img.data.iter().filter(|p| p[3] > 0.99).count() > 1000); + // Classic 3D doesn't draw models. + s.execute("comp.renderer", json!({"renderer": "classic3d"})).unwrap(); + assert!(s.render(cid, s.time(), opts).data.iter().all(|p| p[3] == 0.0)); + } + /// The web app's path: outputs go to a sink (downloads), never to the file system. #[test] fn render_queue_exports_to_a_sink() { diff --git a/crates/lottie/src/export.rs b/crates/lottie/src/export.rs index fa5447d8..eeb6e273 100644 --- a/crates/lottie/src/export.rs +++ b/crates/lottie/src/export.rs @@ -400,6 +400,10 @@ impl Doc<'_> { self.ex.warn(format!("{name}: cameras and lights are not exported (3D; exported as a hidden null to keep parenting)")); hidden = true; } + LayerSource::Model { .. } | LayerSource::Primitive { .. } => { + self.ex.warn(format!("{name}: 3D model layers are not part of Lottie (layer exported as a hidden null)")); + hidden = true; + } } o.insert("ty".into(), json!(ty)); if let Some(p) = l.parent.and_then(|p| comp.index_of(p)) { diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 89992569..adc3edce 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,12 +14,15 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } +exr = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr"] } rayon = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs new file mode 100644 index 00000000..ba0034c3 --- /dev/null +++ b/crates/media/src/exr_channels.rs @@ -0,0 +1,152 @@ +//! Auxiliary channels from multi-layer OpenEXR files (depth, object / material IDs, normals, +//! Cryptomatte, …) for the 3D Channel effects, read with the pure-Rust `exr` crate. +//! +//! Every channel of every layer becomes a plane named `layer.channel` (or just `channel` for +//! the unnamed layer), placed in the display window (pixels outside a layer's data window are +//! 0, or the background depth for depth channels). Cryptomatte manifests come from the +//! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published +//! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_raster::AuxChannels; +use effectcraft_raster::channels3d::BACKGROUND_DEPTH; + +fn is_depth(name: &str) -> bool { + let last = name.rsplit('.').next().unwrap_or(name); + last.eq_ignore_ascii_case("z") || name.to_ascii_lowercase().contains("depth") +} + +/// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). +pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { + let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; + let mut v: Vec<(String, u32)> = m.iter().filter_map(|(k, h)| Some((k.clone(), u32::from_str_radix(h.as_str()?.trim(), 16).ok()?))).collect(); + v.sort(); + v +} + +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + use exr::prelude::*; + let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let (w, h) = (dw.size.0, dw.size.1); + if w == 0 || h == 0 { + return None; + } + let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); + let mut crypto: HashMap, Option)> = HashMap::new(); + let mut collect_meta = |other: &HashMap| { + for (k, v) in other { + let key = k.to_string(); + let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; + let Some((id, field)) = rest.split_once('/') else { continue }; + let AttributeValue::Text(t) = v else { continue }; + let e = crypto.entry(id.to_string()).or_default(); + match field { + "name" => e.0 = Some(t.to_string()), + "manifest" => e.1 = Some(t.to_string()), + _ => {} + } + } + }; + collect_meta(&img.attributes.other); + for layer in img.layer_data.iter() { + collect_meta(&layer.attributes.other); + let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); + let (lw, lh) = (layer.size.0, layer.size.1); + let pos = layer.attributes.layer_position; + let (ox, oy) = (pos.0 as i64 - dw.position.0 as i64, pos.1 as i64 - dw.position.1 as i64); + for ch in layer.channel_data.list.iter() { + let name = match &prefix { + Some(p) => format!("{p}.{}", ch.name), + None => ch.name.to_string(), + }; + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + let mut plane = vec![fill; w * h]; + let vals: Vec = ch.sample_data.values_as_f32().collect(); + if vals.len() < lw * lh { + continue; + } + for y in 0..lh { + let ty = y as i64 + oy; + if ty < 0 || ty >= h as i64 { + continue; + } + for x in 0..lw { + let tx = x as i64 + ox; + if tx < 0 || tx >= w as i64 { + continue; + } + plane[ty as usize * w + tx as usize] = vals[y * lw + x]; + } + } + aux.channels.push((name, plane)); + } + } + for (_, (name, manifest)) in crypto { + if let (Some(n), Some(m)) = (name, manifest) { + aux.manifests.push((n, parse_manifest(&m))); + } + } + aux.manifests.sort(); + Some(aux) +} + +type Cache = Mutex>)>>; + +/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(Vec::new())); + if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { + return v; + } + let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); + if let Ok(mut c) = cache.lock() { + if c.len() >= 4 { + c.remove(0); + } + c.push((path.to_string(), v.clone())); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_layered_channels_and_cryptomatte_metadata() { + use exr::prelude::*; + let (w, h) = (4usize, 2usize); + let z: Vec = (0..w * h).map(|i| i as f32 * 10.0).collect(); + let id: Vec = (0..w * h).map(|i| (i % 2) as f32 + 1.0).collect(); + let r: Vec = vec![0.5; w * h]; + let channels = AnyChannels::sort( + vec![ + AnyChannel::new("R", FlatSamples::F32(r)), + AnyChannel::new("Z", FlatSamples::F32(z.clone())), + AnyChannel::new("ObjectID", FlatSamples::F32(id.clone())), + ] + .into(), + ); + let mut layer = Layer::new((w, h), LayerAttributes::named("depth"), Encoding::FAST_LOSSLESS, channels); + layer.attributes.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + layer + .attributes + .other + .insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + let image = Image::from_layer(layer); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + let aux = read_exr_channels(bytes.get_ref()).unwrap(); + assert_eq!((aux.width, aux.height), (4, 2)); + assert_eq!(aux.get("depth.Z").unwrap(), z.as_slice()); + assert_eq!(aux.depth().unwrap(), z.as_slice()); + assert_eq!(aux.object_id().unwrap(), id.as_slice()); + assert_eq!(aux.manifests, vec![("CryptoObject".to_string(), vec![("bunny".to_string(), 0x1385_1a76), ("floor".to_string(), 0x0a1b_2c3d)])]); + assert!(read_exr_channels(b"not an exr").is_none()); + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e..862fbfc2 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -13,11 +13,12 @@ //! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). mod convert; +pub mod exr_channels; mod pool; mod probe; pub use pool::{DEFAULT_BUDGET, MediaPool, PoolStats}; -pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_single, sequence_files}; +pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_model, probe_single, sequence_files}; /// Errors from probing or decoding footage. #[derive(Debug, thiserror::Error)] @@ -50,10 +51,16 @@ pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "m4v", "mov", "mkv", "webm"]; +/// File extensions recognised as 3D models (Advanced 3D model layers). +pub const MODEL_EXTENSIONS: &[&str] = &["gltf", "glb", "obj"]; + /// Whether `path` has an extension we can import. pub fn is_importable(path: &std::path::Path) -> bool { let ext = ext_of(path); - STILL_EXTENSIONS.contains(&ext.as_str()) || AUDIO_EXTENSIONS.contains(&ext.as_str()) || VIDEO_EXTENSIONS.contains(&ext.as_str()) + MODEL_EXTENSIONS.contains(&ext.as_str()) + || STILL_EXTENSIONS.contains(&ext.as_str()) + || AUDIO_EXTENSIONS.contains(&ext.as_str()) + || VIDEO_EXTENSIONS.contains(&ext.as_str()) } pub(crate) fn ext_of(path: &std::path::Path) -> String { diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef..b6632fba 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -122,6 +122,8 @@ struct Inner { sources: Mutex, Option>>, /// In-memory files registered with `add_bytes` (web builds, tests). files: Mutex>>, + /// Parsed 3D models by path (`None`: failed to load; not retried until `forget`). + models: Mutex>>>, cache: Mutex, done: Condvar, budget: AtomicU64, @@ -183,6 +185,7 @@ impl MediaPool { inner: Arc::new(Inner { sources: Mutex::default(), files: Mutex::default(), + models: Mutex::default(), cache: Mutex::default(), done: Condvar::new(), budget: AtomicU64::new(bytes as u64), @@ -234,6 +237,7 @@ impl MediaPool { /// Forget a file (after it changed on disk, or to retry a file that failed to open). pub fn forget(&self, path: &str) { lock(&self.inner.sources).remove(path); + lock(&self.inner.models).remove(path); let mut c = lock(&self.inner.cache); let keys: Vec = c.lru.map.keys().filter(|k| &*k.path == path).cloned().collect(); for k in keys { @@ -305,6 +309,29 @@ impl Inner { self.budget.load(Ordering::Relaxed) as usize } + /// A 3D model file and its sibling resources (buffers, textures, MTL files), parsed once. + fn model(&self, path: &str) -> Option> { + if let Some(m) = lock(&self.models).get(path) { + return m.clone(); + } + let loaded = self.read(path).map_err(|e| e.to_string()).and_then(|bytes| { + let dir = std::path::Path::new(path).parent().map(|d| d.to_path_buf()).unwrap_or_default(); + let resolve = |uri: &str| -> Option> { + let p = dir.join(uri); + self.read(&p.to_string_lossy()).ok().map(|b| b.to_vec()) + }; + effectcraft_model::load(path, &bytes, &resolve).map_err(|e| e.to_string()) + }); + let m = match loaded { + Ok(m) => Some(Arc::new(m)), + Err(e) => { + log::warn!("media: cannot load model {path}: {e}"); + None + } + }; + lock(&self.models).entry(path.to_string()).or_insert(m).clone() + } + fn read(&self, path: &str) -> Result> { if let Some(b) = lock(&self.files).get(path) { return Ok(b.clone()); @@ -359,7 +386,7 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } - FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, + FootageKind::Still | FootageKind::Sequence | FootageKind::Model => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); @@ -451,6 +478,12 @@ impl Inner { } impl FootageSource for MediaPool { + fn model(&self, _item: ItemId, footage: &Footage) -> Option> { + if footage.missing || footage.kind != FootageKind::Model { + return None; + } + self.inner.model(&footage.path) + } fn set_cache_budget(&self, bytes: usize) { self.set_budget(bytes); } @@ -470,6 +503,18 @@ impl FootageSource for MediaPool { } } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { + if footage.missing || !footage.has_video { + return None; + } + let loc = Inner::locate(footage, t); + if loc.media_t.is_some() || !loc.key.path.to_ascii_lowercase().ends_with(".exr") { + return None; + } + let path = loc.key.path.clone(); + crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 4e1c5611..e97544db 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -28,11 +28,50 @@ pub fn probe(path: impl AsRef) -> Result { pub fn probe_single(path: impl AsRef) -> Result { let path = path.as_ref(); let bytes = std::fs::read(path).map_err(|e| MediaError::Io(format!("{}: {e}", path.display())))?; + if crate::MODEL_EXTENSIONS.contains(&ext_of(path).as_str()) { + let dir = path.parent().map(Path::to_path_buf).unwrap_or_default(); + return probe_model(&path.to_string_lossy(), &bytes, &|uri| std::fs::read(dir.join(uri)).ok()); + } probe_bytes(&path.to_string_lossy(), bytes.into()) } +/// Probe a 3D model (glTF 2.0 / OBJ): a [`FootageKind::Model`] item whose duration is its +/// longest animation. `resolve` reads sibling resources. +pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let m = effectcraft_model::load(path, bytes, resolve).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let s = m.summary(); + let dur = s.animations.iter().map(|a| a.1).fold(0.0f64, f64::max); + let codec = match effectcraft_model::format_of(path) { + Some("obj") => "OBJ", + Some("glb") => "GLB", + _ => "glTF", + }; + Ok(Footage { + path: path.to_string(), + kind: FootageKind::Model, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: DEFAULT_SEQUENCE_RATE, + native_rate: None, + duration: Tick::from_seconds_f64(dur), + has_video: false, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: format!("{codec} ({} meshes, {} triangles)", s.meshes, s.triangles), + missing: false, + sequence: Vec::new(), + color_profile: None, + }) +} + /// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if crate::MODEL_EXTENSIONS.contains(&ext_of(Path::new(path)).as_str()) { + return probe_model(path, &bytes, &|_| None); + } if let Ok(fmt) = image::guess_format(&bytes) { return probe_still_bytes(path, &bytes, fmt); } diff --git a/crates/model/Cargo.toml b/crates/model/Cargo.toml new file mode 100644 index 00000000..20a1fde0 --- /dev/null +++ b/crates/model/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "effectcraft-model" +description = "3D models: glTF 2.0 (.gltf/.glb) and Wavefront OBJ/MTL import, PBR metallic-roughness materials, node hierarchy, skinning and animation, parametric primitives, and extruded/bevelled outline meshes" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +image = { workspace = true } diff --git a/crates/model/src/extrude.rs b/crates/model/src/extrude.rs new file mode 100644 index 00000000..344c67cb --- /dev/null +++ b/crates/model/src/extrude.rs @@ -0,0 +1,301 @@ +//! Extruded, bevelled meshes from 2D outlines (Advanced 3D text and shape layers' Geometry +//! Options: Bevel Style, Bevel Depth, Hole Bevel Depth, Extrusion Depth). +//! +//! Input outlines are closed polylines in layer space (x right, y down). The mesh is in layer +//! space too: the front cap lies on the layer plane (z = 0, facing the camera, −z), the back cap +//! at z = depth. Each contour gets a ring of vertices per profile step (inset from the outline +//! by the bevel, at a depth along the profile); consecutive rings are joined by quads, the caps +//! are triangulated from the innermost front/back rings with holes. Caps and walls share +//! positions exactly, so the mesh is watertight (welded by position). + +use serde::{Deserialize, Serialize}; + +use crate::Primitive; +use crate::triangulate::{signed_area, triangulate}; + +type P = [f64; 2]; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum BevelStyle { + #[default] + None, + Angular, + Concave, + Convex, +} + +impl BevelStyle { + pub const ALL: [BevelStyle; 4] = [BevelStyle::None, BevelStyle::Angular, BevelStyle::Concave, BevelStyle::Convex]; + pub fn label(self) -> &'static str { + match self { + BevelStyle::None => "None", + BevelStyle::Angular => "Angular", + BevelStyle::Concave => "Concave", + BevelStyle::Convex => "Convex", + } + } + pub fn from_index(i: u32) -> BevelStyle { + BevelStyle::ALL.get(i as usize).copied().unwrap_or_default() + } + /// (inset fraction 1 → 0, depth fraction 0 → 1) along the profile at `t` in 0..1. + fn profile(self, t: f64) -> (f64, f64) { + let th = t * std::f64::consts::FRAC_PI_2; + match self { + BevelStyle::None | BevelStyle::Angular => (1.0 - t, t), + // Quarter circle bulging out towards the outline corner. + BevelStyle::Convex => (1.0 - th.sin(), 1.0 - th.cos()), + // Quarter circle scooped in. + BevelStyle::Concave => (th.cos(), th.sin()), + } + } + fn steps(self) -> usize { + match self { + BevelStyle::None => 0, + BevelStyle::Angular => 1, + _ => 6, + } + } +} + +/// Geometry Options of an extruded layer. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ExtrudeParams { + pub bevel: BevelStyle, + /// Pixels. + pub bevel_depth: f64, + /// Hole bevels as a fraction of the bevel depth (0..1). + pub hole_bevel: f64, + /// Pixels (> 0). + pub depth: f64, +} + +impl Default for ExtrudeParams { + fn default() -> Self { + ExtrudeParams { bevel: BevelStyle::None, bevel_depth: 2.0, hole_bevel: 1.0, depth: 50.0 } + } +} + +/// An outer contour with its holes. +#[derive(Clone, Debug, PartialEq)] +pub struct Outline { + pub outer: Vec

, + pub holes: Vec>, +} + +fn point_in(p: P, c: &[P]) -> bool { + let n = c.len(); + let mut inside = false; + let mut j = n - 1; + for i in 0..n { + let (a, b) = (c[i], c[j]); + if (a[1] > p[1]) != (b[1] > p[1]) && p[0] < (b[0] - a[0]) * (p[1] - a[1]) / (b[1] - a[1]) + a[0] { + inside = !inside; + } + j = i; + } + inside +} + +/// Group closed contours into outlines by nesting depth (even depth = outer, odd = hole of the +/// innermost containing outer contour). Degenerate contours are dropped. +pub fn group_contours(contours: Vec>) -> Vec { + let cs: Vec> = contours + .into_iter() + .map(|mut c| { + c.dedup(); + while c.len() > 1 && c.first() == c.last() { + c.pop(); + } + c + }) + .filter(|c| c.len() >= 3 && signed_area(c).abs() > 1e-9) + .collect(); + let n = cs.len(); + let area: Vec = cs.iter().map(|c| signed_area(c).abs()).collect(); + // Containers of each contour (by a vertex inside; nested contours do not cross). + let parents: Vec> = (0..n) + .map(|i| { + let probe = cs[i][0]; + (0..n).filter(|&j| j != i && area[j] > area[i] && point_in(probe, &cs[j])).collect() + }) + .collect(); + let mut outlines: Vec = vec![]; + let mut slot = vec![usize::MAX; n]; + let mut order: Vec = (0..n).collect(); + order.sort_by_key(|&i| parents[i].len()); + for i in order { + let depth = parents[i].len(); + if depth.is_multiple_of(2) { + slot[i] = outlines.len(); + outlines.push(Outline { outer: cs[i].clone(), holes: vec![] }); + } else if let Some(&p) = parents[i].iter().filter(|&&j| parents[j].len() == depth - 1).min_by(|&&a, &&b| area[a].total_cmp(&area[b])) + && slot[p] != usize::MAX + { + outlines[slot[p]].holes.push(cs[i].clone()); + } + } + outlines +} + +struct Contour { + pts: Vec

, + /// Outward unit normal of edge i (pts[i] → pts[i+1]). + edge_n: Vec

, + /// Miter offset direction per vertex (outward, length ≥ 1). + miter: Vec

, + /// Smooth-shaded vertex (shallow corner). + smooth: Vec, + bevel: f64, +} + +fn contour(mut pts: Vec

, outer: bool, bevel: f64) -> Contour { + // Outer counter-clockwise (in x/y), holes clockwise: the outward normal is (dy, −dx). + let ccw = signed_area(&pts) > 0.0; + if ccw != outer { + pts.reverse(); + } + let n = pts.len(); + let edge_n: Vec

= (0..n) + .map(|i| { + let (a, b) = (pts[i], pts[(i + 1) % n]); + let (dx, dy) = (b[0] - a[0], b[1] - a[1]); + let l = dx.hypot(dy).max(1e-12); + [dy / l, -dx / l] + }) + .collect(); + let mut miter = vec![[0.0; 2]; n]; + let mut smooth = vec![false; n]; + for i in 0..n { + let (n0, n1) = (edge_n[(i + n - 1) % n], edge_n[i]); + let d = n0[0] * n1[0] + n0[1] * n1[1]; + let s = [n0[0] + n1[0], n0[1] + n1[1]]; + let k = 1.0 + d; + miter[i] = if k < 0.25 { + // Very sharp corner: limit the miter (offset along the bisector, length 2.8). + let l = s[0].hypot(s[1]).max(1e-12); + [s[0] / l * 2.8, s[1] / l * 2.8] + } else { + [s[0] / k, s[1] / k] + }; + smooth[i] = d > 0.82; // < ~35° turn + } + Contour { pts, edge_n, miter, smooth, bevel } +} + +/// Rings of a contour: (inset, z) from the front cap to the back cap. +fn rings(style: BevelStyle, bevel: f64, depth: f64) -> Vec<(f64, f64)> { + let b = bevel.clamp(0.0, depth * 0.5); + let steps = if b > 1e-9 { style.steps() } else { 0 }; + if steps == 0 { + return vec![(0.0, 0.0), (0.0, depth)]; + } + let front: Vec<(f64, f64)> = (0..=steps) + .map(|k| { + let (f, g) = style.profile(k as f64 / steps as f64); + (b * f, b * g) + }) + .collect(); + let mut out = front.clone(); + let back: Vec<(f64, f64)> = front.iter().rev().map(|&(i, z)| (i, depth - z)).collect(); + // Skip a duplicate ring where the walls have zero height (bevel = depth / 2). + let start = if (back[0].1 - out[out.len() - 1].1).abs() < 1e-9 { 1 } else { 0 }; + out.extend_from_slice(&back[start..]); + out +} + +/// Build the extruded mesh of outlines (one primitive, material 0). +pub fn extrude(outlines: &[Outline], p: &ExtrudeParams) -> Primitive { + let mut prim = Primitive { material: Some(0), ..Default::default() }; + let depth = p.depth.max(1e-3); + let push = |prim: &mut Primitive, pos: [f64; 3], n: [f64; 3]| -> u32 { + prim.positions.push([pos[0] as f32, pos[1] as f32, pos[2] as f32]); + let l = (n[0] * n[0] + n[1] * n[1] + n[2] * n[2]).sqrt().max(1e-12); + prim.normals.push([(n[0] / l) as f32, (n[1] / l) as f32, (n[2] / l) as f32]); + prim.uvs.push([0.0, 0.0]); + (prim.positions.len() - 1) as u32 + }; + for o in outlines { + let mut cs = vec![contour(o.outer.clone(), true, p.bevel_depth)]; + for h in &o.holes { + cs.push(contour(h.clone(), false, p.bevel_depth * p.hole_bevel.clamp(0.0, 1.0))); + } + // Positions per contour per ring. + let mut ring_pos: Vec>> = vec![]; + let mut ring_par: Vec> = vec![]; + for c in &cs { + let rs = rings(p.bevel, c.bevel, depth); + let pos: Vec> = + rs.iter().map(|&(inset, z)| c.pts.iter().zip(&c.miter).map(|(q, m)| [q[0] - m[0] * inset, q[1] - m[1] * inset, z]).collect()).collect(); + ring_pos.push(pos); + ring_par.push(rs); + } + // Walls and bevels. + for (ci, c) in cs.iter().enumerate() { + let rs = &ring_par[ci]; + let n = c.pts.len(); + let curved = matches!(p.bevel, BevelStyle::Concave | BevelStyle::Convex); + // Profile normal (outward weight, z weight) of segment k. + let seg_n = |k: usize| -> (f64, f64) { + let (i0, z0) = rs[k]; + let (i1, z1) = rs[k + 1]; + let (di, dz) = (i1 - i0, z1 - z0); + let l = di.hypot(dz).max(1e-12); + (dz / l, di / l) + }; + for k in 0..rs.len() - 1 { + let is_wall = (rs[k].0 - rs[k + 1].0).abs() < 1e-12; + let (a0, a1) = (seg_n(k), seg_n(k)); + // Curved bevels: average with the neighbouring bevel segments at shared rings. + let blend = |k2: Option, base: (f64, f64)| -> (f64, f64) { + match k2 { + Some(k2) if curved && !is_wall && k2 < rs.len() - 1 && (rs[k2].0 - rs[k2 + 1].0).abs() >= 1e-12 => { + let o = seg_n(k2); + (base.0 + o.0, base.1 + o.1) + } + _ => base, + } + }; + let pa = blend(k.checked_sub(1), a0); + let pb = blend(Some(k + 1), a1); + for i in 0..n { + let j = (i + 1) % n; + let en = c.edge_n[i]; + let vn = |v: usize| -> P { + if c.smooth[v] { + let m = c.miter[v]; + let l = m[0].hypot(m[1]).max(1e-12); + [m[0] / l, m[1] / l] + } else { + en + } + }; + let (ni, nj) = (vn(i), vn(j)); + let nrm = |xy: P, pr: (f64, f64)| [xy[0] * pr.0, xy[1] * pr.0, pr.1]; + let ai = push(&mut prim, ring_pos[ci][k][i], nrm(ni, pa)); + let aj = push(&mut prim, ring_pos[ci][k][j], nrm(nj, pa)); + let bj = push(&mut prim, ring_pos[ci][k + 1][j], nrm(nj, pb)); + let bi = push(&mut prim, ring_pos[ci][k + 1][i], nrm(ni, pb)); + prim.indices.extend([ai, aj, bj, ai, bj, bi]); + } + } + } + // Caps: triangulate in x/y (counter-clockwise → +z normal: the back cap; the front cap + // is reversed to face the camera). + let outer2: Vec

= ring_pos[0][0].iter().map(|q| [q[0], q[1]]).collect(); + let holes2: Vec> = ring_pos[1..].iter().map(|r| r[0].iter().map(|q| [q[0], q[1]]).collect()).collect(); + let tris = triangulate(&outer2, &holes2); + let flat: Vec<(usize, usize)> = (0..cs.len()).flat_map(|ci| (0..cs[ci].pts.len()).map(move |i| (ci, i))).collect(); + for (front, nz) in [(true, -1.0), (false, 1.0)] { + let base = prim.positions.len() as u32; + for &(ci, i) in &flat { + let r = if front { 0 } else { ring_pos[ci].len() - 1 }; + push(&mut prim, ring_pos[ci][r][i], [0.0, 0.0, nz]); + } + for t in &tris { + let (a, b, c) = (base + t[0] as u32, base + t[1] as u32, base + t[2] as u32); + if front { prim.indices.extend([a, c, b]) } else { prim.indices.extend([a, b, c]) } + } + } + } + prim +} diff --git a/crates/model/src/gltf.rs b/crates/model/src/gltf.rs new file mode 100644 index 00000000..375884bc --- /dev/null +++ b/crates/model/src/gltf.rs @@ -0,0 +1,519 @@ +//! glTF 2.0 import, written from the Khronos glTF 2.0 specification. +//! +//! Supports `.gltf` (JSON with external `.bin`/image files or `data:` URIs) and `.glb` (binary +//! container: a JSON chunk and a BIN chunk). Imports the default scene's node hierarchy (TRS or +//! matrix), triangle primitives (`TRIANGLES`, `TRIANGLE_STRIP`, `TRIANGLE_FAN`) with POSITION, +//! NORMAL, TANGENT, TEXCOORD_0, JOINTS_0 and WEIGHTS_0, PBR metallic-roughness materials +//! (base colour, metallic-roughness, normal, occlusion and emissive textures; alpha modes; +//! double-sided; `KHR_materials_emissive_strength`, `KHR_materials_unlit`), embedded or +//! external PNG/JPEG images with sampler wrap modes, skins and animations (step, linear and +//! cubic-spline translation/rotation/scale). Sparse accessors, morph targets, cameras and +//! lights are ignored. + +use std::sync::Arc; + +use serde_json::Value; + +use crate::{ + AlphaMode, Animation, Channel, ChannelPath, Interp, Mat4, Material, Mesh, Model, ModelError, Node, Primitive, Result, Skin, TexRef, Texture, Wrap, perr, +}; + +const GLB_MAGIC: u32 = 0x4654_6C67; +const CHUNK_JSON: u32 = 0x4E4F_534A; +const CHUNK_BIN: u32 = 0x004E_4942; + +/// Split a GLB container into its JSON text and BIN chunk. +pub fn split_glb(b: &[u8]) -> Result<(&[u8], Option<&[u8]>)> { + let u32_at = |o: usize| -> Option { b.get(o..o + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]])) }; + if u32_at(0) != Some(GLB_MAGIC) { + return Err(perr("not a GLB file")); + } + if u32_at(4) != Some(2) { + return Err(ModelError::Unsupported(format!("GLB version {}", u32_at(4).unwrap_or(0)))); + } + let total = (u32_at(8).unwrap_or(0) as usize).min(b.len()); + let mut o = 12; + let mut json = None; + let mut bin = None; + while o + 8 <= total { + let len = u32_at(o).unwrap_or(0) as usize; + let ty = u32_at(o + 4).unwrap_or(0); + let data = b.get(o + 8..o + 8 + len).ok_or_else(|| perr("GLB chunk runs past the end of the file"))?; + match ty { + CHUNK_JSON if json.is_none() => json = Some(data), + CHUNK_BIN if bin.is_none() => bin = Some(data), + _ => {} + } + o += 8 + len; + } + Ok((json.ok_or_else(|| perr("GLB without a JSON chunk"))?, bin)) +} + +/// Decode standard base64 (with or without padding; whitespace ignored). +pub fn base64_decode(s: &str) -> Option> { + let mut out = Vec::with_capacity(s.len() * 3 / 4); + let mut acc = 0u32; + let mut bits = 0; + for c in s.bytes() { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' | b'-' => 62, + b'/' | b'_' => 63, + b'=' => break, + b' ' | b'\n' | b'\r' | b'\t' => continue, + _ => return None, + } as u32; + acc = (acc << 6) | v; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + Some(out) +} + +/// Percent-decode a relative URI. +pub fn uri_decode(s: &str) -> String { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' + && i + 2 < b.len() + && let Ok(v) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or("zz"), 16) + { + out.push(v); + i += 3; + continue; + } + out.push(b[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Bytes of a `data:` URI or an external resource. +fn load_uri(uri: &str, resolve: &dyn Fn(&str) -> Option>) -> Result> { + if let Some(rest) = uri.strip_prefix("data:") { + let (meta, data) = rest.split_once(',').ok_or_else(|| perr("malformed data URI"))?; + return if meta.ends_with(";base64") { base64_decode(data).ok_or_else(|| perr("bad base64 in data URI")) } else { Ok(uri_decode(data).into_bytes()) }; + } + let name = uri_decode(uri); + resolve(&name).ok_or(ModelError::Missing(name)) +} + +struct Doc<'a> { + j: &'a Value, + buffers: Vec>, +} + +fn arr<'v>(v: &'v Value, k: &str) -> &'v [Value] { + v.get(k).and_then(Value::as_array).map(Vec::as_slice).unwrap_or(&[]) +} +fn idx(v: &Value, k: &str) -> Option { + v.get(k).and_then(Value::as_u64).map(|x| x as usize) +} +fn num(v: &Value, k: &str, d: f64) -> f64 { + v.get(k).and_then(Value::as_f64).unwrap_or(d) +} +fn nums(v: &Value, k: &str, d: [f64; N]) -> [f64; N] { + let mut out = d; + if let Some(a) = v.get(k).and_then(Value::as_array) { + for (i, x) in a.iter().take(N).enumerate() { + out[i] = x.as_f64().unwrap_or(d[i]); + } + } + out +} + +impl Doc<'_> { + /// An accessor as `f32` rows of `n` components (normalised integers mapped to 0..1 / -1..1). + fn floats(&self, a: usize) -> Result<(Vec, usize)> { + let acc = arr(self.j, "accessors").get(a).ok_or_else(|| perr(format!("no accessor {a}")))?; + let n = match acc.get("type").and_then(Value::as_str).unwrap_or("") { + "SCALAR" => 1, + "VEC2" => 2, + "VEC3" => 3, + "VEC4" => 4, + "MAT2" => 4, + "MAT3" => 9, + "MAT4" => 16, + t => return Err(perr(format!("accessor type `{t}`"))), + }; + let count = idx(acc, "count").unwrap_or(0); + let ct = idx(acc, "componentType").unwrap_or(5126); + let norm = acc.get("normalized").and_then(Value::as_bool).unwrap_or(false); + let csize = match ct { + 5120 | 5121 => 1, + 5122 | 5123 => 2, + 5125 | 5126 => 4, + _ => return Err(perr(format!("componentType {ct}"))), + }; + let mut out = vec![0.0f32; count * n]; + let Some(bv) = idx(acc, "bufferView") else { + // No buffer view: all zeros (sparse substitution is not supported). + return Ok((out, n)); + }; + let view = arr(self.j, "bufferViews").get(bv).ok_or_else(|| perr(format!("no bufferView {bv}")))?; + let buf = self.buffers.get(idx(view, "buffer").unwrap_or(0)).ok_or_else(|| perr("bufferView buffer out of range"))?; + let base = idx(view, "byteOffset").unwrap_or(0) + idx(acc, "byteOffset").unwrap_or(0); + let elem = csize * n; + // Matrices of 1/2-byte components pad columns to 4 bytes; ignored (rare). + let stride = idx(view, "byteStride").filter(|s| *s > 0).unwrap_or(elem); + for i in 0..count { + for c in 0..n { + let o = base + i * stride + c * csize; + let s = buf.get(o..o + csize).ok_or_else(|| perr("accessor reads past its buffer"))?; + out[i * n + c] = match ct { + 5126 => f32::from_le_bytes([s[0], s[1], s[2], s[3]]), + 5125 => u32::from_le_bytes([s[0], s[1], s[2], s[3]]) as f32, + 5123 => { + let v = u16::from_le_bytes([s[0], s[1]]) as f32; + if norm { v / 65535.0 } else { v } + } + 5122 => { + let v = i16::from_le_bytes([s[0], s[1]]) as f32; + if norm { (v / 32767.0).max(-1.0) } else { v } + } + 5121 => { + let v = s[0] as f32; + if norm { v / 255.0 } else { v } + } + _ => { + let v = s[0] as i8 as f32; + if norm { (v / 127.0).max(-1.0) } else { v } + } + }; + } + } + Ok((out, n)) + } + + /// Integer accessor (indices, joints) read exactly. + fn ints(&self, a: usize) -> Result> { + let acc = arr(self.j, "accessors").get(a).ok_or_else(|| perr(format!("no accessor {a}")))?; + let ct = idx(acc, "componentType").unwrap_or(5125); + if ct == 5126 { + return Ok(self.floats(a)?.0.into_iter().map(|v| v as u32).collect()); + } + let n = match acc.get("type").and_then(Value::as_str).unwrap_or("SCALAR") { + "VEC4" => 4, + "VEC3" => 3, + "VEC2" => 2, + _ => 1, + }; + let count = idx(acc, "count").unwrap_or(0); + let csize = match ct { + 5120 | 5121 => 1, + 5122 | 5123 => 2, + _ => 4, + }; + let Some(bv) = idx(acc, "bufferView") else { return Ok(vec![0; count * n]) }; + let view = arr(self.j, "bufferViews").get(bv).ok_or_else(|| perr(format!("no bufferView {bv}")))?; + let buf = self.buffers.get(idx(view, "buffer").unwrap_or(0)).ok_or_else(|| perr("bufferView buffer out of range"))?; + let base = idx(view, "byteOffset").unwrap_or(0) + idx(acc, "byteOffset").unwrap_or(0); + let stride = idx(view, "byteStride").filter(|s| *s > 0).unwrap_or(csize * n); + let mut out = Vec::with_capacity(count * n); + for i in 0..count { + for c in 0..n { + let o = base + i * stride + c * csize; + let s = buf.get(o..o + csize).ok_or_else(|| perr("accessor reads past its buffer"))?; + out.push(match csize { + 1 => s[0] as u32, + 2 => u16::from_le_bytes([s[0], s[1]]) as u32, + _ => u32::from_le_bytes([s[0], s[1], s[2], s[3]]), + }); + } + } + Ok(out) + } +} + +/// Parse a `.gltf` (JSON) or `.glb` file. +pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let (json, bin) = if bytes.starts_with(b"glTF") { split_glb(bytes)? } else { (bytes, None) }; + let j: Value = serde_json::from_slice(json).map_err(|e| perr(format!("glTF JSON: {e}")))?; + let ver = j.get("asset").and_then(|a| a.get("version")).and_then(Value::as_str).unwrap_or(""); + if !ver.starts_with('2') { + return Err(ModelError::Unsupported(format!("glTF version `{ver}` (2.0 required)"))); + } + for ext in arr(&j, "extensionsRequired") { + let e = ext.as_str().unwrap_or(""); + if !matches!(e, "KHR_materials_unlit" | "KHR_materials_emissive_strength") { + return Err(ModelError::Unsupported(format!("required extension {e}"))); + } + } + let mut buffers = vec![]; + for (i, b) in arr(&j, "buffers").iter().enumerate() { + let data = match b.get("uri").and_then(Value::as_str) { + Some(u) => load_uri(u, resolve)?, + None if i == 0 => bin.map(<[u8]>::to_vec).ok_or_else(|| perr("buffer 0 has no uri and the file has no BIN chunk"))?, + None => vec![], + }; + buffers.push(data); + } + let d = Doc { j: &j, buffers }; + + // Images → textures (decoded once per image; samplers set the wrap modes). + let mut images: Vec>> = vec![]; + for im in arr(&j, "images") { + let data = if let Some(u) = im.get("uri").and_then(Value::as_str) { + load_uri(u, resolve).ok() + } else if let Some(bv) = idx(im, "bufferView") { + arr(&j, "bufferViews").get(bv).and_then(|v| { + let b = d.buffers.get(idx(v, "buffer").unwrap_or(0))?; + let o = idx(v, "byteOffset").unwrap_or(0); + b.get(o..o + idx(v, "byteLength").unwrap_or(0)).map(<[u8]>::to_vec) + }) + } else { + None + }; + images.push(data.and_then(|b| Texture::decode(&b).ok()).map(Arc::new)); + } + let wrap = |v: u64| match v { + 33071 => Wrap::Clamp, + 33648 => Wrap::Mirror, + _ => Wrap::Repeat, + }; + let mut textures: Vec> = vec![]; + let mut tex_map: Vec> = vec![]; + for t in arr(&j, "textures") { + let Some(img) = idx(t, "source").and_then(|s| images.get(s).cloned().flatten()) else { + tex_map.push(None); + continue; + }; + let samp = idx(t, "sampler").and_then(|s| arr(&j, "samplers").get(s)); + let ws = samp.and_then(|s| s.get("wrapS")).and_then(Value::as_u64).unwrap_or(10497); + let wt = samp.and_then(|s| s.get("wrapT")).and_then(Value::as_u64).unwrap_or(10497); + let w = [wrap(ws), wrap(wt)]; + let tex = if img.wrap == w { img } else { Arc::new(Texture { wrap: w, ..(*img).clone() }) }; + tex_map.push(Some(textures.len())); + textures.push(tex); + } + let tref = |v: Option<&Value>| -> Option { + let v = v?; + let t = tex_map.get(idx(v, "index")?).copied().flatten()?; + Some(TexRef { texture: t, uv_set: idx(v, "texCoord").unwrap_or(0) as u32 }) + }; + + let mut materials = vec![]; + for m in arr(&j, "materials") { + let pbr = m.get("pbrMetallicRoughness").cloned().unwrap_or(Value::Null); + let bc = nums(&pbr, "baseColorFactor", [1.0; 4]); + let ext = m.get("extensions"); + let strength = ext.and_then(|e| e.get("KHR_materials_emissive_strength")).map_or(1.0, |e| num(e, "emissiveStrength", 1.0)); + let em = nums(m, "emissiveFactor", [0.0; 3]); + materials.push(Material { + name: m.get("name").and_then(Value::as_str).unwrap_or("").to_string(), + base_color: bc.map(|v| v as f32), + base_color_tex: tref(pbr.get("baseColorTexture")), + metallic: num(&pbr, "metallicFactor", 1.0) as f32, + roughness: num(&pbr, "roughnessFactor", 1.0) as f32, + metallic_roughness_tex: tref(pbr.get("metallicRoughnessTexture")), + normal_tex: tref(m.get("normalTexture")), + normal_scale: m.get("normalTexture").map_or(1.0, |n| num(n, "scale", 1.0)) as f32, + occlusion_tex: tref(m.get("occlusionTexture")), + occlusion_strength: m.get("occlusionTexture").map_or(1.0, |n| num(n, "strength", 1.0)) as f32, + emissive: [(em[0] * strength) as f32, (em[1] * strength) as f32, (em[2] * strength) as f32], + emissive_tex: tref(m.get("emissiveTexture")), + alpha_mode: match m.get("alphaMode").and_then(Value::as_str) { + Some("MASK") => AlphaMode::Mask, + Some("BLEND") => AlphaMode::Blend, + _ => AlphaMode::Opaque, + }, + alpha_cutoff: num(m, "alphaCutoff", 0.5) as f32, + double_sided: m.get("doubleSided").and_then(Value::as_bool).unwrap_or(false), + unlit: ext.and_then(|e| e.get("KHR_materials_unlit")).is_some(), + }); + } + + let mut meshes = vec![]; + for m in arr(&j, "meshes") { + let mut mesh = Mesh { name: m.get("name").and_then(Value::as_str).unwrap_or("").to_string(), primitives: vec![] }; + for p in arr(m, "primitives") { + if let Some(prim) = primitive(&d, p, &materials)? { + mesh.primitives.push(prim); + } + } + meshes.push(mesh); + } + + let mut nodes = vec![]; + for n in arr(&j, "nodes") { + let matrix = n.get("matrix").and_then(Value::as_array).filter(|a| a.len() == 16).map(|a| { + // Column-major in the file. + let v: Vec = a.iter().map(|x| x.as_f64().unwrap_or(0.0)).collect(); + let mut m = [[0.0; 4]; 4]; + for c in 0..4 { + for r in 0..4 { + m[r][c] = v[c * 4 + r]; + } + } + Mat4(m) + }); + nodes.push(Node { + name: n.get("name").and_then(Value::as_str).unwrap_or("").to_string(), + translation: nums(n, "translation", [0.0; 3]), + rotation: nums(n, "rotation", [0.0, 0.0, 0.0, 1.0]), + scale: nums(n, "scale", [1.0; 3]), + matrix, + mesh: idx(n, "mesh").filter(|&m| m < meshes.len()), + skin: idx(n, "skin"), + children: arr(n, "children").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(), + }); + } + + let scene = idx(&j, "scene").unwrap_or(0); + let roots: Vec = match arr(&j, "scenes").get(scene) { + Some(s) => arr(s, "nodes").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(), + None => { + // No scene: every node that is nobody's child. + let mut child = vec![false; nodes.len()]; + for n in &nodes { + for &c in &n.children { + if c < child.len() { + child[c] = true; + } + } + } + (0..nodes.len()).filter(|&i| !child[i]).collect() + } + }; + + let mut skins = vec![]; + for s in arr(&j, "skins") { + let joints: Vec = arr(s, "joints").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(); + let inverse_bind = match idx(s, "inverseBindMatrices") { + Some(a) => { + let (f, _) = d.floats(a)?; + f.chunks_exact(16) + .map(|v| { + let mut m = [[0.0; 4]; 4]; + for c in 0..4 { + for r in 0..4 { + m[r][c] = v[c * 4 + r] as f64; + } + } + Mat4(m) + }) + .collect() + } + None => vec![Mat4::IDENTITY; joints.len()], + }; + skins.push(Skin { joints, inverse_bind }); + } + + let mut animations = vec![]; + for a in arr(&j, "animations") { + let samplers = arr(a, "samplers"); + let mut anim = Animation { name: a.get("name").and_then(Value::as_str).unwrap_or("").to_string(), channels: vec![] }; + for c in arr(a, "channels") { + let Some(t) = c.get("target") else { continue }; + let Some(node) = idx(t, "node") else { continue }; + let path = match t.get("path").and_then(Value::as_str) { + Some("translation") => ChannelPath::Translation, + Some("rotation") => ChannelPath::Rotation, + Some("scale") => ChannelPath::Scale, + _ => continue, + }; + let Some(s) = idx(c, "sampler").and_then(|s| samplers.get(s)) else { continue }; + let interp = match s.get("interpolation").and_then(Value::as_str) { + Some("STEP") => Interp::Step, + Some("CUBICSPLINE") => Interp::CubicSpline, + _ => Interp::Linear, + }; + let (Some(input), Some(output)) = (idx(s, "input"), idx(s, "output")) else { continue }; + let (times, _) = d.floats(input)?; + let (values, _) = d.floats(output)?; + anim.channels.push(Channel { node, path, interp, times, values }); + } + animations.push(anim); + } + + Ok(Model { meshes, nodes, roots, materials, textures, skins, animations }) +} + +fn primitive(d: &Doc, p: &Value, materials: &[Material]) -> Result> { + let mode = idx(p, "mode").unwrap_or(4); + if !matches!(mode, 4..=6) { + return Ok(None); + } + let attrs = p.get("attributes").ok_or_else(|| perr("primitive without attributes"))?; + let Some(pa) = idx(attrs, "POSITION") else { return Ok(None) }; + let (pos, _) = d.floats(pa)?; + let positions: Vec<[f32; 3]> = pos.chunks_exact(3).map(|c| [c[0], c[1], c[2]]).collect(); + let n = positions.len(); + let vec3s = |a: Option| -> Result> { + Ok(match a { + Some(a) => d.floats(a)?.0.chunks_exact(3).map(|c| [c[0], c[1], c[2]]).collect(), + None => vec![], + }) + }; + let mut normals = vec3s(idx(attrs, "NORMAL"))?; + let uvs: Vec<[f32; 2]> = match idx(attrs, "TEXCOORD_0") { + Some(a) => d.floats(a)?.0.chunks_exact(2).map(|c| [c[0], c[1]]).collect(), + None => vec![], + }; + let tangents: Vec<[f32; 4]> = match idx(attrs, "TANGENT") { + Some(a) => d.floats(a)?.0.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]).collect(), + None => vec![], + }; + let joints: Vec<[u16; 4]> = match idx(attrs, "JOINTS_0") { + Some(a) => d.ints(a)?.chunks_exact(4).map(|c| [c[0] as u16, c[1] as u16, c[2] as u16, c[3] as u16]).collect(), + None => vec![], + }; + let weights: Vec<[f32; 4]> = match idx(attrs, "WEIGHTS_0") { + Some(a) => d.floats(a)?.0.chunks_exact(4).map(|c| [c[0], c[1], c[2], c[3]]).collect(), + None => vec![], + }; + let raw: Vec = match idx(p, "indices") { + Some(a) => d.ints(a)?, + None => (0..n as u32).collect(), + }; + if raw.iter().any(|&i| i as usize >= n) { + return Err(perr("index out of range")); + } + let mut indices = Vec::with_capacity(raw.len()); + match mode { + 5 => { + for i in 2..raw.len() { + let (a, b, c) = (raw[i - 2], raw[i - 1], raw[i]); + if i % 2 == 0 { indices.extend([a, b, c]) } else { indices.extend([b, a, c]) } + } + } + 6 => { + for i in 2..raw.len() { + indices.extend([raw[0], raw[i - 1], raw[i]]); + } + } + _ => indices.extend(raw.chunks_exact(3).flatten()), + } + if normals.len() != n { + normals.clear(); + } + let material = idx(p, "material").filter(|&m| m < materials.len()); + let skinned = joints.len() == n && weights.len() == n; + let mut prim = Primitive { + positions, + normals, + uvs: if uvs.len() == n { uvs } else { vec![] }, + tangents: if tangents.len() == n { tangents } else { vec![] }, + joints: if skinned { joints } else { vec![] }, + weights: if skinned { weights } else { vec![] }, + indices, + material, + }; + if prim.normals.is_empty() { + prim.flat_normals(); + prim.tangents.clear(); + } + let needs_tangents = material.is_some_and(|m| materials[m].normal_tex.is_some()); + if needs_tangents && prim.tangents.is_empty() { + prim.compute_tangents(); + } + Ok(Some(prim)) +} diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs new file mode 100644 index 00000000..290e5500 --- /dev/null +++ b/crates/model/src/lib.rs @@ -0,0 +1,691 @@ +//! 3D models for the Advanced 3D renderer. +//! +//! - [`gltf`]: glTF 2.0 import (`.gltf` with external or embedded buffers, `.glb`), written from +//! the Khronos glTF 2.0 specification: meshes, PBR metallic-roughness materials (base colour, +//! metallic-roughness, normal, occlusion and emissive maps), the node hierarchy, skins and +//! animations. +//! - [`obj`]: Wavefront OBJ geometry with MTL materials (mapped onto the metallic-roughness model). +//! - [`prim`]: parametric primitives (cube, sphere, plane, torus, cone, cylinder). +//! - [`extrude`]: extruded, bevelled meshes from 2D outlines (text and shape layers), with +//! [`triangulate`] for the caps. +//! +//! Models are in their own units with +Y up (glTF's convention); the renderer maps them into +//! After Effects' Y-down pixel space. + +pub mod extrude; +pub mod gltf; +pub mod obj; +pub mod prim; +pub mod triangulate; + +use std::sync::Arc; + +pub use effectcraft_geom::{Mat4, Vec3, vec3}; + +#[derive(Debug, thiserror::Error)] +pub enum ModelError { + #[error("{0}")] + Parse(String), + #[error("missing resource `{0}`")] + Missing(String), + #[error("unsupported: {0}")] + Unsupported(String), +} + +pub type Result = std::result::Result; + +pub(crate) fn perr(s: impl Into) -> ModelError { + ModelError::Parse(s.into()) +} + +/// Texture coordinate wrapping (glTF sampler wrap modes). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Wrap { + #[default] + Repeat, + Clamp, + Mirror, +} + +/// A decoded texture: straight RGBA, 0..1, as stored in the file (colour textures are still +/// sRGB-encoded; the renderer linearises them). +#[derive(Clone, Debug, PartialEq)] +pub struct Texture { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, + pub wrap: [Wrap; 2], +} + +impl Texture { + pub fn solid(c: [f32; 4]) -> Texture { + Texture { width: 1, height: 1, data: vec![c], wrap: [Wrap::Repeat; 2] } + } + /// Decode a PNG or JPEG. + pub fn decode(bytes: &[u8]) -> Result { + let img = image::load_from_memory(bytes).map_err(|e| perr(format!("image: {e}")))?.to_rgba32f(); + let (w, h) = (img.width(), img.height()); + let data = img.pixels().map(|p| p.0).collect(); + Ok(Texture { width: w, height: h, data, wrap: [Wrap::Repeat; 2] }) + } +} + +/// How a material's alpha is used (glTF `alphaMode`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AlphaMode { + #[default] + Opaque, + Mask, + Blend, +} + +/// A texture reference of a material. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TexRef { + pub texture: usize, + /// TEXCOORD set (only set 0 is imported; others fall back to set 0). + pub uv_set: u32, +} + +/// A metallic-roughness PBR material (glTF 2.0 core material model). +#[derive(Clone, Debug, PartialEq)] +pub struct Material { + pub name: String, + /// Linear RGBA factor. + pub base_color: [f32; 4], + pub base_color_tex: Option, + pub metallic: f32, + pub roughness: f32, + /// Roughness in G, metalness in B. + pub metallic_roughness_tex: Option, + pub normal_tex: Option, + pub normal_scale: f32, + /// Occlusion in R. + pub occlusion_tex: Option, + pub occlusion_strength: f32, + /// Linear emissive factor (× KHR_materials_emissive_strength). + pub emissive: [f32; 3], + pub emissive_tex: Option, + pub alpha_mode: AlphaMode, + pub alpha_cutoff: f32, + pub double_sided: bool, + /// KHR_materials_unlit. + pub unlit: bool, +} + +impl Default for Material { + fn default() -> Self { + Material { + name: String::new(), + base_color: [1.0; 4], + base_color_tex: None, + metallic: 1.0, + roughness: 1.0, + metallic_roughness_tex: None, + normal_tex: None, + normal_scale: 1.0, + occlusion_tex: None, + occlusion_strength: 1.0, + emissive: [0.0; 3], + emissive_tex: None, + alpha_mode: AlphaMode::Opaque, + alpha_cutoff: 0.5, + double_sided: false, + unlit: false, + } + } +} + +impl Material { + /// A plain dielectric material of a colour (primitives, OBJ without MTL). + pub fn plain(base: [f32; 4], metallic: f32, roughness: f32) -> Material { + Material { base_color: base, metallic, roughness, ..Material::default() } + } +} + +/// Triangles with one material. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Primitive { + pub positions: Vec<[f32; 3]>, + pub normals: Vec<[f32; 3]>, + /// TEXCOORD_0 (empty when the primitive has none). + pub uvs: Vec<[f32; 2]>, + /// Tangents (xyz, w = bitangent sign); empty when not needed. + pub tangents: Vec<[f32; 4]>, + /// JOINTS_0 / WEIGHTS_0 (skinned meshes; empty otherwise). + pub joints: Vec<[u16; 4]>, + pub weights: Vec<[f32; 4]>, + /// Triangle list. + pub indices: Vec, + pub material: Option, +} + +impl Primitive { + pub fn triangle_count(&self) -> usize { + self.indices.len() / 3 + } + /// Smooth vertex normals from the triangles (area weighted), for meshes without normals. + pub fn compute_normals(&mut self) { + let mut n = vec![[0.0f32; 3]; self.positions.len()]; + for t in self.indices.chunks_exact(3) { + let (a, b, c) = (self.positions[t[0] as usize], self.positions[t[1] as usize], self.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + for &i in t { + let v = &mut n[i as usize]; + v[0] += f[0]; + v[1] += f[1]; + v[2] += f[2]; + } + } + self.normals = n.into_iter().map(|v| normalize(v).unwrap_or([0.0, 0.0, 1.0])).collect(); + } + /// Split every triangle into its own vertices with a face normal (glTF: flat normals when a + /// primitive has none). + pub fn flat_normals(&mut self) { + let mut p = Primitive { material: self.material, ..Default::default() }; + for t in self.indices.chunks_exact(3) { + let (a, b, c) = (self.positions[t[0] as usize], self.positions[t[1] as usize], self.positions[t[2] as usize]); + let f = normalize(cross(sub(b, a), sub(c, a))).unwrap_or([0.0, 0.0, 1.0]); + for &i in t { + let i = i as usize; + p.indices.push(p.positions.len() as u32); + p.positions.push(self.positions[i]); + p.normals.push(f); + if !self.uvs.is_empty() { + p.uvs.push(self.uvs[i]); + } + if !self.joints.is_empty() { + p.joints.push(self.joints[i]); + p.weights.push(self.weights[i]); + } + } + } + *self = p; + } + /// Per-vertex tangents from the UV layout (accumulated per triangle, Gram-Schmidt + /// orthogonalised against the normal, handedness in w). + pub fn compute_tangents(&mut self) { + let n = self.positions.len(); + if self.uvs.len() != n || self.normals.len() != n { + return; + } + let mut tan = vec![[0.0f32; 3]; n]; + let mut bit = vec![[0.0f32; 3]; n]; + for t in self.indices.chunks_exact(3) { + let (i0, i1, i2) = (t[0] as usize, t[1] as usize, t[2] as usize); + let (p0, p1, p2) = (self.positions[i0], self.positions[i1], self.positions[i2]); + let (w0, w1, w2) = (self.uvs[i0], self.uvs[i1], self.uvs[i2]); + let e1 = sub(p1, p0); + let e2 = sub(p2, p0); + let (du1, dv1, du2, dv2) = (w1[0] - w0[0], w1[1] - w0[1], w2[0] - w0[0], w2[1] - w0[1]); + let det = du1 * dv2 - du2 * dv1; + if det.abs() < 1e-12 { + continue; + } + let r = 1.0 / det; + let sdir = [(e1[0] * dv2 - e2[0] * dv1) * r, (e1[1] * dv2 - e2[1] * dv1) * r, (e1[2] * dv2 - e2[2] * dv1) * r]; + let tdir = [(e2[0] * du1 - e1[0] * du2) * r, (e2[1] * du1 - e1[1] * du2) * r, (e2[2] * du1 - e1[2] * du2) * r]; + for i in [i0, i1, i2] { + for k in 0..3 { + tan[i][k] += sdir[k]; + bit[i][k] += tdir[k]; + } + } + } + self.tangents = (0..n) + .map(|i| { + let nn = self.normals[i]; + let t = tan[i]; + let d = dot(nn, t); + let o = [t[0] - nn[0] * d, t[1] - nn[1] * d, t[2] - nn[2] * d]; + let o = normalize(o).unwrap_or_else(|| any_perpendicular(nn)); + let w = if dot(cross(nn, o), bit[i]) < 0.0 { -1.0 } else { 1.0 }; + [o[0], o[1], o[2], w] + }) + .collect(); + } + /// Axis-aligned bounds of the positions. + pub fn bounds(&self) -> Option<([f32; 3], [f32; 3])> { + let mut it = self.positions.iter(); + let f = *it.next()?; + let (mut lo, mut hi) = (f, f); + for p in it { + for k in 0..3 { + lo[k] = lo[k].min(p[k]); + hi[k] = hi[k].max(p[k]); + } + } + Some((lo, hi)) + } +} + +pub(crate) fn sub(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub(crate) fn cross(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub(crate) fn dot(a: [f32; 3], b: [f32; 3]) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub(crate) fn normalize(a: [f32; 3]) -> Option<[f32; 3]> { + let l = dot(a, a).sqrt(); + (l > 1e-20 && l.is_finite()).then(|| [a[0] / l, a[1] / l, a[2] / l]) +} +fn any_perpendicular(n: [f32; 3]) -> [f32; 3] { + let a = if n[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + normalize(cross(n, a)).unwrap_or([1.0, 0.0, 0.0]) +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Mesh { + pub name: String, + pub primitives: Vec, +} + +/// A node of the scene hierarchy: local transform (TRS or a matrix), an optional mesh and skin. +#[derive(Clone, Debug, PartialEq)] +pub struct Node { + pub name: String, + pub translation: [f64; 3], + /// Unit quaternion (x, y, z, w). + pub rotation: [f64; 4], + pub scale: [f64; 3], + /// A fixed local matrix (nodes given by `matrix` are not animated). + pub matrix: Option, + pub mesh: Option, + pub skin: Option, + pub children: Vec, +} + +impl Default for Node { + fn default() -> Self { + Node { + name: String::new(), + translation: [0.0; 3], + rotation: [0.0, 0.0, 0.0, 1.0], + scale: [1.0; 3], + matrix: None, + mesh: None, + skin: None, + children: vec![], + } + } +} + +impl Node { + pub fn local(&self) -> Mat4 { + self.matrix.unwrap_or_else(|| trs(self.translation, self.rotation, self.scale)) + } +} + +/// Translation × rotation (quaternion x, y, z, w) × scale. +pub fn trs(t: [f64; 3], q: [f64; 4], s: [f64; 3]) -> Mat4 { + let r = quat_matrix(q); + let mut m = r; + for i in 0..3 { + for j in 0..3 { + m.0[i][j] *= s[j]; + } + m.0[i][3] = t[i]; + } + m +} + +/// Rotation matrix of a quaternion (x, y, z, w); normalised first. +pub fn quat_matrix(q: [f64; 4]) -> Mat4 { + let l = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt(); + let [x, y, z, w] = if l > 1e-12 { [q[0] / l, q[1] / l, q[2] / l, q[3] / l] } else { [0.0, 0.0, 0.0, 1.0] }; + Mat4([ + [1.0 - 2.0 * (y * y + z * z), 2.0 * (x * y - z * w), 2.0 * (x * z + y * w), 0.0], + [2.0 * (x * y + z * w), 1.0 - 2.0 * (x * x + z * z), 2.0 * (y * z - x * w), 0.0], + [2.0 * (x * z - y * w), 2.0 * (y * z + x * w), 1.0 - 2.0 * (x * x + y * y), 0.0], + [0.0, 0.0, 0.0, 1.0], + ]) +} + +/// Spherical linear interpolation of unit quaternions (shortest path). +pub fn slerp(a: [f64; 4], b: [f64; 4], t: f64) -> [f64; 4] { + let mut d = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3]; + let mut b = b; + if d < 0.0 { + d = -d; + b = [-b[0], -b[1], -b[2], -b[3]]; + } + let (ka, kb) = if d > 0.9995 { + (1.0 - t, t) + } else { + let th = d.clamp(-1.0, 1.0).acos(); + let s = th.sin(); + (((1.0 - t) * th).sin() / s, (t * th).sin() / s) + }; + let q = [a[0] * ka + b[0] * kb, a[1] * ka + b[1] * kb, a[2] * ka + b[2] * kb, a[3] * ka + b[3] * kb]; + let l = (q[0] * q[0] + q[1] * q[1] + q[2] * q[2] + q[3] * q[3]).sqrt().max(1e-12); + [q[0] / l, q[1] / l, q[2] / l, q[3] / l] +} + +/// A skin: joint nodes and their inverse bind matrices. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Skin { + pub joints: Vec, + pub inverse_bind: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChannelPath { + Translation, + Rotation, + Scale, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Interp { + Step, + Linear, + CubicSpline, +} + +/// One animated node property. +#[derive(Clone, Debug, PartialEq)] +pub struct Channel { + pub node: usize, + pub path: ChannelPath, + pub interp: Interp, + /// Key times (seconds). + pub times: Vec, + /// Values, `width` floats per key (cubic spline: in-tangent, value, out-tangent per key). + pub values: Vec, +} + +impl Channel { + fn width(&self) -> usize { + match self.path { + ChannelPath::Rotation => 4, + _ => 3, + } + } + /// The value at time `t` (seconds), clamped to the key range. + pub fn sample(&self, t: f64) -> Option<[f64; 4]> { + let n = self.times.len(); + let w = self.width(); + let stride = if self.interp == Interp::CubicSpline { 3 * w } else { w }; + let off = if self.interp == Interp::CubicSpline { w } else { 0 }; + if n == 0 || self.values.len() < n * stride { + return None; + } + let get = |k: usize, o: usize| -> [f64; 4] { + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = self.values[k * stride + o + i] as f64; + } + v + }; + let t0 = self.times[0] as f64; + if t <= t0 || n == 1 { + return Some(get(0, off)); + } + if t >= self.times[n - 1] as f64 { + return Some(get(n - 1, off)); + } + let k = self.times.partition_point(|&x| (x as f64) <= t).saturating_sub(1).min(n - 2); + let (ta, tb) = (self.times[k] as f64, self.times[k + 1] as f64); + let dt = (tb - ta).max(1e-9); + let u = ((t - ta) / dt).clamp(0.0, 1.0); + let (a, b) = (get(k, off), get(k + 1, off)); + let v = match self.interp { + Interp::Step => a, + Interp::Linear => { + if self.path == ChannelPath::Rotation { + slerp(a, b, u) + } else { + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = a[i] + (b[i] - a[i]) * u; + } + v + } + } + Interp::CubicSpline => { + // Hermite spline with tangents scaled by the key interval (glTF Appendix C). + let out_a = get(k, 2 * w); + let in_b = get(k + 1, 0); + let (u2, u3) = (u * u, u * u * u); + let h00 = 2.0 * u3 - 3.0 * u2 + 1.0; + let h10 = u3 - 2.0 * u2 + u; + let h01 = -2.0 * u3 + 3.0 * u2; + let h11 = u3 - u2; + let mut v = [0.0; 4]; + for i in 0..w { + v[i] = h00 * a[i] + h10 * dt * out_a[i] + h01 * b[i] + h11 * dt * in_b[i]; + } + if self.path == ChannelPath::Rotation { + let l = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2] + v[3] * v[3]).sqrt().max(1e-12); + for c in v.iter_mut() { + *c /= l; + } + } + v + } + }; + Some(v) + } +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Animation { + pub name: String, + pub channels: Vec, +} + +impl Animation { + /// Length in seconds (the last key time of any channel). + pub fn duration(&self) -> f64 { + self.channels.iter().filter_map(|c| c.times.last()).fold(0.0f64, |a, &b| a.max(b as f64)) + } +} + +/// An imported model (one glTF scene, one OBJ file, or a primitive). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Model { + pub meshes: Vec, + pub nodes: Vec, + /// Root nodes of the displayed scene. + pub roots: Vec, + pub materials: Vec, + pub textures: Vec>, + pub skins: Vec, + pub animations: Vec, +} + +/// One mesh placed in model space by [`Model::instances`]. +#[derive(Clone, Debug)] +pub struct Instance<'m> { + pub mesh: &'m Mesh, + /// Node → model space. + pub world: Mat4, + /// Skinned meshes: joint matrices (model space, `joint_world × inverse_bind`). + pub joints: Option>, +} + +impl Model { + /// A model of one mesh at the root. + pub fn single(mesh: Mesh, materials: Vec) -> Model { + Model { meshes: vec![mesh], nodes: vec![Node { mesh: Some(0), ..Node::default() }], roots: vec![0], materials, ..Model::default() } + } + + /// Local node matrices at animation `clip` time `t` (seconds); `None` = rest pose. + pub fn local_pose(&self, clip: Option, t: f64) -> Vec { + let mut trs_v: Vec<([f64; 3], [f64; 4], [f64; 3])> = self.nodes.iter().map(|n| (n.translation, n.rotation, n.scale)).collect(); + if let Some(a) = clip.and_then(|c| self.animations.get(c)) { + for ch in &a.channels { + let (Some(v), Some(e)) = (ch.sample(t), trs_v.get_mut(ch.node)) else { continue }; + match ch.path { + ChannelPath::Translation => e.0 = [v[0], v[1], v[2]], + ChannelPath::Rotation => e.1 = v, + ChannelPath::Scale => e.2 = [v[0], v[1], v[2]], + } + } + } + self.nodes.iter().zip(trs_v).map(|(n, (t, r, s))| n.matrix.unwrap_or_else(|| trs(t, r, s))).collect() + } + + /// Model-space matrices of every node (hierarchy applied) for a local pose. + pub fn world_pose(&self, local: &[Mat4]) -> Vec { + let mut world = vec![Mat4::IDENTITY; self.nodes.len()]; + let mut seen = vec![false; self.nodes.len()]; + let mut stack: Vec<(usize, Mat4)> = self.roots.iter().map(|&r| (r, Mat4::IDENTITY)).collect(); + while let Some((i, parent)) = stack.pop() { + if i >= self.nodes.len() || seen[i] { + continue; + } + seen[i] = true; + let w = parent * local[i]; + world[i] = w; + for &c in &self.nodes[i].children { + stack.push((c, w)); + } + } + world + } + + /// The meshes to draw at animation `clip` time `t`. + pub fn instances(&self, clip: Option, t: f64) -> Vec> { + let local = self.local_pose(clip, t); + let world = self.world_pose(&local); + let mut seen = vec![false; self.nodes.len()]; + let mut stack: Vec = self.roots.clone(); + let mut out = vec![]; + while let Some(i) = stack.pop() { + if i >= self.nodes.len() || seen[i] { + continue; + } + seen[i] = true; + let n = &self.nodes[i]; + if let Some(m) = n.mesh.and_then(|m| self.meshes.get(m)) { + let joints = n.skin.and_then(|s| self.skins.get(s)).map(|s| { + s.joints + .iter() + .enumerate() + .map(|(k, &j)| world.get(j).copied().unwrap_or(Mat4::IDENTITY) * s.inverse_bind.get(k).copied().unwrap_or(Mat4::IDENTITY)) + .collect() + }); + // Skinned vertices are placed by their joints alone (glTF: the node transform of + // a skinned mesh is ignored). + let w = if joints.is_some() { Mat4::IDENTITY } else { world[i] }; + out.push(Instance { mesh: m, world: w, joints }); + } + stack.extend(n.children.iter().rev()); + } + out + } + + /// Model-space bounds of the rest pose (None for an empty model). + pub fn bounds(&self) -> Option<([f64; 3], [f64; 3])> { + let mut lo = [f64::INFINITY; 3]; + let mut hi = [f64::NEG_INFINITY; 3]; + for inst in self.instances(None, 0.0) { + for p in &inst.mesh.primitives { + for (k, v) in p.positions.iter().enumerate() { + let q = skin_point(&inst, p, k, *v); + for c in 0..3 { + lo[c] = lo[c].min(q[c]); + hi[c] = hi[c].max(q[c]); + } + } + } + } + lo[0].is_finite().then_some((lo, hi)) + } + + pub fn vertex_count(&self) -> usize { + self.meshes.iter().flat_map(|m| &m.primitives).map(|p| p.positions.len()).sum() + } + pub fn triangle_count(&self) -> usize { + self.meshes.iter().flat_map(|m| &m.primitives).map(|p| p.triangle_count()).sum() + } +} + +/// The skinning (or node) matrix of vertex `k` of an instance's primitive. +pub fn vertex_matrix(inst: &Instance, p: &Primitive, k: usize) -> Mat4 { + let Some(j) = &inst.joints else { return inst.world }; + let (Some(ji), Some(wi)) = (p.joints.get(k), p.weights.get(k)) else { return inst.world }; + let mut m = [[0.0; 4]; 4]; + let mut total = 0.0; + for c in 0..4 { + let w = wi[c] as f64; + if w == 0.0 { + continue; + } + let Some(jm) = j.get(ji[c] as usize) else { continue }; + total += w; + for r in 0..4 { + for q in 0..4 { + m[r][q] += jm.0[r][q] * w; + } + } + } + if total <= 1e-9 { inst.world } else { Mat4(m) } +} + +/// A vertex position of an instance in model space. +pub fn skin_point(inst: &Instance, p: &Primitive, k: usize, v: [f32; 3]) -> [f64; 3] { + let q = vertex_matrix(inst, p, k).apply(vec3(v[0] as f64, v[1] as f64, v[2] as f64)); + [q.x, q.y, q.z] +} + +/// Which loader handles a file name (`gltf`, `glb`, `obj`), by extension. +pub fn format_of(path: &str) -> Option<&'static str> { + let ext = path.rsplit('.').next()?.to_ascii_lowercase(); + match ext.as_str() { + "gltf" => Some("gltf"), + "glb" => Some("glb"), + "obj" => Some("obj"), + _ => None, + } +} + +/// Load a model from its main file's bytes; `resolve(uri)` reads sibling resources (buffers, +/// textures, MTL files) by their relative URI. +pub fn load(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + match format_of(path) { + Some("obj") => obj::parse(bytes, resolve), + Some(_) => gltf::parse(bytes, resolve), + None if bytes.starts_with(b"glTF") => gltf::parse(bytes, resolve), + None => Err(ModelError::Unsupported(format!("{path}: not a glTF or OBJ file"))), + } +} + +/// Summary for footage metadata (Project panel, Interpret Footage). +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize)] +pub struct Summary { + pub meshes: usize, + pub vertices: usize, + pub triangles: usize, + pub materials: usize, + pub textures: usize, + pub animations: Vec<(String, f64)>, + pub skinned: bool, + /// Rest-pose bounds (model units). + pub min: [f64; 3], + pub max: [f64; 3], +} + +impl Model { + pub fn summary(&self) -> Summary { + let (min, max) = self.bounds().unwrap_or(([0.0; 3], [0.0; 3])); + Summary { + meshes: self.meshes.len(), + vertices: self.vertex_count(), + triangles: self.triangle_count(), + materials: self.materials.len(), + textures: self.textures.len(), + animations: self.animations.iter().map(|a| (a.name.clone(), a.duration())).collect(), + skinned: !self.skins.is_empty(), + min, + max, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/model/src/obj.rs b/crates/model/src/obj.rs new file mode 100644 index 00000000..e6727e73 --- /dev/null +++ b/crates/model/src/obj.rs @@ -0,0 +1,262 @@ +//! Wavefront OBJ geometry and MTL materials. +//! +//! OBJ: `v` (optional vertex colours ignored), `vt`, `vn`, `f` with `v`, `v/vt`, `v//vn`, +//! `v/vt/vn` references (negative = relative), polygons fan-triangulated, `o`/`g` groups, `s` +//! smoothing (normals computed per smoothing state when a face has none), `usemtl`, `mtllib`. +//! +//! MTL materials map onto metallic-roughness: `Kd` → base colour, `d`/`Tr` → alpha, `Ke` → +//! emissive, `Pr`/`Pm` (the PBR extension) → roughness/metallic, otherwise roughness from the +//! Phong exponent `Ns` (`sqrt(2 / (Ns + 2))`) and metallic 0; `map_Kd`, `map_Ke`, +//! `map_Bump`/`bump`/`norm` load base colour, emissive and normal textures. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::{AlphaMode, Material, Mesh, Model, Primitive, Result, TexRef, Texture, perr}; + +/// Parse MTL text into named materials (textures appended to `textures`). +pub fn parse_mtl(text: &str, resolve: &dyn Fn(&str) -> Option>, textures: &mut Vec>) -> Vec { + let mut out: Vec = vec![]; + let mut loaded: HashMap> = HashMap::new(); + let mut tex = |name: &str, textures: &mut Vec>| -> Option { + // Options (`-bm 1 file.png`): the file name is the last token. + let file = name.split_whitespace().last()?.to_string(); + let t = *loaded.entry(file.clone()).or_insert_with(|| { + let b = resolve(&file)?; + let t = Texture::decode(&b).ok()?; + textures.push(Arc::new(t)); + Some(textures.len() - 1) + }); + t.map(|texture| TexRef { texture, uv_set: 0 }) + }; + let mut ns_seen = false; + let mut pr_seen = false; + for line in text.lines() { + let line = line.trim(); + let (key, rest) = line.split_once(char::is_whitespace).map(|(k, r)| (k, r.trim())).unwrap_or((line, "")); + let f3 = || -> [f32; 3] { + let v: Vec = rest.split_whitespace().filter_map(|x| x.parse().ok()).collect(); + match v.len() { + 0 => [0.0; 3], + 1 | 2 => [v[0]; 3], + _ => [v[0], v[1], v[2]], + } + }; + let f1 = || rest.split_whitespace().next().and_then(|x| x.parse::().ok()); + if key == "newmtl" { + if let Some(m) = out.last_mut() { + finish(m, ns_seen, pr_seen); + } + ns_seen = false; + pr_seen = false; + out.push(Material { name: rest.to_string(), metallic: 0.0, roughness: 0.5, ..Material::default() }); + continue; + } + let Some(m) = out.last_mut() else { continue }; + match key { + "Kd" => { + let c = f3(); + m.base_color = [srgb_to_linear(c[0]), srgb_to_linear(c[1]), srgb_to_linear(c[2]), m.base_color[3]]; + } + "Ke" => m.emissive = f3().map(srgb_to_linear), + "d" => m.base_color[3] = f1().unwrap_or(1.0), + "Tr" => m.base_color[3] = 1.0 - f1().unwrap_or(0.0), + "Ns" => { + let ns = f1().unwrap_or(0.0).max(0.0); + if !pr_seen { + m.roughness = (2.0 / (ns + 2.0)).sqrt(); + } + ns_seen = true; + } + "Pr" => { + m.roughness = f1().unwrap_or(0.5).clamp(0.0, 1.0); + pr_seen = true; + } + "Pm" => m.metallic = f1().unwrap_or(0.0).clamp(0.0, 1.0), + "map_Kd" => m.base_color_tex = tex(rest, textures), + "map_Ke" => { + m.emissive_tex = tex(rest, textures); + if m.emissive == [0.0; 3] { + m.emissive = [1.0; 3]; + } + } + "map_Bump" | "map_bump" | "bump" | "norm" => m.normal_tex = tex(rest, textures), + _ => {} + } + } + if let Some(m) = out.last_mut() { + finish(m, ns_seen, pr_seen); + } + out +} + +fn finish(m: &mut Material, _ns: bool, _pr: bool) { + if m.base_color[3] < 0.999 { + m.alpha_mode = AlphaMode::Blend; + } +} + +/// sRGB-encoded component → linear (MTL colours are display colours). +pub fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct Key { + v: u32, + t: u32, + n: u32, + /// Smoothing state (faces without normals get computed normals per state). + s: u32, +} + +/// Parse OBJ text (and its MTL libraries through `resolve`). +pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result { + let text = String::from_utf8_lossy(bytes); + let mut v: Vec<[f32; 3]> = vec![]; + let mut vt: Vec<[f32; 2]> = vec![]; + let mut vn: Vec<[f32; 3]> = vec![]; + let mut materials: Vec = vec![]; + let mut textures: Vec> = vec![]; + // One primitive per material, in order of first use. + let mut prims: Vec<(Option, Primitive, HashMap, Vec)> = vec![]; + let mut current: Option = None; + let mut smooth = 0u32; + let mut flat_id = 1u32 << 31; + let mut faces = 0usize; + // Join `\` line continuations. + let joined = text.replace("\\\r\n", " ").replace("\\\n", " "); + for (ln, line) in joined.lines().enumerate() { + let line = line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let mut it = line.split_whitespace(); + let key = it.next().unwrap_or(""); + let rest: Vec<&str> = it.collect(); + let fl = |i: usize| rest.get(i).and_then(|x| x.parse::().ok()); + match key { + "v" => v.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0), fl(2).unwrap_or(0.0)]), + "vt" => vt.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0)]), + "vn" => vn.push([fl(0).unwrap_or(0.0), fl(1).unwrap_or(0.0), fl(2).unwrap_or(1.0)]), + "s" => { + smooth = match rest.first().copied() { + None | Some("off") | Some("0") => 0, + Some(x) => x.parse().unwrap_or(1).max(1), + } + } + "mtllib" => { + let name = line["mtllib".len()..].trim(); + for file in std::iter::once(name).chain(name.split_whitespace()) { + if let Some(b) = resolve(file) { + let mut ms = parse_mtl(&String::from_utf8_lossy(&b), resolve, &mut textures); + materials.append(&mut ms); + break; + } + } + } + "usemtl" => { + let name = line["usemtl".len()..].trim(); + current = materials.iter().position(|m| m.name == name); + if current.is_none() { + // Unknown material: a named grey default. + materials.push(Material { name: name.to_string(), metallic: 0.0, roughness: 0.5, base_color: [0.8, 0.8, 0.8, 1.0], ..Material::default() }); + current = Some(materials.len() - 1); + } + } + "f" => { + if rest.len() < 3 { + continue; + } + faces += 1; + let resolve_i = |s: &str, len: usize| -> Option { + if s.is_empty() { + return None; + } + let i: i64 = s.parse().ok()?; + let k = if i < 0 { len as i64 + i } else { i - 1 }; + (k >= 0 && (k as usize) < len).then_some(k as u32) + }; + let mut refs = vec![]; + for r in &rest { + let mut parts = r.split('/'); + let vi = resolve_i(parts.next().unwrap_or(""), v.len()).ok_or_else(|| perr(format!("line {}: bad vertex index `{r}`", ln + 1)))?; + let ti = parts.next().and_then(|s| resolve_i(s, vt.len())); + let ni = parts.next().and_then(|s| resolve_i(s, vn.len())); + refs.push((vi, ti, ni)); + } + let has_n = refs.iter().all(|r| r.2.is_some()); + // Faces without normals: shared within a smoothing group, unique otherwise. + let s = if has_n { + 0 + } else if smooth == 0 { + flat_id += 1; + flat_id + } else { + smooth + }; + let slot = match prims.iter().position(|p| p.0 == current) { + Some(i) => i, + None => { + prims.push((current, Primitive { material: current, ..Default::default() }, HashMap::new(), vec![])); + prims.len() - 1 + } + }; + let (_, prim, map, need) = &mut prims[slot]; + let mut ids = vec![]; + for (vi, ti, ni) in refs { + let k = Key { v: vi, t: ti.map_or(u32::MAX, |x| x), n: ni.map_or(u32::MAX, |x| x), s }; + let id = *map.entry(k).or_insert_with(|| { + prim.positions.push(v[vi as usize]); + prim.uvs.push(ti.map_or([0.0, 0.0], |t| vt[t as usize])); + prim.normals.push(ni.map_or([0.0; 3], |n| vn[n as usize])); + need.push(ni.is_none()); + (prim.positions.len() - 1) as u32 + }); + ids.push(id); + } + for i in 2..ids.len() { + prim.indices.extend([ids[0], ids[i - 1], ids[i]]); + } + } + _ => {} + } + } + if faces == 0 { + return Err(perr("OBJ file without faces")); + } + let any_uv = !vt.is_empty(); + let mut mesh = Mesh { name: String::new(), primitives: vec![] }; + for (_, mut p, _, need) in prims { + if need.iter().any(|&b| b) { + // Fill missing normals from the triangles (area weighted), keep given ones. + let given = p.normals.clone(); + p.compute_normals(); + for (i, n) in p.normals.iter_mut().enumerate() { + if !need[i] { + *n = given[i]; + } + } + } + if !any_uv { + p.uvs.clear(); + } else { + // OBJ's v axis points up the image; textures are stored top row first. + for uv in p.uvs.iter_mut() { + uv[1] = 1.0 - uv[1]; + } + } + if p.material.is_some_and(|m| materials[m].normal_tex.is_some()) { + p.compute_tangents(); + } + mesh.primitives.push(p); + } + Ok(Model::single(mesh, materials).with_textures(textures)) +} + +impl Model { + fn with_textures(mut self, t: Vec>) -> Model { + self.textures = t; + self + } +} diff --git a/crates/model/src/prim.rs b/crates/model/src/prim.rs new file mode 100644 index 00000000..f04873df --- /dev/null +++ b/crates/model/src/prim.rs @@ -0,0 +1,235 @@ +//! Parametric primitives (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder). +//! +//! Meshes are built in model space (+Y up, counter-clockwise front faces, like glTF), centred +//! on the origin, in pixels, with normals, UVs (0..1 across each face/around each surface) and +//! tangents. + +use std::f32::consts::{PI, TAU}; + +use serde::{Deserialize, Serialize}; + +use crate::{Material, Mesh, Model, Primitive}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum PrimitiveKind { + #[default] + Cube, + Sphere, + Plane, + Torus, + Cone, + Cylinder, +} + +impl PrimitiveKind { + pub const ALL: [PrimitiveKind; 6] = + [PrimitiveKind::Cube, PrimitiveKind::Sphere, PrimitiveKind::Plane, PrimitiveKind::Torus, PrimitiveKind::Cone, PrimitiveKind::Cylinder]; + pub fn label(self) -> &'static str { + match self { + PrimitiveKind::Cube => "Cube", + PrimitiveKind::Sphere => "Sphere", + PrimitiveKind::Plane => "Plane", + PrimitiveKind::Torus => "Torus", + PrimitiveKind::Cone => "Cone", + PrimitiveKind::Cylinder => "Cylinder", + } + } + pub fn parse(s: &str) -> Option { + PrimitiveKind::ALL.into_iter().find(|k| k.label().eq_ignore_ascii_case(s.trim())) + } +} + +/// Primitive dimensions (pixels) and tessellation. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PrimParams { + /// Width, height, depth (cube, plane: width × height). + pub size: [f32; 3], + /// Sphere/cone/cylinder radius; torus ring radius. + pub radius: f32, + /// Torus tube radius. + pub tube: f32, + /// Cone/cylinder height. + pub height: f32, + /// Segments around. + pub segments: u32, + /// Sphere rings / torus tube sides. + pub rings: u32, +} + +impl Default for PrimParams { + fn default() -> Self { + PrimParams { size: [300.0; 3], radius: 150.0, tube: 50.0, height: 300.0, segments: 48, rings: 24 } + } +} + +struct B { + p: Primitive, +} + +impl B { + fn new() -> B { + B { p: Primitive { material: Some(0), ..Default::default() } } + } + fn v(&mut self, pos: [f32; 3], n: [f32; 3], uv: [f32; 2]) -> u32 { + self.p.positions.push(pos); + self.p.normals.push(n); + self.p.uvs.push(uv); + (self.p.positions.len() - 1) as u32 + } + fn tri(&mut self, a: u32, b: u32, c: u32) { + self.p.indices.extend([a, b, c]); + } + fn quad(&mut self, a: u32, b: u32, c: u32, d: u32) { + // a-b-c-d counter-clockwise. + self.tri(a, b, c); + self.tri(a, c, d); + } +} + +/// The mesh of a primitive. +pub fn mesh(kind: PrimitiveKind, pr: &PrimParams) -> Primitive { + let seg = pr.segments.clamp(3, 512); + let rings = pr.rings.clamp(2, 512); + let mut b = B::new(); + match kind { + PrimitiveKind::Cube => { + let [w, h, d] = pr.size.map(|v| v.abs() * 0.5); + // (normal, u axis, v axis) per face; v runs down the texture. + let faces: [([f32; 3], [f32; 3], [f32; 3]); 6] = [ + ([0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, -1.0, 0.0]), + ([0.0, 0.0, -1.0], [-1.0, 0.0, 0.0], [0.0, -1.0, 0.0]), + ([1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, -1.0, 0.0]), + ([-1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, -1.0, 0.0]), + ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, -1.0]), + ]; + let half = [w, h, d]; + for (n, u, v) in faces { + let c = |su: f32, sv: f32| -> [f32; 3] { [0, 1, 2].map(|k| (n[k] + u[k] * su + v[k] * sv) * half[k]) }; + let i0 = b.v(c(-1.0, -1.0), n, [0.0, 0.0]); + let i1 = b.v(c(1.0, -1.0), n, [1.0, 0.0]); + let i2 = b.v(c(1.0, 1.0), n, [1.0, 1.0]); + let i3 = b.v(c(-1.0, 1.0), n, [0.0, 1.0]); + // u × v points along -n for these axes (v down): wind so the face is CCW from outside. + b.quad(i0, i3, i2, i1); + } + } + PrimitiveKind::Plane => { + let (w, h) = (pr.size[0].abs() * 0.5, pr.size[1].abs() * 0.5); + let n = [0.0, 0.0, 1.0]; + let i0 = b.v([-w, h, 0.0], n, [0.0, 0.0]); + let i1 = b.v([w, h, 0.0], n, [1.0, 0.0]); + let i2 = b.v([w, -h, 0.0], n, [1.0, 1.0]); + let i3 = b.v([-w, -h, 0.0], n, [0.0, 1.0]); + b.quad(i0, i3, i2, i1); + } + PrimitiveKind::Sphere => { + let r = pr.radius.abs(); + for j in 0..=rings { + let th = PI * j as f32 / rings as f32; + // Exact poles and seam (welded positions). + let (st, ct) = if j == 0 { + (0.0, 1.0) + } else if j == rings { + (0.0, -1.0) + } else { + th.sin_cos() + }; + for i in 0..=seg { + let ph = TAU * (i % seg) as f32 / seg as f32; + let n = [st * ph.sin(), ct, st * ph.cos()]; + b.v([n[0] * r, n[1] * r, n[2] * r], n, [i as f32 / seg as f32, j as f32 / rings as f32]); + } + } + let row = seg + 1; + for j in 0..rings { + for i in 0..seg { + let a = j * row + i; + let (b0, c, d) = (a + row, a + row + 1, a + 1); + if j != 0 { + b.tri(a, b0, d); + } + if j != rings - 1 { + b.tri(d, b0, c); + } + } + } + } + PrimitiveKind::Torus => { + let (rr, tr) = (pr.radius.abs(), pr.tube.abs()); + for j in 0..=rings { + let v = TAU * (j % rings) as f32 / rings as f32; + for i in 0..=seg { + let u = TAU * (i % seg) as f32 / seg as f32; + // Ring in the XZ plane. + let (cu, su, cv, sv) = (u.cos(), u.sin(), v.cos(), v.sin()); + let n = [cv * su, sv, cv * cu]; + let p = [(rr + tr * cv) * su, tr * sv, (rr + tr * cv) * cu]; + b.v(p, n, [i as f32 / seg as f32, j as f32 / rings as f32]); + } + } + let row = seg + 1; + for j in 0..rings { + for i in 0..seg { + let a = j * row + i; + b.quad(a, a + 1, a + row + 1, a + row); + } + } + } + PrimitiveKind::Cylinder | PrimitiveKind::Cone => { + let r = pr.radius.abs(); + let h = pr.height.abs() * 0.5; + let top_r = if kind == PrimitiveKind::Cone { 0.0 } else { r }; + // Side normal tilt for the cone (slope). + let (ny, nk) = { + let s = (r - top_r) / (2.0 * h).max(1e-6); + let l = (1.0 + s * s).sqrt(); + (s / l, 1.0 / l) + }; + let base = b.p.positions.len() as u32; + for i in 0..=seg { + let a = TAU * (i % seg) as f32 / seg as f32; + let (s, c) = a.sin_cos(); + let n = [s * nk, ny, c * nk]; + let u = i as f32 / seg as f32; + b.v([s * top_r, h, c * top_r], n, [u, 0.0]); + b.v([s * r, -h, c * r], n, [u, 1.0]); + } + for i in 0..seg { + let t0 = base + 2 * i; + let (b0, t1, b1) = (t0 + 1, t0 + 2, t0 + 3); + if kind == PrimitiveKind::Cone { + b.tri(t0, b0, b1); + } else { + b.quad(t0, b0, b1, t1); + } + } + let caps: &[(f32, f32)] = if kind == PrimitiveKind::Cone { &[(-1.0, r)] } else { &[(1.0, r), (-1.0, r)] }; + for &(side, rad) in caps { + let n = [0.0, side, 0.0]; + let c = b.v([0.0, h * side, 0.0], n, [0.5, 0.5]); + let first = b.p.positions.len() as u32; + for i in 0..=seg { + let a = TAU * (i % seg) as f32 / seg as f32; + let (s, co) = a.sin_cos(); + b.v([s * rad, h * side, co * rad], n, [0.5 + 0.5 * s, 0.5 - 0.5 * co * side]); + } + for i in 0..seg { + let (p0, p1) = (first + i, first + i + 1); + if side > 0.0 { b.tri(c, p0, p1) } else { b.tri(c, p1, p0) } + } + } + } + } + b.p.compute_tangents(); + b.p +} + +/// A primitive as a one-mesh model with one material. +pub fn model(kind: PrimitiveKind, pr: &PrimParams, material: Material) -> Model { + let mut material = material; + if kind == PrimitiveKind::Plane { + material.double_sided = true; + } + Model::single(Mesh { name: kind.label().to_string(), primitives: vec![mesh(kind, pr)] }, vec![material]) +} diff --git a/crates/model/src/tests.rs b/crates/model/src/tests.rs new file mode 100644 index 00000000..31040b7c --- /dev/null +++ b/crates/model/src/tests.rs @@ -0,0 +1,318 @@ +use std::collections::HashMap; + +use super::extrude::{BevelStyle, ExtrudeParams, Outline, extrude, group_contours}; +use super::prim::{PrimParams, PrimitiveKind, mesh}; +use super::triangulate::{signed_area, triangulate}; +use super::*; + +fn fixture(name: &str) -> Vec { + std::fs::read(format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn resolver(name: &str) -> Option> { + std::fs::read(format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR"))).ok() +} + +fn check_quad(m: &Model) { + assert_eq!(m.nodes.len(), 2); + assert_eq!(m.roots, vec![0]); + assert_eq!(m.nodes[0].children, vec![1]); + assert_eq!(m.meshes.len(), 1); + let p = &m.meshes[0].primitives[0]; + assert_eq!(p.positions.len(), 4); + assert_eq!(p.indices, vec![0, 1, 2, 0, 2, 3]); + assert_eq!(p.uvs[0], [0.0, 1.0]); + assert_eq!(p.normals[2], [0.0, 0.0, 1.0]); + let mat = &m.materials[0]; + assert_eq!(mat.name, "Painted"); + assert_eq!(mat.base_color, [1.0, 0.5, 0.25, 1.0]); + assert!((mat.metallic - 0.25).abs() < 1e-6 && (mat.roughness - 0.75).abs() < 1e-6); + assert_eq!(mat.emissive, [0.1, 0.2, 0.3]); + assert!(mat.double_sided); + let t = &m.textures[mat.base_color_tex.unwrap().texture]; + assert_eq!((t.width, t.height), (2, 2)); + assert_eq!(t.data[0], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(t.data[3], [1.0, 1.0, 1.0, 1.0]); + assert_eq!(t.wrap, [Wrap::Clamp, Wrap::Repeat]); + // Hierarchy: scale 2 under a +1 Y translation. + let (lo, hi) = m.bounds().unwrap(); + assert!((lo[0] + 1.0).abs() < 1e-9 && (hi[0] - 1.0).abs() < 1e-9); + assert!((lo[1] - 0.0).abs() < 1e-9 && (hi[1] - 2.0).abs() < 1e-9); + // Animation: root translation (0,1,0) → (2,1,0) over 1 s. + assert_eq!(m.animations.len(), 1); + assert_eq!(m.animations[0].name, "Move"); + assert!((m.animations[0].duration() - 1.0).abs() < 1e-9); + let inst = m.instances(Some(0), 0.5); + assert_eq!(inst.len(), 1); + let c = inst[0].world.apply(vec3(0.0, 0.0, 0.0)); + assert!((c.x - 1.0).abs() < 1e-6 && (c.y - 1.0).abs() < 1e-6, "{c:?}"); +} + +#[test] +fn gltf_with_data_uris() { + let m = gltf::parse(&fixture("quad.gltf"), &|_| None).unwrap(); + check_quad(&m); +} + +#[test] +fn glb_matches_gltf() { + let m = load("quad.glb", &fixture("quad.glb"), &resolver).unwrap(); + check_quad(&m); + let g = gltf::parse(&fixture("quad.gltf"), &|_| None).unwrap(); + assert_eq!(m.meshes, g.meshes); + assert_eq!(m.materials, g.materials); +} + +#[test] +fn gltf_errors_are_reported() { + assert!(gltf::parse(b"{\"asset\":{\"version\":\"1.0\"}}", &|_| None).is_err()); + assert!(gltf::parse(b"not json", &|_| None).is_err()); + let missing = br#"{"asset":{"version":"2.0"},"buffers":[{"uri":"nowhere.bin","byteLength":4}]}"#; + assert!(matches!(gltf::parse(missing, &|_| None), Err(ModelError::Missing(_)))); + assert!(gltf::split_glb(b"glTF\x01\0\0\0").is_err()); +} + +#[test] +fn base64_and_uri_decoding() { + assert_eq!(gltf::base64_decode("aGVsbG8=").unwrap(), b"hello"); + assert_eq!(gltf::base64_decode("aGVsbG8").unwrap(), b"hello"); + assert_eq!(gltf::uri_decode("my%20file.bin"), "my file.bin"); +} + +#[test] +fn obj_with_mtl() { + let m = load("cube.obj", &fixture("cube.obj"), &resolver).unwrap(); + assert_eq!(m.materials.len(), 2); + let red = &m.materials[0]; + assert_eq!(red.name, "Red"); + assert_eq!(red.base_color, [1.0, 0.0, 0.0, 1.0]); + assert!((red.roughness - (2.0f32 / 100.0).sqrt()).abs() < 1e-6); + assert_eq!(red.metallic, 0.0); + let gold = &m.materials[1]; + assert_eq!((gold.metallic, gold.roughness), (1.0, 0.3)); + let prims = &m.meshes[0].primitives; + assert_eq!(prims.len(), 2); + assert_eq!(prims[0].material, Some(0)); + assert_eq!(prims.iter().map(|p| p.triangle_count()).sum::(), 12); + // Four unique (v, vt, vn) corners per face. + assert_eq!(m.vertex_count(), 24); + // Every face is wound counter-clockwise from outside (face normal agrees with vn). + for p in prims { + for t in p.indices.chunks(3) { + let (a, b, c) = (p.positions[t[0] as usize], p.positions[t[1] as usize], p.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + assert!(dot(f, p.normals[t[0] as usize]) > 0.0); + } + } + let (lo, hi) = m.bounds().unwrap(); + assert_eq!((lo, hi), ([-0.5; 3], [0.5; 3])); + // V is flipped to the image's top-down rows. + assert_eq!(prims[0].uvs[0], [0.0, 1.0]); +} + +#[test] +fn obj_without_normals_gets_them() { + let src = b"v 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n"; + let m = obj::parse(src, &|_| None).unwrap(); + let p = &m.meshes[0].primitives[0]; + assert_eq!(p.normals, vec![[0.0, 0.0, 1.0]; 3]); + assert!(obj::parse(b"v 0 0 0\n", &|_| None).is_err()); +} + +#[test] +fn channel_interpolation_modes() { + let mut c = Channel { node: 0, path: ChannelPath::Translation, interp: Interp::Step, times: vec![0.0, 1.0], values: vec![0.0, 0.0, 0.0, 10.0, 0.0, 0.0] }; + assert_eq!(c.sample(0.7).unwrap()[0], 0.0); + c.interp = Interp::Linear; + assert!((c.sample(0.7).unwrap()[0] - 7.0).abs() < 1e-9); + assert_eq!(c.sample(5.0).unwrap()[0], 10.0); + // Cubic spline with zero tangents: smoothstep. + c.interp = Interp::CubicSpline; + c.values = vec![0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 10.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + assert!((c.sample(0.5).unwrap()[0] - 5.0).abs() < 1e-9); + assert!((c.sample(0.25).unwrap()[0] - 10.0 * (3.0 * 0.0625 - 2.0 * 0.015625)).abs() < 1e-9); + // Rotation: slerp keeps unit length, half way around Z. + let r = + Channel { node: 0, path: ChannelPath::Rotation, interp: Interp::Linear, times: vec![0.0, 1.0], values: vec![0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0] }; + let q = r.sample(0.5).unwrap(); + assert!((q[2] - (0.5f64).sqrt()).abs() < 1e-9 && (q[3] - (0.5f64).sqrt()).abs() < 1e-9); +} + +#[test] +fn skinning_moves_vertices_with_joints() { + let mut p = mesh(PrimitiveKind::Plane, &PrimParams { size: [2.0, 2.0, 0.0], ..Default::default() }); + p.joints = vec![[0, 0, 0, 0], [1, 0, 0, 0], [1, 0, 0, 0], [0, 0, 0, 0]]; + p.weights = vec![[1.0, 0.0, 0.0, 0.0]; 4]; + let mut m = Model::single(Mesh { name: "s".into(), primitives: vec![p] }, vec![Material::default()]); + m.nodes[0].skin = Some(0); + m.nodes.push(Node::default()); + m.nodes.push(Node { translation: [0.0, 0.0, 5.0], ..Node::default() }); + m.roots = vec![0, 1, 2]; + m.skins.push(Skin { joints: vec![1, 2], inverse_bind: vec![Mat4::IDENTITY; 2] }); + let inst = m.instances(None, 0.0); + let pr = &inst[0].mesh.primitives[0]; + assert_eq!(skin_point(&inst[0], pr, 0, pr.positions[0])[2], 0.0); + assert_eq!(skin_point(&inst[0], pr, 1, pr.positions[1])[2], 5.0); +} + +/// Edges of a mesh welded by position: (undirected edge → uses, directed edge → uses). +fn edge_stats(p: &Primitive) -> (HashMap<(u32, u32), usize>, HashMap<(u32, u32), usize>) { + let mut weld: HashMap<[u32; 3], u32> = HashMap::new(); + let ids: Vec = p + .positions + .iter() + .map(|q| { + let k = q.map(|v| (v + 0.0).to_bits()); + let n = weld.len() as u32; + *weld.entry(k).or_insert(n) + }) + .collect(); + let mut und = HashMap::new(); + let mut dir = HashMap::new(); + for t in p.indices.chunks(3) { + let v = [ids[t[0] as usize], ids[t[1] as usize], ids[t[2] as usize]]; + if v[0] == v[1] || v[1] == v[2] || v[0] == v[2] { + continue; + } + for k in 0..3 { + let (a, b) = (v[k], v[(k + 1) % 3]); + *und.entry((a.min(b), a.max(b))).or_insert(0) += 1; + *dir.entry((a, b)).or_insert(0) += 1; + } + } + (und, dir) +} + +fn assert_watertight(p: &Primitive) { + let (und, dir) = edge_stats(p); + for (e, n) in &und { + assert_eq!(*n, 2, "edge {e:?} used {n} times"); + } + for (e, n) in &dir { + assert_eq!(*n, 1, "directed edge {e:?} used {n} times (inconsistent winding)"); + } +} + +/// Signed volume (positive when faces wind outward). +fn volume(p: &Primitive) -> f64 { + p.indices + .chunks(3) + .map(|t| { + let [a, b, c] = [0, 1, 2].map(|k| p.positions[t[k] as usize].map(|v| v as f64)); + (a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0])) / 6.0 + }) + .sum() +} + +/// Vertex normals agree with the faces they belong to. +fn assert_normals_outward(p: &Primitive) { + for t in p.indices.chunks(3) { + let (a, b, c) = (p.positions[t[0] as usize], p.positions[t[1] as usize], p.positions[t[2] as usize]); + let f = cross(sub(b, a), sub(c, a)); + if dot(f, f) < 1e-10 { + continue; + } + for &i in t { + let n = p.normals[i as usize]; + assert!((dot(n, n) - 1.0).abs() < 1e-3, "normal not unit"); + assert!(dot(f, n) > 0.0, "normal {n:?} faces away from triangle {f:?}"); + } + } +} + +#[test] +fn primitive_vertex_counts_and_normals() { + let pr = PrimParams { segments: 16, rings: 8, ..Default::default() }; + let cases = [ + (PrimitiveKind::Cube, 24, 36), + (PrimitiveKind::Plane, 4, 6), + (PrimitiveKind::Sphere, 17 * 9, 6 * 16 * 7), + (PrimitiveKind::Torus, 17 * 9, 6 * 16 * 8), + (PrimitiveKind::Cylinder, 2 * 17 + 2 * 18, 6 * 16 + 2 * 3 * 16), + (PrimitiveKind::Cone, 2 * 17 + 18, 3 * 16 + 3 * 16), + ]; + for (k, v, i) in cases { + let m = mesh(k, &pr); + assert_eq!((m.positions.len(), m.indices.len()), (v, i), "{k:?}"); + assert_eq!(m.normals.len(), v); + assert_eq!(m.uvs.len(), v); + assert_eq!(m.tangents.len(), v); + assert_normals_outward(&m); + if k != PrimitiveKind::Plane { + assert!(volume(&m) > 0.0, "{k:?} winds inward"); + assert_watertight(&m); + } + } + // Volumes approach the analytic ones. + let cube = mesh(PrimitiveKind::Cube, &PrimParams { size: [2.0, 3.0, 4.0], ..Default::default() }); + assert!((volume(&cube) - 24.0).abs() < 1e-4); + let s = mesh(PrimitiveKind::Sphere, &PrimParams { radius: 1.0, segments: 128, rings: 64, ..Default::default() }); + assert!((volume(&s) - 4.0 / 3.0 * std::f64::consts::PI).abs() < 0.01); +} + +#[test] +fn triangulation_covers_polygon_with_hole() { + let outer = vec![[0.0, 0.0], [10.0, 0.0], [10.0, 10.0], [5.0, 12.0], [0.0, 10.0]]; + let hole = vec![[3.0, 3.0], [3.0, 7.0], [7.0, 7.0], [7.0, 3.0]]; + let pts: Vec<[f64; 2]> = outer.iter().chain(hole.iter()).copied().collect(); + let tris = triangulate(&outer, std::slice::from_ref(&hole)); + let area: f64 = tris.iter().map(|t| signed_area(&[pts[t[0]], pts[t[1]], pts[t[2]]]) * 0.5).sum(); + let expect = signed_area(&outer).abs() * 0.5 - 16.0; + assert!((area - expect).abs() < 1e-9, "{area} vs {expect}"); + assert!(tris.iter().all(|t| signed_area(&[pts[t[0]], pts[t[1]], pts[t[2]]]) >= 0.0)); + // A concave "C". + let c = vec![[0.0, 0.0], [6.0, 0.0], [6.0, 2.0], [2.0, 2.0], [2.0, 4.0], [6.0, 4.0], [6.0, 6.0], [0.0, 6.0]]; + let tris = triangulate(&c, &[]); + let area: f64 = tris.iter().map(|t| signed_area(&[c[t[0]], c[t[1]], c[t[2]]]) * 0.5).sum(); + assert!((area - 28.0).abs() < 1e-9); +} + +fn ring_outline() -> Outline { + let circle = + |r: f64, n: usize| -> Vec<[f64; 2]> { (0..n).map(|i| (i as f64 / n as f64) * std::f64::consts::TAU).map(|a| [r * a.cos(), r * a.sin()]).collect() }; + Outline { outer: circle(100.0, 48), holes: vec![circle(50.0, 32)] } +} + +#[test] +fn extrusion_is_watertight_with_outward_normals() { + let square = Outline { outer: vec![[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]], holes: vec![] }; + for bevel in BevelStyle::ALL { + for o in [square.clone(), ring_outline()] { + let p = ExtrudeParams { bevel, bevel_depth: 8.0, hole_bevel: 0.5, depth: 40.0 }; + let m = extrude(std::slice::from_ref(&o), &p); + assert_watertight(&m); + assert_normals_outward(&m); + assert!(volume(&m) > 0.0, "{bevel:?}: inward winding"); + let (lo, hi) = m.bounds().unwrap(); + assert!(lo[2].abs() < 1e-6 && (hi[2] - 40.0).abs() < 1e-4); + } + } + // Without a bevel the square block's volume is exact. + let m = extrude(&[square], &ExtrudeParams { bevel: BevelStyle::None, bevel_depth: 0.0, hole_bevel: 1.0, depth: 10.0 }); + assert!((volume(&m) - 100_000.0).abs() < 1e-2); + // Front cap faces the camera (−z), back cap away. + assert!(m.normals.iter().any(|n| n[2] < -0.99) && m.normals.iter().any(|n| n[2] > 0.99)); +} + +#[test] +fn bevels_shrink_the_caps() { + let square = Outline { outer: vec![[0.0, 0.0], [100.0, 0.0], [100.0, 100.0], [0.0, 100.0]], holes: vec![] }; + let m = extrude(std::slice::from_ref(&square), &ExtrudeParams { bevel: BevelStyle::Angular, bevel_depth: 10.0, hole_bevel: 1.0, depth: 50.0 }); + // Front cap vertices (z = 0) are inset by the bevel. + let front: Vec<_> = m.positions.iter().filter(|p| p[2] == 0.0).collect(); + assert!(front.iter().all(|p| p[0] >= 9.999 && p[0] <= 90.001)); + // The widest ring (the walls) still reaches the outline. + assert!(m.positions.iter().any(|p| p[0] == 0.0 && p[2] == 10.0)); + // Convex bevels have more profile steps than angular ones. + let c = extrude(std::slice::from_ref(&square), &ExtrudeParams { bevel: BevelStyle::Convex, bevel_depth: 10.0, hole_bevel: 1.0, depth: 50.0 }); + assert!(c.positions.len() > m.positions.len()); +} + +#[test] +fn contours_group_into_outlines_by_nesting() { + let sq = |x: f64, y: f64, s: f64| vec![[x, y], [x + s, y], [x + s, y + s], [x, y + s]]; + // "O" (outer + hole) next to an island inside another hole. + let g = group_contours(vec![sq(0.0, 0.0, 10.0), sq(2.0, 2.0, 6.0), sq(20.0, 0.0, 10.0), sq(22.0, 2.0, 6.0), sq(24.0, 4.0, 2.0)]); + assert_eq!(g.len(), 3); + assert_eq!(g.iter().map(|o| o.holes.len()).sum::(), 2); +} diff --git a/crates/model/src/triangulate.rs b/crates/model/src/triangulate.rs new file mode 100644 index 00000000..7505f5f9 --- /dev/null +++ b/crates/model/src/triangulate.rs @@ -0,0 +1,187 @@ +//! Polygon triangulation with holes: holes are bridged into the outer contour (each hole's +//! rightmost vertex joined to a visible outer vertex), then ears are clipped. + +type P = [f64; 2]; + +/// Twice the signed area (positive = counter-clockwise in a Y-up frame). +pub fn signed_area(c: &[P]) -> f64 { + let n = c.len(); + (0..n) + .map(|i| { + let (a, b) = (c[i], c[(i + 1) % n]); + a[0] * b[1] - b[0] * a[1] + }) + .sum() +} + +#[inline] +fn cross(o: P, a: P, b: P) -> f64 { + (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) +} + +fn in_triangle(p: P, a: P, b: P, c: P) -> bool { + let d1 = cross(a, b, p); + let d2 = cross(b, c, p); + let d3 = cross(c, a, p); + d1 >= 0.0 && d2 >= 0.0 && d3 >= 0.0 +} + +/// Triangulate an outer contour with holes. Contours are closed polylines (last point not +/// repeated); orientation is normalised internally. Returns triangles as indices into the +/// contours concatenated in order (`outer`, then `holes`), counter-clockwise (Y up). +pub fn triangulate(outer: &[P], holes: &[Vec

]) -> Vec<[usize; 3]> { + if outer.len() < 3 { + return vec![]; + } + let mut pts: Vec

= outer.to_vec(); + for h in holes { + pts.extend_from_slice(h); + } + // Ring of global indices, outer counter-clockwise. + let mut ring: Vec = (0..outer.len()).collect(); + if signed_area(outer) < 0.0 { + ring.reverse(); + } + // Holes clockwise, rightmost first. + let mut hs: Vec> = vec![]; + let mut off = outer.len(); + for h in holes { + let mut r: Vec = (off..off + h.len()).collect(); + off += h.len(); + if h.len() < 3 { + continue; + } + if signed_area(h) > 0.0 { + r.reverse(); + } + hs.push(r); + } + hs.sort_by(|a, b| { + let mx = |r: &Vec| r.iter().map(|&i| pts[i][0]).fold(f64::NEG_INFINITY, f64::max); + mx(b).total_cmp(&mx(a)) + }); + for h in hs { + bridge(&pts, &mut ring, &h); + } + ear_clip(&pts, ring) +} + +/// Join hole `h` into `ring` through a bridge from the hole's rightmost vertex. +fn bridge(pts: &[P], ring: &mut Vec, h: &[usize]) { + let (hk, &m) = h.iter().enumerate().max_by(|a, b| pts[*a.1][0].total_cmp(&pts[*b.1][0])).unwrap_or((0, &h[0])); + let mp = pts[m]; + // Nearest ring edge hit by the ray +x from M. + let n = ring.len(); + let mut best: Option<(f64, usize)> = None; + for i in 0..n { + let (a, b) = (pts[ring[i]], pts[ring[(i + 1) % n]]); + if (a[1] > mp[1]) == (b[1] > mp[1]) && a[1] != mp[1] && b[1] != mp[1] { + continue; + } + if a[1] == b[1] { + continue; + } + let t = (mp[1] - a[1]) / (b[1] - a[1]); + if !(0.0..=1.0).contains(&t) { + continue; + } + let x = a[0] + t * (b[0] - a[0]); + if x >= mp[0] && best.is_none_or(|(bx, _)| x < bx) { + best = Some((x, i)); + } + } + let Some((x, i)) = best else { return }; + let (a, b) = (ring[i], ring[(i + 1) % n]); + // Candidate: the edge endpoint with the larger x. + let mut cand = if pts[a][0] > pts[b][0] { (i, a) } else { ((i + 1) % n, b) }; + let ip = [x, mp[1]]; + // Reflex ring vertices inside triangle (M, I, P) block the view: take the one with the + // smallest angle to the ray (closest on ties). + let pp = pts[cand.1]; + let (t0, t1, t2) = if cross(mp, ip, pp) >= 0.0 { (mp, ip, pp) } else { (mp, pp, ip) }; + let mut best_ang = f64::INFINITY; + for k in 0..n { + let v = ring[k]; + if v == cand.1 { + continue; + } + let q = pts[v]; + let prev = pts[ring[(k + n - 1) % n]]; + let next = pts[ring[(k + 1) % n]]; + let reflex = cross(prev, q, next) <= 0.0; + if reflex && in_triangle(q, t0, t1, t2) { + let ang = (q[1] - mp[1]).abs().atan2(q[0] - mp[0]); + let d = (q[0] - mp[0]).hypot(q[1] - mp[1]); + let cd = (pts[cand.1][0] - mp[0]).hypot(pts[cand.1][1] - mp[1]); + if ang < best_ang || (ang == best_ang && d < cd) { + best_ang = ang; + cand = (k, v); + } + } + } + let (k, _) = cand; + // ring[..=k], hole from M around back to M, ring[k] again, rest. + let mut out = Vec::with_capacity(ring.len() + h.len() + 2); + out.extend_from_slice(&ring[..=k]); + for j in 0..=h.len() { + out.push(h[(hk + j) % h.len()]); + } + out.extend_from_slice(&ring[k..]); + *ring = out; +} + +fn ear_clip(pts: &[P], mut ring: Vec) -> Vec<[usize; 3]> { + let mut tris = vec![]; + let mut guard = 0usize; + let mut i = 0usize; + while ring.len() > 3 { + let n = ring.len(); + if guard > 2 * n { + // No ear found (degenerate/self-intersecting input): clip anyway to finish. + let (a, b, c) = (ring[(i + n - 1) % n], ring[i % n], ring[(i + 1) % n]); + tris.push([a, b, c]); + ring.remove(i % n); + guard = 0; + continue; + } + let ii = i % n; + let (a, b, c) = (ring[(ii + n - 1) % n], ring[ii], ring[(ii + 1) % n]); + let (pa, pb, pc) = (pts[a], pts[b], pts[c]); + let cr = cross(pa, pb, pc); + let scale = 1.0 + pa[0].abs() + pa[1].abs() + pc[0].abs() + pc[1].abs(); + // A straight-through vertex is clipped as a zero-area triangle (keeps every contour edge + // in the triangulation, so caps meet walls without T-junctions). + let straight = cr.abs() <= 1e-12 * scale * scale && (pa[0] - pb[0]) * (pc[0] - pb[0]) + (pa[1] - pb[1]) * (pc[1] - pb[1]) <= 0.0; + let convex = cr > 0.0 && !straight; + let mut ear = convex || straight; + if convex { + for &v in &ring { + if v == a || v == b || v == c { + continue; + } + let q = pts[v]; + // Bridged duplicates share positions with the ear's corners. + if q == pa || q == pb || q == pc { + continue; + } + if in_triangle(q, pa, pb, pc) { + ear = false; + break; + } + } + } + if ear { + tris.push([a, b, c]); + ring.remove(ii); + guard = 0; + i = if ii == 0 { 0 } else { ii - 1 }; + } else { + i = ii + 1; + guard += 1; + } + } + if ring.len() == 3 { + tris.push([ring[0], ring[1], ring[2]]); + } + tris +} diff --git a/crates/model/tests/fixtures/cube.mtl b/crates/model/tests/fixtures/cube.mtl new file mode 100644 index 00000000..9ea4c99c --- /dev/null +++ b/crates/model/tests/fixtures/cube.mtl @@ -0,0 +1,11 @@ +# EffectCraft test fixture: materials for cube.obj (original work). +newmtl Red +Kd 1.0 0.0 0.0 +Ns 98 +d 1.0 + +newmtl Gold +Kd 1.0 0.766 0.336 +Ke 0.0 0.0 0.0 +Pr 0.3 +Pm 1.0 diff --git a/crates/model/tests/fixtures/cube.mtl.attribution b/crates/model/tests/fixtures/cube.mtl.attribution new file mode 100644 index 00000000..7af62d2d --- /dev/null +++ b/crates/model/tests/fixtures/cube.mtl.attribution @@ -0,0 +1,7 @@ +asset: cube.mtl +title: MTL fixture: the two materials of cube.obj (Phong exponent and PBR extension keys) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/cube.obj b/crates/model/tests/fixtures/cube.obj new file mode 100644 index 00000000..e50ef30c --- /dev/null +++ b/crates/model/tests/fixtures/cube.obj @@ -0,0 +1,29 @@ +# EffectCraft test fixture: a unit cube with texture coordinates and normals (original work). +mtllib cube.mtl +o Cube +v -0.5 -0.5 0.5 +v 0.5 -0.5 0.5 +v 0.5 0.5 0.5 +v -0.5 0.5 0.5 +v -0.5 -0.5 -0.5 +v 0.5 -0.5 -0.5 +v 0.5 0.5 -0.5 +v -0.5 0.5 -0.5 +vt 0 0 +vt 1 0 +vt 1 1 +vt 0 1 +vn 0 0 1 +vn 0 0 -1 +vn 1 0 0 +vn -1 0 0 +vn 0 1 0 +vn 0 -1 0 +usemtl Red +f 1/1/1 2/2/1 3/3/1 4/4/1 +f 6/1/2 5/2/2 8/3/2 7/4/2 +f 2/1/3 6/2/3 7/3/3 3/4/3 +usemtl Gold +f 5/1/4 1/2/4 4/3/4 8/4/4 +f 4/1/5 3/2/5 7/3/5 8/4/5 +f -4/1/6 -3/2/6 -7/3/6 -8/4/6 diff --git a/crates/model/tests/fixtures/cube.obj.attribution b/crates/model/tests/fixtures/cube.obj.attribution new file mode 100644 index 00000000..faf95fc0 --- /dev/null +++ b/crates/model/tests/fixtures/cube.obj.attribution @@ -0,0 +1,7 @@ +asset: cube.obj +title: Wavefront OBJ fixture: a unit cube with texture coordinates, normals and two materials +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/quad.glb b/crates/model/tests/fixtures/quad.glb new file mode 100644 index 00000000..ab978546 Binary files /dev/null and b/crates/model/tests/fixtures/quad.glb differ diff --git a/crates/model/tests/fixtures/quad.glb.attribution b/crates/model/tests/fixtures/quad.glb.attribution new file mode 100644 index 00000000..c1b0e631 --- /dev/null +++ b/crates/model/tests/fixtures/quad.glb.attribution @@ -0,0 +1,7 @@ +asset: quad.glb +title: Binary glTF (GLB) fixture: the same textured, animated quad with its PNG texture in the BIN chunk +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/model/tests/fixtures/quad.gltf b/crates/model/tests/fixtures/quad.gltf new file mode 100644 index 00000000..82afef9b --- /dev/null +++ b/crates/model/tests/fixtures/quad.gltf @@ -0,0 +1,210 @@ +{ + "asset": { + "version": "2.0", + "generator": "EffectCraft fixture script (original work)" + }, + "scene": 0, + "scenes": [ + { + "nodes": [ + 0 + ] + } + ], + "nodes": [ + { + "name": "Root", + "translation": [ + 0.0, + 1.0, + 0.0 + ], + "children": [ + 1 + ] + }, + { + "name": "Quad", + "mesh": 0, + "scale": [ + 2.0, + 2.0, + 2.0 + ] + } + ], + "meshes": [ + { + "name": "Quad", + "primitives": [ + { + "attributes": { + "POSITION": 0, + "NORMAL": 1, + "TEXCOORD_0": 2 + }, + "indices": 3, + "material": 0 + } + ] + } + ], + "materials": [ + { + "name": "Painted", + "pbrMetallicRoughness": { + "baseColorFactor": [ + 1.0, + 0.5, + 0.25, + 1.0 + ], + "metallicFactor": 0.25, + "roughnessFactor": 0.75, + "baseColorTexture": { + "index": 0 + } + }, + "emissiveFactor": [ + 0.1, + 0.2, + 0.3 + ], + "doubleSided": true + } + ], + "textures": [ + { + "source": 0, + "sampler": 0 + } + ], + "samplers": [ + { + "wrapS": 33071, + "wrapT": 10497 + } + ], + "images": [ + { + "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAEklEQVR4nGP4z8DwHwyBNBgAAEnICff5q7YNAAAAAElFTkSuQmCC" + } + ], + "animations": [ + { + "name": "Move", + "samplers": [ + { + "input": 4, + "output": 5, + "interpolation": "LINEAR" + } + ], + "channels": [ + { + "sampler": 0, + "target": { + "node": 0, + "path": "translation" + } + } + ] + } + ], + "accessors": [ + { + "bufferView": 0, + "componentType": 5126, + "count": 4, + "type": "VEC3", + "min": [ + -0.5, + -0.5, + 0.0 + ], + "max": [ + 0.5, + 0.5, + 0.0 + ] + }, + { + "bufferView": 1, + "componentType": 5126, + "count": 4, + "type": "VEC3" + }, + { + "bufferView": 2, + "componentType": 5126, + "count": 4, + "type": "VEC2" + }, + { + "bufferView": 3, + "componentType": 5123, + "count": 6, + "type": "SCALAR" + }, + { + "bufferView": 4, + "componentType": 5126, + "count": 2, + "type": "SCALAR", + "min": [ + 0.0 + ], + "max": [ + 1.0 + ] + }, + { + "bufferView": 5, + "componentType": 5126, + "count": 2, + "type": "VEC3" + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 48, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 48, + "byteLength": 48, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 96, + "byteLength": 32, + "target": 34962 + }, + { + "buffer": 0, + "byteOffset": 128, + "byteLength": 12, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 140, + "byteLength": 8 + }, + { + "buffer": 0, + "byteOffset": 148, + "byteLength": 24 + } + ], + "buffers": [ + { + "byteLength": 172, + "uri": "data:application/octet-stream;base64,AAAAvwAAAL8AAAAAAAAAPwAAAL8AAAAAAAAAPwAAAD8AAAAAAAAAvwAAAD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAgD8AAIA/AACAPwAAgD8AAAAAAAAAAAAAAAAAAAEAAgAAAAIAAwAAAAAAAACAPwAAAAAAAIA/AAAAAAAAAEAAAIA/AAAAAA==" + } + ] +} diff --git a/crates/model/tests/fixtures/quad.gltf.attribution b/crates/model/tests/fixtures/quad.gltf.attribution new file mode 100644 index 00000000..1ccf03b7 --- /dev/null +++ b/crates/model/tests/fixtures/quad.gltf.attribution @@ -0,0 +1,7 @@ +asset: quad.gltf +title: glTF 2.0 fixture: a textured quad under a parent node with a translation animation (all data embedded as data URIs) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft agent (M7.4) +notes: Hand-written / script-generated test data for the model importer, made from the public Khronos glTF 2.0 specification and the Wavefront OBJ/MTL format description; not taken from any third-party file. The 2x2 texture is four solid colours. diff --git a/crates/project/src/build.rs b/crates/project/src/build.rs index 12e384d6..a201879d 100644 --- a/crates/project/src/build.rs +++ b/crates/project/src/build.rs @@ -584,6 +584,13 @@ pub fn light_options(ids: &mut Ids, kind: LightKind) -> PropGroup { .with(ids.prop("coneAngle", "Cone Angle", Value::Scalar(90.0)).with_ui(ParamUi::Angle)) .with(ids.prop("coneFeather", "Cone Feather", Value::Scalar(50.0)).with_ui(ParamUi::Percent)); } + if kind == LightKind::Environment { + // The environment image: a layer (footage, comp…) mapped as an equirectangular panorama; + // none = the comp's Environment Layer. + return g + .with(ids.prop("source", "Source", Value::Layer(None)).with_ui(ParamUi::Layer)) + .with(ids.prop("rotation", "Environment Rotation", Value::Scalar(0.0)).with_ui(ParamUi::Angle)); + } if kind != LightKind::Ambient { g = g .with(ids.prop("falloff", "Falloff", Value::Enum(0)).with_ui(popup(&["None", "Smooth", "Inverse Square Clamped"]))) @@ -609,6 +616,70 @@ pub fn material_options(ids: &mut Ids) -> PropGroup { .with(ids.prop("metal", "Metal", Value::Scalar(100.0)).with_ui(ParamUi::Percent)) } +/// Material Options of a 3D model or primitive layer (Advanced 3D): shadows and lights, plus +/// the primitive's own metallic-roughness material (models keep their file's materials). +pub fn model_material_options(ids: &mut Ids, primitive: bool) -> PropGroup { + let mut g = ids + .group("materialOptions", "Material Options") + .with(ids.prop("castsShadows", "Casts Shadows", Value::Enum(0)).with_ui(popup(&["Off", "On", "Only"]))) + .with(ids.prop("acceptsShadows", "Accepts Shadows", Value::Enum(1)).with_ui(popup(&["Off", "On", "Only"]))) + .with(ids.prop("acceptsLights", "Accepts Lights", Value::Bool(true)).with_ui(ParamUi::Checkbox)) + .with(ids.prop("appearsInReflections", "Appears in Reflections", Value::Bool(true)).with_ui(ParamUi::Checkbox)); + if primitive { + g = g + .with(ids.prop("baseColor", "Base Color", Value::Color([0.8, 0.8, 0.8, 1.0])).with_ui(ParamUi::Color)) + .with(ids.prop("metallic", "Metallic", Value::Scalar(0.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("roughness", "Roughness", Value::Scalar(50.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("emissive", "Emissive", Value::Color([0.0, 0.0, 0.0, 1.0])).with_ui(ParamUi::Color)); + } + g +} + +/// Geometry Options of an imported model layer: model units → pixels, and the animation clip +/// played along layer time. +pub fn model_geometry_options(ids: &mut Ids, unit_scale: f64, clips: &[String]) -> PropGroup { + let mut opts: Vec<&str> = vec!["None"]; + opts.extend(clips.iter().map(String::as_str)); + ids.group("geometryOptions", "Geometry Options") + .with(ids.prop("unitScale", "Model Scale", Value::Scalar(unit_scale)).with_ui(slider(0.0, 1.0e6, 0.0, 1000.0, 2))) + .with(ids.prop("animation", "Animation", Value::Enum(if clips.is_empty() { 0 } else { 1 })).with_ui(popup(&opts))) + .with(ids.prop("loopAnimation", "Loop Animation", Value::Bool(true)).with_ui(ParamUi::Checkbox)) + .with(ids.prop("animationSpeed", "Animation Speed", Value::Scalar(100.0)).with_ui(ParamUi::Percent)) +} + +/// Geometry Options of a primitive layer (pixels). +pub fn primitive_geometry_options(ids: &mut Ids, kind: crate::PrimitiveKind) -> PropGroup { + use crate::PrimitiveKind as K; + let px = |ids: &mut Ids, m: &str, n: &str, v: f64| ids.prop(m, n, Value::Scalar(v)).with_ui(slider(0.0, 100_000.0, 0.0, 2000.0, 1)); + let count = |ids: &mut Ids, m: &str, n: &str, v: f64| ids.prop(m, n, Value::Scalar(v)).with_ui(slider(3.0, 512.0, 3.0, 128.0, 0)); + let mut g = ids.group("geometryOptions", "Geometry Options"); + let props = match kind { + K::Cube => vec![px(ids, "width", "Width", 300.0), px(ids, "height", "Height", 300.0), px(ids, "depth", "Depth", 300.0)], + K::Plane => vec![px(ids, "width", "Width", 400.0), px(ids, "height", "Height", 400.0)], + K::Sphere => vec![px(ids, "radius", "Radius", 150.0), count(ids, "segments", "Segments", 48.0), count(ids, "rings", "Rings", 24.0)], + K::Torus => vec![ + px(ids, "radius", "Radius", 150.0), + px(ids, "tubeRadius", "Tube Radius", 50.0), + count(ids, "segments", "Segments", 48.0), + count(ids, "rings", "Sides", 24.0), + ], + K::Cone | K::Cylinder => vec![px(ids, "radius", "Radius", 150.0), px(ids, "height", "Height", 300.0), count(ids, "segments", "Segments", 48.0)], + }; + for p in props { + g = g.with(p); + } + g +} + +/// Geometry Options of a text or shape layer in an Advanced 3D comp (extrusion and bevels). +pub fn extrusion_geometry_options(ids: &mut Ids) -> PropGroup { + ids.group("geometryOptions", "Geometry Options") + .with(ids.prop("bevelStyle", "Bevel Style", Value::Enum(0)).with_ui(popup(&["None", "Angular", "Concave", "Convex"]))) + .with(ids.prop("bevelDepth", "Bevel Depth", Value::Scalar(2.0)).with_ui(slider(0.0, 1000.0, 0.0, 100.0, 1))) + .with(ids.prop("holeBevelDepth", "Hole Bevel Depth", Value::Scalar(100.0)).with_ui(slider(0.0, 100.0, 0.0, 100.0, 1))) + .with(ids.prop("extrusionDepth", "Extrusion Depth", Value::Scalar(0.0)).with_ui(slider(0.0, 10_000.0, 0.0, 1000.0, 1))) +} + pub fn audio(ids: &mut Ids) -> PropGroup { ids.group("audio", "Audio").with(ids.prop("levels", "Audio Levels", Value::Vec2([0.0, 0.0]))) } @@ -629,6 +700,7 @@ pub fn default_label(src: &LayerSource, project: &Project) -> Label { LayerSource::Shape => Label::Blue, LayerSource::Null => Label::Red, LayerSource::Camera | LayerSource::Light { .. } => Label::Pink, + LayerSource::Model { .. } | LayerSource::Primitive { .. } => Label::Aqua, } } @@ -640,7 +712,12 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource let (w, h) = (size.0 as f64, size.1 as f64); let (cw, ch) = (comp.width as f64, comp.height as f64); let anchor = match source { - LayerSource::Text | LayerSource::Shape | LayerSource::Camera | LayerSource::Light { .. } => [0.0, 0.0], + LayerSource::Text + | LayerSource::Shape + | LayerSource::Camera + | LayerSource::Light { .. } + | LayerSource::Model { .. } + | LayerSource::Primitive { .. } => [0.0, 0.0], LayerSource::Null => [50.0, 50.0], _ => [w / 2.0, h / 2.0], }; @@ -654,7 +731,9 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource } _ => {} } - if source.is_av() { + if source.is_model() { + // Model layers have no masks or effects (as in After Effects). + } else if source.is_av() { root.children.push(masks(&mut ids).into()); root.children.push(effects(&mut ids).into()); } else if matches!(source, LayerSource::Null) { @@ -694,8 +773,17 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource // Nulls are 100×100. } root.children.push(std::mem::replace(&mut tr, PropGroup::new(0, "", "")).into()); - if source.is_av() { - root.children.push(material_options(&mut ids).into()); + match source { + LayerSource::Primitive { kind } => { + root.children.push(primitive_geometry_options(&mut ids, kind).into()); + root.children.push(model_material_options(&mut ids, true).into()); + } + LayerSource::Model { .. } => { + root.children.push(model_geometry_options(&mut ids, 1.0, &[]).into()); + root.children.push(model_material_options(&mut ids, false).into()); + } + _ if source.is_av() => root.children.push(material_options(&mut ids).into()), + _ => {} } } let has_audio = matches!(&source, LayerSource::Footage { item } if matches!(project.item(*item).map(|i| &i.kind), Some(crate::ItemKind::Footage(f)) if f.has_audio)) @@ -711,6 +799,7 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource } else { crate::AutoOrient::Off }; + let switches = Switches { three_d: source.is_model(), ..Switches::default() }; Layer { id, name: comp.unique_layer_name(name), @@ -721,13 +810,14 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource in_point: Tick::ZERO, out_point: out, stretch: 100.0, - switches: Switches::default(), + switches, blend_mode: BlendMode::Normal, preserve_transparency: false, track_matte: None, parent: None, markers: vec![], markers_locked: false, + environment: false, auto_orient, props: root, } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 1ed35c09..4a00c413 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -310,6 +310,36 @@ impl LightKind { } } +/// A parametric 3D primitive (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum PrimitiveKind { + #[default] + Cube, + Sphere, + Plane, + Torus, + Cone, + Cylinder, +} + +impl PrimitiveKind { + pub const ALL: [PrimitiveKind; 6] = + [PrimitiveKind::Cube, PrimitiveKind::Sphere, PrimitiveKind::Plane, PrimitiveKind::Torus, PrimitiveKind::Cone, PrimitiveKind::Cylinder]; + pub fn label(self) -> &'static str { + match self { + PrimitiveKind::Cube => "Cube", + PrimitiveKind::Sphere => "Sphere", + PrimitiveKind::Plane => "Plane", + PrimitiveKind::Torus => "Torus", + PrimitiveKind::Cone => "Cone", + PrimitiveKind::Cylinder => "Cylinder", + } + } + pub fn parse(s: &str) -> Option { + PrimitiveKind::ALL.into_iter().find(|k| k.label().eq_ignore_ascii_case(s.trim())) + } +} + /// What a layer shows. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] @@ -333,12 +363,20 @@ pub enum LayerSource { Light { kind: LightKind, }, + /// An imported 3D model (a footage item of kind [`FootageKind::Model`]; Advanced 3D). + Model { + item: ItemId, + }, + /// A parametric 3D primitive (Advanced 3D). + Primitive { + kind: PrimitiveKind, + }, } impl LayerSource { pub fn item(&self) -> Option { match self { - LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } => Some(*item), + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => Some(*item), _ => None, } } @@ -352,12 +390,18 @@ impl LayerSource { LayerSource::Null => "Null", LayerSource::Camera => "Camera", LayerSource::Light { .. } => "Light", + LayerSource::Model { .. } => "3D Model", + LayerSource::Primitive { .. } => "3D Primitive", } } /// Layers with pixels (cameras, lights and nulls have none). pub fn is_av(&self) -> bool { !matches!(self, LayerSource::Camera | LayerSource::Light { .. } | LayerSource::Null) } + /// A 3D model or primitive layer (meshes; drawn by the Advanced 3D renderer). + pub fn is_model(&self) -> bool { + matches!(self, LayerSource::Model { .. } | LayerSource::Primitive { .. }) + } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -489,6 +533,10 @@ pub struct Layer { pub markers_locked: bool, #[serde(default)] pub auto_orient: AutoOrient, + /// Layer ▸ Environment Layer: the layer is the environment (image-based light and + /// reflections) of Advanced 3D comps instead of being drawn. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub environment: bool, /// The property tree (Masks, Effects, Transform, Text, Contents, Camera/Light options…). pub props: PropGroup, } @@ -540,6 +588,8 @@ pub enum FootageKind { Still, Sequence, Audio, + /// A 3D model (glTF 2.0 / OBJ), placed as a model layer. + Model, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -625,6 +675,7 @@ impl Item { FootageKind::Still => "Image", FootageKind::Sequence => "Image Sequence", FootageKind::Audio => "Audio", + FootageKind::Model => "3D Model", }, ItemKind::Solid(_) => "Solid", } diff --git a/crates/raster/src/channels3d.rs b/crates/raster/src/channels3d.rs new file mode 100644 index 00000000..0402349b --- /dev/null +++ b/crates/raster/src/channels3d.rs @@ -0,0 +1,206 @@ +//! Auxiliary (3D) channels that travel with a layer's pixels: depth, object / material IDs, +//! surface normals, texture UVs, coverage and Cryptomatte ID/coverage pairs. +//! +//! They come from multi-layer OpenEXR footage (any channel by its full `layer.channel` name) +//! or from the compositor's own render passes for precomps of 3D comps. The 3D Channel effects +//! read them through `EffectHost::aux`. +//! +//! Conventions (shared by the EXR reader and the render pass): +//! +//! * channel planes are row-major `width × height` `f32`s covering the layer's source rectangle, +//! with `scale` aux pixels per layer pixel; +//! * depth is camera-space distance in pixels (`Z`), background `BACKGROUND_DEPTH`; +//! * `ObjectID` / `MaterialID` are small integers (0 = background); +//! * Cryptomatte layers follow the published Cryptomatte specification (Psyop, 2015): a layer +//! named `CryptoObject` stores rank pairs in `CryptoObject00.R/G/B/A`, `CryptoObject01.R/…` +//! (ID float, coverage, ID float, coverage, …), IDs are MurmurHash3 (x86, 32-bit, seed 0) of +//! the object name reinterpreted as a float with the exponent nudged off 0 and 255, and the +//! manifest maps names to hashes. + +/// Depth written where nothing was rendered. +pub const BACKGROUND_DEPTH: f32 = 1.0e10; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AuxChannels { + pub width: u32, + pub height: u32, + /// Aux pixels per layer pixel. + pub scale: f64, + /// Named planes (`Z`, `ObjectID`, `CryptoObject00.R`, …). + pub channels: Vec<(String, Vec)>, + /// Cryptomatte manifests: (layer name, [(object name, hash)]). + pub manifests: Vec<(String, Vec<(String, u32)>)>, +} + +impl AuxChannels { + pub fn new(width: u32, height: u32, scale: f64) -> AuxChannels { + AuxChannels { width, height, scale, channels: vec![], manifests: vec![] } + } + + /// A channel by exact name (case-insensitive). + pub fn get(&self, name: &str) -> Option<&[f32]> { + self.channels.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_slice()) + } + + /// The first channel present among `names` (also matching `*.name` suffixes, so `Z` finds + /// `depth.Z`). + pub fn find(&self, names: &[&str]) -> Option<&[f32]> { + for n in names { + if let Some(v) = self.get(n) { + return Some(v); + } + } + for n in names { + let suffix = format!(".{}", n.to_ascii_lowercase()); + if let Some((_, v)) = self.channels.iter().find(|(c, _)| c.to_ascii_lowercase().ends_with(&suffix)) { + return Some(v); + } + } + None + } + + pub fn names(&self) -> Vec<&str> { + self.channels.iter().map(|(n, _)| n.as_str()).collect() + } + + pub fn depth(&self) -> Option<&[f32]> { + self.find(&["Z", "depth", "Depth", "zDepth", "Z-Depth"]) + } + pub fn object_id(&self) -> Option<&[f32]> { + self.find(&["ObjectID", "object_id", "objectId", "id"]) + } + pub fn material_id(&self) -> Option<&[f32]> { + self.find(&["MaterialID", "material_id", "materialId"]) + } + pub fn coverage(&self) -> Option<&[f32]> { + self.find(&["Coverage", "coverage"]) + } + + /// Cryptomatte layer names present (prefixes with a `00.R` channel). + pub fn crypto_layers(&self) -> Vec { + let mut v: Vec = + self.channels.iter().filter_map(|(n, _)| n.strip_suffix("00.R").or_else(|| n.strip_suffix("00.r")).map(str::to_string)).collect(); + v.sort(); + v.dedup(); + v + } + + /// Rank pairs (id, coverage) of Cryptomatte layer `layer` at aux pixel index `i`. + pub fn crypto_ranks(&self, layer: &str, i: usize) -> Vec<(f32, f32)> { + let mut out = vec![]; + for k in 0.. { + let (r, g, b, a) = ( + self.get(&format!("{layer}{k:02}.R")), + self.get(&format!("{layer}{k:02}.G")), + self.get(&format!("{layer}{k:02}.B")), + self.get(&format!("{layer}{k:02}.A")), + ); + let (Some(r), Some(g)) = (r, g) else { break }; + out.push((r[i], g[i])); + if let (Some(b), Some(a)) = (b, a) { + out.push((b[i], a[i])); + } + } + out + } + + /// Aux pixel index of a layer-space point (nearest; `None` outside). + #[inline] + pub fn index_at(&self, lx: f64, ly: f64) -> Option { + let x = (lx * self.scale).floor(); + let y = (ly * self.scale).floor(); + if x < 0.0 || y < 0.0 || x >= self.width as f64 || y >= self.height as f64 { + return None; + } + Some(y as usize * self.width as usize + x as usize) + } + + /// Manifest entry name for a hash in Cryptomatte layer `layer`. + pub fn manifest_name(&self, layer: &str, hash: u32) -> Option<&str> { + self.manifests.iter().find(|(l, _)| l == layer)?.1.iter().find(|(_, h)| *h == hash).map(|(n, _)| n.as_str()) + } +} + +/// MurmurHash3 x86 32-bit (public-domain algorithm by Austin Appleby). +pub fn murmur3_32(data: &[u8], seed: u32) -> u32 { + const C1: u32 = 0xcc9e_2d51; + const C2: u32 = 0x1b87_3593; + let mut h = seed; + let chunks = data.chunks_exact(4); + let tail = chunks.remainder(); + for c in chunks { + let mut k = u32::from_le_bytes([c[0], c[1], c[2], c[3]]); + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + h = h.rotate_left(13).wrapping_mul(5).wrapping_add(0xe654_6b64); + } + let mut k = 0u32; + for (i, b) in tail.iter().enumerate() { + k ^= (*b as u32) << (8 * i); + } + if !tail.is_empty() { + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + } + h ^= data.len() as u32; + h ^= h >> 16; + h = h.wrapping_mul(0x85eb_ca6b); + h ^= h >> 13; + h = h.wrapping_mul(0xc2b2_ae35); + h ^= h >> 16; + h +} + +/// Cryptomatte hash of a name (the manifest value). +pub fn crypto_hash(name: &str) -> u32 { + let h = murmur3_32(name.as_bytes(), 0); + // Keep the float finite and normal: flip the low exponent bit when it is all 0s or 1s. + let e = (h >> 23) & 0xff; + if e == 0 || e == 255 { h ^ (1 << 23) } else { h } +} + +/// The ID float stored in Cryptomatte channels for a hash. +pub fn crypto_float(hash: u32) -> f32 { + f32::from_bits(hash) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn murmur3_reference_values() { + // Published MurmurHash3_x86_32 test vectors. + assert_eq!(murmur3_32(b"", 0), 0); + assert_eq!(murmur3_32(b"", 1), 0x514e_28b7); + assert_eq!(murmur3_32(b"", 0xffff_ffff), 0x81f1_6f39); + assert_eq!(murmur3_32(&[0, 0, 0, 0], 0), 0x2362_f9de); + assert_eq!(murmur3_32(b"a", 0x9747_b28c), 0x7fa0_9ea6); + assert_eq!(murmur3_32(b"aaaa", 0x9747_b28c), 0x5a97_808a); + assert_eq!(murmur3_32(b"Hello, world!", 0x9747_b28c), 0x2488_4cba); + assert_eq!(murmur3_32(b"The quick brown fox jumps over the lazy dog", 0x9747_b28c), 0x2fa8_26cd); + } + + #[test] + fn crypto_floats_are_normal_and_round_trip() { + for n in ["bunny", "set", "ground", "a", ""] { + let h = crypto_hash(n); + let f = crypto_float(h); + assert!(f.is_finite() && (f == 0.0 || f.is_normal() || f.abs() > 0.0), "{n}"); + assert_eq!(f.to_bits(), h); + } + } + + #[test] + fn lookup_helpers() { + let mut a = AuxChannels::new(2, 1, 1.0); + a.channels.push(("depth.Z".into(), vec![5.0, 6.0])); + a.channels.push(("CryptoObject00.R".into(), vec![1.0, 2.0])); + a.channels.push(("CryptoObject00.G".into(), vec![0.5, 1.0])); + assert_eq!(a.depth(), Some(&[5.0, 6.0][..])); + assert_eq!(a.crypto_layers(), vec!["CryptoObject".to_string()]); + assert_eq!(a.crypto_ranks("CryptoObject", 0), vec![(1.0, 0.5)]); + assert_eq!(a.index_at(1.5, 0.2), Some(1)); + assert_eq!(a.index_at(2.5, 0.2), None); + } +} diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index dfe53848..4f244199 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -4,6 +4,7 @@ //! heavy operation is row-parallel with rayon. pub mod blur; +pub mod channels3d; pub mod flow; pub mod warp; @@ -11,6 +12,7 @@ use effectcraft_color::{BlendMode, blend_pixel}; use rayon::prelude::*; pub use blur::{box_blur, directional_blur, gaussian_blur, radial_blur}; +pub use channels3d::AuxChannels; pub use warp::{Sampling, WarpOpts, accumulate_warp, composite_warp, composite_warp_reference, resample}; pub type Px = [f32; 4]; diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 4c8a9787..80e30455 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +effectcraft-model = { workspace = true } kurbo = { workspace = true } rayon = { workspace = true } web-time = { workspace = true } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 14a81575..64e2684b 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -39,6 +39,16 @@ pub trait FootageSource: Send + Sync { fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { None } + /// Auxiliary 3D channels of `item` at source time `t` (multi-layer OpenEXR: depth, IDs, + /// Cryptomatte, any named channel), with `scale` relative to the file's pixels. `None` when + /// the footage has none. + fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { + None + } + /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). + fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { + None + } /// Set the decoded-frame cache budget in bytes (Settings ▸ Memory & CPU); sources without /// a cache ignore it. fn set_cache_budget(&self, _bytes: usize) {} @@ -107,6 +117,29 @@ impl EffectHost for FxHost<'_, '_, '_> { sub.layer_input(&self.ctx.at(t), self.layer, n).map(|b| (*b).clone()) } + fn aux(&self) -> Option> { + match &self.layer.source { + LayerSource::Footage { item } => { + let ItemKind::Footage(f) = &self.r.project.item(*item)?.kind else { return None }; + let a = self.r.footage.aux(*item, f, self.ctx.source_time(self.layer))?; + // Aux pixels per layer pixel (the layer is the footage at its nominal size). + if f.width > 0 && a.width > 0 { + let mut a = (*a).clone(); + a.scale = a.width as f64 / f.width as f64; + return Some(Arc::new(a)); + } + Some(a) + } + LayerSource::Comp { item } => { + if self.r.depth > MAX_FX_DEPTH || self.r.project.comp_contains(*item, self.ctx.comp_id) { + return None; + } + self.r.nested().comp_aux(*item, self.ctx.source_time(self.layer)).map(Arc::new) + } + _ => None, + } + } + fn layer_at(&self, id: u64, comp_time: f64, masks_and_effects: bool) -> Option { let other = self.ctx.layer(effectcraft_project::LayerId(id))?; if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { @@ -174,6 +207,12 @@ pub trait Accelerator: Send + Sync { /// Run a chain of supported effects on `buf`, clamping and quantising to `levels` after /// every effect when set (8/16 bpc). `None` = not handled. fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option; + /// Rasterise and shade an Advanced 3D scene at its raster size (depth buffer, PBR, image + /// based light, shadow maps), with the CPU rasteriser's semantics + /// ([`three_d::adv::raster::render`]). `None` = not handled (the CPU renders it). + fn raster_3d(&self, _scene: &three_d::adv::Scene) -> Option { + None + } } #[derive(Clone, Copy, Debug)] @@ -527,8 +566,15 @@ impl<'a> Renderer<'a> { let layer_size = if size.0 == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; let mask_shapes = masks::shapes(ctx, layer); let host = FxHost { r: self, ctx, layer, index: Default::default() }; - let env = - EffectEnv { masks: &mask_shapes, host: Some(&host), comp_time: ctx.time.seconds(), frame_rate: ctx.comp.frame_rate.as_f64(), effect_index: 0 }; + let env = EffectEnv { + masks: &mask_shapes, + host: Some(&host), + comp_time: ctx.time.seconds(), + frame_rate: ctx.comp.frame_rate.as_f64(), + effect_index: 0, + working_space: self.pipe.space, + working_linear: self.pipe.linear, + }; // Video effects in stack order (index, group, spec); disabled and audio effects skipped. let stack: Vec<(usize, &effectcraft_project::PropGroup, &'static effectcraft_effects::EffectSpec)> = fx .groups() @@ -1168,6 +1214,13 @@ impl<'a> Renderer<'a> { self.depth } + /// The auxiliary 3D channels of comp `comp_id` at comp time `t` (depth, layer IDs, normals, + /// UVs, Cryptomatte; see `three_d::compose::aux_pass`), at the renderer's output scale. + pub fn comp_aux(&self, comp_id: ItemId, t: Tick) -> Option { + let ctx = self.eval_ctx(comp_id, t)?; + Some(three_d::compose::aux_pass(self, &ctx)) + } + /// Evaluation context of comp `comp_id` at comp time `t`. pub fn eval_ctx(&self, comp_id: ItemId, t: Tick) -> Option> { Some(self.ctx(comp_id, self.project.comp(comp_id)?, t)) @@ -1181,7 +1234,7 @@ impl<'a> Renderer<'a> { comp.layers .iter() .rev() - .filter(|l| l.is_active_at(t) && l.has_video() && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) + .filter(|l| l.is_active_at(t) && l.has_video() && !l.environment && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) .collect() } @@ -1288,6 +1341,8 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { #[cfg(test)] mod tests_audio_fx; #[cfg(test)] +mod tests_aux; +#[cfg(test)] mod tests_collapse; #[cfg(test)] mod tests_color; diff --git a/crates/render/src/shapes.rs b/crates/render/src/shapes.rs index 25eb6090..7cb59135 100644 --- a/crates/render/src/shapes.rs +++ b/crates/render/src/shapes.rs @@ -508,6 +508,27 @@ pub fn render(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup, s: f64) -> Buf #[cfg(test)] mod tests; +/// The filled outlines of a shape layer in layer space with their straight colours (gradient +/// fills: the colour half way along), for Advanced 3D extrusion. Strokes are not extruded. +pub fn fill_outlines(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup) -> Vec<(Vec, [f32; 4])> { + let mut arena = Vec::new(); + let (draws, _) = collect(ctx, layer, contents, &mut arena); + draws + .iter() + .rev() + .filter(|d| d.opacity > 0.0) + .filter_map(|d| { + let c = match &d.paint { + Paint::Fill { color, .. } => *color, + Paint::Gradient { g, stroke: None, .. } => g.sample(0.5), + _ => return None, + }; + let paths: Vec = d.slots.iter().map(|&i| arena[i].clone()).filter(|p| !p.elements().is_empty()).collect(); + (!paths.is_empty()).then_some((paths, [c[0], c[1], c[2], c[3] * d.opacity])) + }) + .collect() +} + /// Layer-space bounds of a shape layer's painted contents (strokes included), without /// rasterising. pub fn content_bounds(ctx: &EvalCtx, layer: &Layer, contents: &PropGroup) -> Option { diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index dfd49ccc..a249955f 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -174,7 +174,17 @@ fn blocks_match_one_long_mix() { let x = ((t * 12345.678).sin() * 43758.5453).fract() as f32; (x, -x) } - for id in ["ec.audio.reverb", "ec.audio.basstreble", "ec.audio.flangechorus", "ec.audio.modulator", "ec.audio.delay", "ec.audio.parametriceq"] { + for id in [ + "ec.audio.reverb", + "ec.audio.basstreble", + "ec.audio.flangechorus", + "ec.audio.modulator", + "ec.audio.delay", + "ec.audio.parametriceq", + "ec.audio.compressor", + "ec.audio.gate", + "ec.audio.distortion", + ] { let vals: Vec<(&str, Value)> = match id { "ec.audio.basstreble" => vec![("bass", Value::Scalar(50.0))], "ec.audio.parametriceq" => vec![("band1BoostCut", Value::Scalar(9.0))], diff --git a/crates/render/src/tests_aux.rs b/crates/render/src/tests_aux.rs new file mode 100644 index 00000000..31bb1881 --- /dev/null +++ b/crates/render/src/tests_aux.rs @@ -0,0 +1,98 @@ +//! Auxiliary 3D channels: the comp's depth / ID / Cryptomatte pass and the 3D Channel effects on +//! a precomp of a 3D comp. + +use effectcraft_color::Label; +use effectcraft_keyframe::Value; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project, Solid}; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{NoFootage, RenderOpts, Renderer, render_frame}; + +const W: u32 = 200; +const H: u32 = 100; + +fn zoom() -> f64 { + W as f64 * 50.0 / 36.0 +} + +fn solid3(p: &mut Project, comp: &Comp, name: &str, color: [f32; 3], pos: [f64; 3]) -> Layer { + let sid = p.add_item(name, Label::Red, None, ItemKind::Solid(Solid { color, width: 60, height: 60, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, name, LayerSource::Solid { item: sid }, (60, 60), None); + l.name = name.to_string(); + l.switches.three_d = true; + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3(pos); + l +} + +/// Inner 3D comp: "Near" (red, z 0, left) above "Far" (green, z 500, right). Outer comp: the +/// precomp layer, optionally with an effect. +fn scene(effect: Option<(&str, Vec<(&str, Value)>)>) -> (Project, ItemId, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = effectcraft_project::BitDepth::Bpc32; + let inner = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let iid = p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into())); + let near = solid3(&mut p, &inner, "Near", [1.0, 0.0, 0.0], [60.0, 50.0, 0.0]); + let far = solid3(&mut p, &inner, "Far", [0.0, 1.0, 0.0], [140.0, 50.0, 500.0]); + p.comp_mut(iid).unwrap().layers.push(near); + p.comp_mut(iid).unwrap().layers.push(far); + let outer = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let oid = p.add_item("Outer", Label::Sandstone, None, ItemKind::Comp(outer.clone().into())); + let mut pre = build::layer(&mut p, &outer, "Inner", LayerSource::Comp { item: iid }, (W, H), None); + if let Some((id, vals)) = effect { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [W as f64, H as f64]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap().value = v.clone(); + } + pre.props.sub_mut("effects").unwrap().children.push(g.into()); + } + p.comp_mut(oid).unwrap().layers.push(pre); + (p, iid, oid) +} + +#[test] +fn comp_aux_pass_has_depth_ids_and_cryptomatte() { + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts::default()); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height), (W, H)); + let at = |name: &str, x: usize, y: usize| a.get(name).unwrap()[y * W as usize + x]; + // Near plane at the camera's zoom distance, the far one 500 px behind it. + assert!((at("Z", 60, 50) as f64 - zoom()).abs() < 1e-2, "{}", at("Z", 60, 50)); + assert!((at("Z", 114, 50) as f64 - (zoom() + 500.0)).abs() < 1e-2, "{}", at("Z", 114, 50)); + assert_eq!(at("Z", 5, 5), BACKGROUND_DEPTH); + assert_eq!((at("ObjectID", 60, 50), at("ObjectID", 114, 50), at("ObjectID", 5, 5)), (1.0, 2.0, 0.0)); + assert_eq!(at("Coverage", 60, 50), 1.0); + // Planes face the camera. + assert!((at("N.Z", 60, 50) + 1.0).abs() < 1e-5); + // Cryptomatte rank 0 = the layer under the pixel, full coverage; manifest lists names. + assert_eq!(at("CryptoObject00.R", 60, 50).to_bits(), crypto_hash("Near")); + assert_eq!(at("CryptoObject00.G", 60, 50), 1.0); + assert_eq!(a.manifest_name("CryptoObject", crypto_hash("Far")), Some("Far")); +} + +#[test] +fn three_d_channel_effects_work_on_a_precomp_of_a_3d_comp() { + let near = (60, 50); + let far = (114, 50); + let alpha = |p: &Project, oid: ItemId, (x, y): (i64, i64)| render_frame(p, oid, Tick::ZERO, 1.0).get(x, y)[3]; + let (p, _, oid) = scene(Some(("ec.3d.idmatte", vec![("idSelection", Value::Scalar(2.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.depthmatte", vec![("depth", Value::Scalar(zoom() + 250.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.cryptomatte", vec![("selection", Value::Str("Near".into())), ("display", Value::Enum(1))]))); + assert!(alpha(&p, oid, near) > 0.99); + assert_eq!(alpha(&p, oid, far), 0.0); + // Half resolution: the pass follows the render scale. + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts { scale: 0.5, ..Default::default() }); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height, a.scale), (W / 2, H / 2, 0.5)); + assert_eq!(a.get("ObjectID").unwrap()[25 * 100 + 30], 1.0); +} diff --git a/crates/render/src/three_d/adv/mod.rs b/crates/render/src/three_d/adv/mod.rs new file mode 100644 index 00000000..04f23885 --- /dev/null +++ b/crates/render/src/three_d/adv/mod.rs @@ -0,0 +1,165 @@ +//! The Advanced 3D renderer (Composition Settings ▸ 3D Renderer ▸ Advanced 3D). +//! +//! A run of 3D layers becomes one [`Scene`] of world-space triangles (models, primitives, +//! extruded text and shapes, and textured cards for every other layer), lit physically +//! ([`shade`]) by the comp's lights, an Environment light's image and shadow maps. The scene +//! is rasterised with a depth buffer on the GPU when an [`crate::Accelerator`] implements +//! [`crate::Accelerator::raster_3d`], otherwise by the software rasteriser ([`raster`]), at 2×2 +//! supersampling. The resolve (box filter), depth of field (a gather blur by each pixel's +//! circle of confusion) and the conversion back to the working encoding run on the CPU for +//! both paths, and the result is composited over the layers below the run. + +pub mod raster; +pub mod scene; +pub mod shade; + +use effectcraft_project::Layer; +use rayon::prelude::*; + +pub use raster::Target; +pub use scene::{EnvInfo, Light, Material, Scene, ShadowMap, TexInfo, Vertex}; + +use super::camera::Dof; +use super::compose::camera_for; +use crate::{EvalCtx, Image, Renderer}; + +/// Box-filter a supersampled target down by `k`: (premultiplied colour, mean depth of the +/// covered subsamples, ∞ where none). +pub fn resolve(t: &Target, k: u32) -> (Image, Vec) { + let (w, h) = (t.width / k, t.height / k); + let mut img = Image::new(w, h); + let mut depth = vec![f32::INFINITY; (w * h) as usize]; + let n = (k * k) as f32; + img.data.par_chunks_mut(w as usize).zip(depth.par_chunks_mut(w as usize)).enumerate().for_each(|(y, (row, drow))| { + for x in 0..w as usize { + let mut acc = [0.0f32; 4]; + let (mut dz, mut dn) = (0.0f32, 0.0f32); + for j in 0..k as usize { + for i in 0..k as usize { + let s = (y * k as usize + j) * t.width as usize + x * k as usize + i; + let c = t.color[s]; + for q in 0..4 { + acc[q] += c[q]; + } + if t.depth[s].is_finite() { + dz += t.depth[s]; + dn += 1.0; + } + } + } + row[x] = acc.map(|v| v / n); + if dn > 0.0 { + drow[x] = dz / dn; + } + } + }); + (img, depth) +} + +/// Depth of field: a gather blur where each pixel collects the neighbours whose circle of +/// confusion (from their depth, in output pixels) reaches it; nearer in-focus surfaces are not +/// smeared by the blur of what is behind them. +pub fn depth_of_field(img: &Image, depth: &[f32], dof: &Dof, scale: f64) -> Image { + let (w, h) = (img.width as i64, img.height as i64); + let max_r = 48.0f32; + let radius: Vec = depth + .iter() + .map(|&z| { + let z = if z.is_finite() { z as f64 } else { 1.0e9 }; + ((dof.coc(z) * scale * 0.5) as f32).min(max_r) + }) + .collect(); + let rmax = radius.iter().fold(0.0f32, |a, &b| a.max(b)); + if rmax < 0.5 { + return img.clone(); + } + // Disk samples (golden-angle spiral) over the largest radius. + const N: usize = 64; + let golden = std::f32::consts::PI * (3.0 - 5.0f32.sqrt()); + let taps: Vec<(f32, f32, f32)> = (1..=N) + .map(|i| { + let r = (i as f32 / N as f32).sqrt() * rmax; + let a = i as f32 * golden; + (r * a.cos(), r * a.sin(), r) + }) + .collect(); + let mut out = Image::new(img.width, img.height); + out.data.par_chunks_mut(w as usize).enumerate().for_each(|(y, row)| { + for x in 0..w as usize { + let i = y * w as usize + x; + let (rp, zp) = (radius[i], depth[i]); + let cw = 1.0 / (std::f32::consts::PI * rp * rp).max(1.0); + let c = img.data[i]; + let mut acc = [c[0] * cw, c[1] * cw, c[2] * cw, c[3] * cw]; + let mut wsum = cw; + for &(dx, dy, d) in &taps { + let (sx, sy) = (x as i64 + dx.round() as i64, y as i64 + dy.round() as i64); + if sx < 0 || sy < 0 || sx >= w || sy >= h { + continue; + } + let j = (sy * w + sx) as usize; + // A sample behind this pixel blurs no further than this pixel's own circle. + let rq = if depth[j] > zp { radius[j].min(rp) } else { radius[j] }; + let cov = (rq - d + 0.5).clamp(0.0, 1.0); + if cov <= 0.0 { + continue; + } + // Each tap stands for an annulus of area ≈ π·rmax²/N. + let wq = cov / (std::f32::consts::PI * rq * rq).max(1.0) * (std::f32::consts::PI * rmax * rmax / N as f32); + let q = img.data[j]; + for k in 0..4 { + acc[k] += q[k] * wq; + } + wsum += wq; + } + row[x] = acc.map(|v| v / wsum); + } + }); + out +} + +/// Linear → sRGB-encoded premultiplied pixels (the working encoding of non-linear projects). +pub fn encode(img: &mut Image) { + img.data.par_iter_mut().for_each(|p| { + let a = p[3]; + if a > 1e-6 { + for c in 0..3 { + p[c] = shade::linear_to_srgb(p[c] / a) * a; + } + } + }); +} + +/// Render a scene on the CPU, resolved: (premultiplied image at output size, depth). +pub fn render_cpu(s: &Scene) -> (Image, Vec) { + resolve(&raster::render(s), s.ssaa.max(1)) +} + +/// Build the scene of a run (for tests and tools). +pub fn scene_of(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Scene { + scene::build(r, ctx, run, out) +} + +/// Draw a run of 3D layers with the Advanced 3D renderer into `canvas`. +pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut Image) { + let s = scene::build(r, ctx, run, (canvas.width, canvas.height)); + if s.indices.is_empty() { + return; + } + let target = r.active_accel().and_then(|a| a.raster_3d(&s)).filter(|t| t.width == s.width && t.height == s.height).unwrap_or_else(|| raster::render(&s)); + let (mut img, depth) = resolve(&target, s.ssaa.max(1)); + let cam = camera_for(r, ctx); + if let Some(dof) = cam.dof.filter(|_| !ctx.comp.draft_3d && !r.opts.draft) { + img = depth_of_field(&img, &depth, &dof, r.opts.scale); + } + if !s.linear_io { + encode(&mut img); + } + canvas.data.par_iter_mut().zip(img.data.par_iter()).for_each(|(d, s)| { + let k = 1.0 - s[3]; + *d = [s[0] + d[0] * k, s[1] + d[1] * k, s[2] + d[2] * k, s[3] + d[3] * k]; + }); +} + +#[cfg(test)] +mod tests; diff --git a/crates/render/src/three_d/adv/raster.rs b/crates/render/src/three_d/adv/raster.rs new file mode 100644 index 00000000..972a627b --- /dev/null +++ b/crates/render/src/three_d/adv/raster.rs @@ -0,0 +1,492 @@ +//! The software rasteriser of the Advanced 3D renderer (the CPU reference and the fallback +//! when no GPU is available) and the shadow-map rasteriser shared by both paths. +//! +//! Triangles are clipped against the depth range (`0 ≤ z ≤ w`, as GPUs do), set up in screen +//! space and scanned per band of rows in parallel with edge functions (pixel centres, top-left +//! fill rule) and perspective-correct barycentrics. Opaque triangles resolve visibility first +//! (reversed depth: larger ndc z is nearer) and each pixel is shaded once; transparent triangles +//! (already sorted back to front) are then shaded per fragment and blended over, depth-tested +//! against the opaque surfaces. + +use rayon::prelude::*; + +use super::scene::{Scene, ShadowMap, Vertex}; +use super::shade::{Frag, sample, shade}; + +/// A rendered target at raster size. +pub struct Target { + pub width: u32, + pub height: u32, + /// Premultiplied linear RGBA. + pub color: Vec<[f32; 4]>, + /// Camera-space depth of the nearest opaque surface (∞ where empty). + pub depth: Vec, +} + +/// A clipped triangle in screen space; `w` holds, per corner, its weights over the original +/// triangle's three vertices. +#[derive(Clone, Copy)] +struct STri { + p: [[f64; 2]; 3], + z: [f32; 3], + inv_w: [f32; 3], + w: [[f32; 3]; 3], + tri: u32, + front: bool, + bbox: [i32; 4], +} + +#[inline] +fn mat_mul(m: &[[f32; 4]; 4], p: [f32; 3]) -> [f32; 4] { + let mut o = [0.0; 4]; + for r in 0..4 { + o[r] = m[r][0] * p[0] + m[r][1] * p[1] + m[r][2] * p[2] + m[r][3]; + } + o +} + +/// Clip a polygon (clip coords + original weights) against `0 ≤ z ≤ w`. +fn clip_poly(poly: Vec<([f32; 4], [f32; 3])>) -> Vec<([f32; 4], [f32; 3])> { + let mut p = poly; + for plane in 0..2 { + if p.is_empty() { + break; + } + let d = |c: &[f32; 4]| if plane == 0 { c[2] } else { c[3] - c[2] }; + let mut out = Vec::with_capacity(p.len() + 2); + for i in 0..p.len() { + let (a, b) = (p[i], p[(i + 1) % p.len()]); + let (da, db) = (d(&a.0), d(&b.0)); + if da >= 0.0 { + out.push(a); + } + if (da >= 0.0) != (db >= 0.0) { + let t = da / (da - db); + let mut c = [0.0; 4]; + let mut w = [0.0; 3]; + for k in 0..4 { + c[k] = a.0[k] + (b.0[k] - a.0[k]) * t; + } + for k in 0..3 { + w[k] = a.1[k] + (b.1[k] - a.1[k]) * t; + } + out.push((c, w)); + } + } + p = out; + } + p +} + +/// Set up the screen triangles of index range `range`. +fn setup(s: &Scene, clip: &[[f32; 4]], range: std::ops::Range) -> Vec { + let (w, h) = (s.width as f64, s.height as f64); + let mut out = vec![]; + for t in range.step_by(3) { + let ids = [s.indices[t] as usize, s.indices[t + 1] as usize, s.indices[t + 2] as usize]; + let mat = s.vertices[ids[0]].material as usize; + if !s.materials.get(mat).is_some_and(|m| m.visible) { + continue; + } + let poly = vec![(clip[ids[0]], [1.0, 0.0, 0.0]), (clip[ids[1]], [0.0, 1.0, 0.0]), (clip[ids[2]], [0.0, 0.0, 1.0])]; + let poly = clip_poly(poly); + if poly.len() < 3 { + continue; + } + let scr: Vec<([f64; 2], f32, f32, [f32; 3])> = poly + .iter() + .map(|(c, wt)| { + let iw = 1.0 / c[3]; + let (nx, ny) = (c[0] * iw, c[1] * iw); + ([(nx as f64 + 1.0) * 0.5 * w, (1.0 - ny as f64) * 0.5 * h], c[2] * iw, iw, *wt) + }) + .collect(); + // Facing from the polygon's screen-space winding (y down: negative = front). + let mut area = 0.0; + for i in 0..scr.len() { + let (a, b) = (scr[i].0, scr[(i + 1) % scr.len()].0); + area += a[0] * b[1] - b[0] * a[1]; + } + if area == 0.0 || !area.is_finite() { + continue; + } + let front = area < 0.0; + for k in 1..scr.len() - 1 { + let c = [scr[0], scr[k], scr[k + 1]]; + let mut x0 = f64::INFINITY; + let mut y0 = f64::INFINITY; + let mut x1 = f64::NEG_INFINITY; + let mut y1 = f64::NEG_INFINITY; + for v in &c { + x0 = x0.min(v.0[0]); + y0 = y0.min(v.0[1]); + x1 = x1.max(v.0[0]); + y1 = y1.max(v.0[1]); + } + let bbox = [ + (x0.floor() as i64).max(0) as i32, + (y0.floor() as i64).max(0) as i32, + (x1.ceil() as i64).min(s.width as i64) as i32, + (y1.ceil() as i64).min(s.height as i64) as i32, + ]; + if bbox[0] >= bbox[2] || bbox[1] >= bbox[3] { + continue; + } + out.push(STri { + p: [c[0].0, c[1].0, c[2].0], + z: [c[0].1, c[1].1, c[2].1], + inv_w: [c[0].2, c[1].2, c[2].2], + w: [c[0].3, c[1].3, c[2].3], + tri: (t / 3) as u32, + front, + bbox, + }); + } + } + out +} + +/// Coverage of a pixel centre: (ndc z, perspective-correct weights over the screen corners). +#[inline] +fn cover(t: &STri, px: f64, py: f64) -> Option<(f32, [f32; 3])> { + let [a, b, c] = t.p; + let area = (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]); + // Orient so the area is positive. + let (b, c, swap) = if area < 0.0 { (c, b, true) } else { (b, c, false) }; + let area = area.abs(); + let edge = |p: [f64; 2], q: [f64; 2]| -> f64 { (q[0] - p[0]) * (py - p[1]) - (q[1] - p[1]) * (px - p[0]) }; + let top_left = |p: [f64; 2], q: [f64; 2]| -> bool { + let (dx, dy) = (q[0] - p[0], q[1] - p[1]); + dy < 0.0 || (dy == 0.0 && dx > 0.0) + }; + let e0 = edge(b, c); // opposite a + let e1 = edge(c, a); // opposite b + let e2 = edge(a, b); // opposite c + let inside = |e: f64, p: [f64; 2], q: [f64; 2]| e > 0.0 || (e == 0.0 && top_left(p, q)); + if !(inside(e0, b, c) && inside(e1, c, a) && inside(e2, a, b)) { + return None; + } + let (l0, l1, l2) = ((e0 / area) as f32, (e1 / area) as f32, (e2 / area) as f32); + // Back to the stored corner order. + let l = if swap { [l0, l2, l1] } else { [l0, l1, l2] }; + let z = l[0] * t.z[0] + l[1] * t.z[1] + l[2] * t.z[2]; + let q = [l[0] * t.inv_w[0], l[1] * t.inv_w[1], l[2] * t.inv_w[2]]; + let sum = q[0] + q[1] + q[2]; + if sum <= 0.0 { + return None; + } + Some((z, [q[0] / sum, q[1] / sum, q[2] / sum])) +} + +/// Interpolate the original triangle's vertex attributes at screen-corner weights `l`. +fn frag(s: &Scene, t: &STri, l: [f32; 3]) -> Frag { + let base = t.tri as usize * 3; + let v: [&Vertex; 3] = [0, 1, 2].map(|k| &s.vertices[s.indices[base + k] as usize]); + // Weights over the original vertices. + let mut wv = [0.0f32; 3]; + for c in 0..3 { + for k in 0..3 { + wv[k] += l[c] * t.w[c][k]; + } + } + let mut f = Frag { pos: [0.0; 3], normal: [0.0; 3], uv: [0.0; 2], tangent: [0.0; 4], material: v[0].material, front: t.front }; + for k in 0..3 { + let x = wv[k]; + for i in 0..3 { + f.pos[i] += v[k].pos[i] * x; + f.normal[i] += v[k].normal[i] * x; + } + for i in 0..2 { + f.uv[i] += v[k].uv[i] * x; + } + for i in 0..4 { + f.tangent[i] += v[k].tangent[i] * x; + } + } + f.tangent[3] = v[0].tangent[3]; + f +} + +const BAND: i32 = 16; + +/// Rasterise and shade the scene. +pub fn render(s: &Scene) -> Target { + let (w, h) = (s.width as usize, s.height as usize); + let clip: Vec<[f32; 4]> = s.vertices.iter().map(|v| mat_mul(&s.clip, v.pos)).collect(); + let opaque = setup(s, &clip, 0..s.opaque_count as usize); + let trans = setup(s, &clip, s.opaque_count as usize..s.indices.len()); + let bands = (h as i32 + BAND - 1) / BAND; + let bin = |ts: &[STri]| -> Vec> { + let mut b = vec![vec![]; bands.max(0) as usize]; + for (i, t) in ts.iter().enumerate() { + for band in (t.bbox[1] / BAND)..((t.bbox[3] + BAND - 1) / BAND).min(bands) { + b[band as usize].push(i as u32); + } + } + b + }; + let ob = bin(&opaque); + let tb = bin(&trans); + let view = s.view; + let cam_z = |p: [f32; 3]| view[2][0] * p[0] + view[2][1] * p[1] + view[2][2] * p[2] + view[2][3]; + let mut color = vec![[0.0f32; 4]; w * h]; + let mut depth = vec![f32::INFINITY; w * h]; + color.par_chunks_mut(w * BAND as usize).zip(depth.par_chunks_mut(w * BAND as usize)).enumerate().for_each(|(band, (crow, drow))| { + let y0 = band as i32 * BAND; + let rows = crow.len() / w; + // Visibility: (screen tri, ndc z, weights). + let mut vis: Vec<(u32, f32, [f32; 3])> = vec![(u32::MAX, 0.0, [0.0; 3]); rows * w]; + for &ti in &ob[band] { + let t = &opaque[ti as usize]; + for y in t.bbox[1].max(y0)..t.bbox[3].min(y0 + rows as i32) { + let r = (y - y0) as usize * w; + for x in t.bbox[0]..t.bbox[2] { + if let Some((z, l)) = cover(t, x as f64 + 0.5, y as f64 + 0.5) { + let v = &mut vis[r + x as usize]; + if z > v.1 && z <= 1.0 { + *v = (ti, z, l); + } + } + } + } + } + // A discarded opaque fragment (alpha test, back face) lets the next nearer-than-nothing + // surface through only on a second look; keep it simple: resolve again without it. + for i in 0..rows * w { + let (ti, _, l) = vis[i]; + if ti == u32::MAX { + continue; + } + let t = &opaque[ti as usize]; + let f = frag(s, t, l); + match shade(s, &f) { + Some(c) => { + crow[i] = c; + drow[i] = cam_z(f.pos); + } + None => { + // Find the nearest surviving fragment under this pixel. + let (x, y) = ((i % w) as f64 + 0.5, (y0 as usize + i / w) as f64 + 0.5); + let mut best: Option<(f32, [f32; 4], f32)> = None; + for &tj in &ob[band] { + if tj == ti { + continue; + } + let t2 = &opaque[tj as usize]; + if (x as i32) < t2.bbox[0] || (x as i32) >= t2.bbox[2] || (y as i32) < t2.bbox[1] || (y as i32) >= t2.bbox[3] { + continue; + } + if let Some((z, l2)) = cover(t2, x, y) + && best.is_none_or(|b| z > b.0) + { + let f2 = frag(s, t2, l2); + if let Some(c) = shade(s, &f2) { + best = Some((z, c, cam_z(f2.pos))); + } + } + } + if let Some((z, c, d)) = best { + crow[i] = c; + drow[i] = d; + vis[i].1 = z; + } else { + vis[i].1 = 0.0; + } + } + } + } + // Transparent fragments in order, over the opaque result. + for &ti in &tb[band] { + let t = &trans[ti as usize]; + for y in t.bbox[1].max(y0)..t.bbox[3].min(y0 + rows as i32) { + let r = (y - y0) as usize * w; + for x in t.bbox[0]..t.bbox[2] { + let i = r + x as usize; + if let Some((z, l)) = cover(t, x as f64 + 0.5, y as f64 + 0.5) + && z > vis[i].1 + && z <= 1.0 + && let Some(c) = shade(s, &frag(s, t, l)) + { + let d = crow[i]; + let k = 1.0 - c[3]; + crow[i] = [c[0] + d[0] * k, c[1] + d[1] * k, c[2] + d[2] * k, c[3] + d[3] * k]; + } + } + } + } + }); + Target { width: s.width, height: s.height, color, depth } +} + +// ---------------------------------------------------------------- shadow maps + +fn basis(eye: [f64; 3], fwd: [f64; 3], down: [f64; 3]) -> [[f32; 4]; 3] { + use effectcraft_geom::vec3; + let m = crate::three_d::camera::basis_view(vec3(eye[0], eye[1], eye[2]), vec3(fwd[0], fwd[1], fwd[2]), vec3(down[0], down[1], down[2])); + [0, 1, 2].map(|r| m.0[r].map(|v| v as f32)) +} + +/// Rasterise the shadow-casting triangles' light-space depth into `m`'s texels. +fn shadow_raster(s: &Scene, m: &ShadowMap, casters: &[usize]) -> Vec { + let n = m.size as usize; + let mut d = vec![f32::INFINITY; n * n]; + let c = n as f32 * 0.5; + const NEAR: f32 = 1.0; + for &t in casters { + let ids = [s.indices[3 * t] as usize, s.indices[3 * t + 1] as usize, s.indices[3 * t + 2] as usize]; + let mat = &s.materials[s.vertices[ids[0]].material as usize]; + let alpha_tex = (mat.alpha_mode != 0 && mat.tex_base >= 0).then(|| &s.textures[mat.tex_base as usize]); + // Light space, clipped against the near plane (perspective). + let q: Vec<([f32; 3], [f32; 2])> = ids + .iter() + .map(|&i| { + let p = s.vertices[i].pos; + let v = &m.view; + ([0, 1, 2].map(|r| v[r][0] * p[0] + v[r][1] * p[1] + v[r][2] * p[2] + v[r][3]), s.vertices[i].uv) + }) + .collect(); + let mut poly = q; + if !m.ortho { + let mut out = vec![]; + for i in 0..poly.len() { + let (a, b) = (poly[i], poly[(i + 1) % poly.len()]); + let (da, db) = (a.0[2] - NEAR, b.0[2] - NEAR); + if da >= 0.0 { + out.push(a); + } + if (da >= 0.0) != (db >= 0.0) { + let k = da / (da - db); + let lerp = |x: f32, y: f32| x + (y - x) * k; + out.push(([lerp(a.0[0], b.0[0]), lerp(a.0[1], b.0[1]), lerp(a.0[2], b.0[2])], [lerp(a.1[0], b.1[0]), lerp(a.1[1], b.1[1])])); + } + } + poly = out; + } + if poly.len() < 3 { + continue; + } + // Screen: (x, y, 1/z or 1, depth) per corner. + let scr: Vec<([f64; 2], f32, f32, [f32; 2])> = poly + .iter() + .map(|(p, uv)| { + if m.ortho { + ([(p[0] * m.focal + c) as f64, (p[1] * m.focal + c) as f64], 1.0, p[2], *uv) + } else { + let iz = 1.0 / p[2]; + ([(p[0] * m.focal * iz + c) as f64, (p[1] * m.focal * iz + c) as f64], iz, p[2], *uv) + } + }) + .collect(); + for k in 1..scr.len() - 1 { + let tri = [scr[0], scr[k], scr[k + 1]]; + let (a, b, cc) = (tri[0].0, tri[1].0, tri[2].0); + let area = (b[0] - a[0]) * (cc[1] - a[1]) - (b[1] - a[1]) * (cc[0] - a[0]); + if area.abs() < 1e-12 { + continue; + } + let x0 = a[0].min(b[0]).min(cc[0]).floor().max(0.0) as usize; + let x1 = (a[0].max(b[0]).max(cc[0]).ceil().max(0.0) as usize).min(n); + let y0 = a[1].min(b[1]).min(cc[1]).floor().max(0.0) as usize; + let y1 = (a[1].max(b[1]).max(cc[1]).ceil().max(0.0) as usize).min(n); + for y in y0..y1 { + for x in x0..x1 { + let (px, py) = (x as f64 + 0.5, y as f64 + 0.5); + let e0 = ((cc[0] - b[0]) * (py - b[1]) - (cc[1] - b[1]) * (px - b[0])) / area; + let e1 = ((a[0] - cc[0]) * (py - cc[1]) - (a[1] - cc[1]) * (px - cc[0])) / area; + let e2 = 1.0 - e0 - e1; + if e0 < 0.0 || e1 < 0.0 || e2 < 0.0 { + continue; + } + let l = [e0 as f32, e1 as f32, e2 as f32]; + let (z, uv) = if m.ortho { + (l[0] * tri[0].2 + l[1] * tri[1].2 + l[2] * tri[2].2, [0, 1].map(|i| l[0] * tri[0].3[i] + l[1] * tri[1].3[i] + l[2] * tri[2].3[i])) + } else { + let iz = l[0] * tri[0].1 + l[1] * tri[1].1 + l[2] * tri[2].1; + let z = 1.0 / iz; + (z, [0, 1].map(|i| (l[0] * tri[0].3[i] * tri[0].1 + l[1] * tri[1].3[i] * tri[1].1 + l[2] * tri[2].3[i] * tri[2].1) * z)) + }; + let px = &mut d[y * n + x]; + if z >= *px { + continue; + } + if let Some(t) = alpha_tex + && sample(t, &s.texels, uv[0], uv[1])[3] * mat.base[3] * mat.opacity < 0.5 + { + continue; + } + *px = z; + } + } + } + } + d +} + +/// Build the shadow maps of every shadow-casting light (lights marked `shadow == 0`). +pub(crate) fn shadow_maps(s: &mut Scene, draft: bool) { + let casters: Vec = (0..s.indices.len() / 3) + .filter(|&t| s.materials.get(s.vertices[s.indices[3 * t] as usize].material as usize).is_some_and(|m| m.casts_shadows)) + .collect(); + if casters.is_empty() { + for l in &mut s.lights { + l.shadow = -1; + } + return; + } + // Bounding sphere of the scene (parallel lights' orthographic frustum). + let (mut lo, mut hi) = ([f64::INFINITY; 3], [f64::NEG_INFINITY; 3]); + for v in &s.vertices { + for k in 0..3 { + lo[k] = lo[k].min(v.pos[k] as f64); + hi[k] = hi[k].max(v.pos[k] as f64); + } + } + let centre = [0, 1, 2].map(|k| (lo[k] + hi[k]) * 0.5); + let radius = ((hi[0] - lo[0]).powi(2) + (hi[1] - lo[1]).powi(2) + (hi[2] - lo[2]).powi(2)).sqrt() * 0.5 + 1.0; + let size = if draft { 512 } else { 1024 }; + let mut maps: Vec = vec![]; + for li in 0..s.lights.len() { + if s.lights[li].shadow != 0 { + s.lights[li].shadow = -1; + continue; + } + let l = s.lights[li]; + let first = maps.len() as i32; + let soft = 1.0 + (l.shadow_diffusion * 0.5).min(24.0); + let pos = l.pos.map(|v| v as f64); + let dir = l.dir.map(|v| v as f64); + let hint = if dir[1].abs() < 0.9 { [0.0, 1.0, 0.0] } else { [0.0, 0.0, 1.0] }; + match l.kind { + 0 => { + let eye = [centre[0] - dir[0] * radius * 2.0, centre[1] - dir[1] * radius * 2.0, centre[2] - dir[2] * radius * 2.0]; + let focal = (size as f64 * 0.5 / radius) as f32; + maps.push(ShadowMap { view: basis(eye, dir, hint), focal, ortho: true, size, offset: 0, bias: 1.5, radius: soft }); + } + 1 => { + let half = (l.cos_outer.clamp(-1.0, 1.0).acos() as f64 + 0.05).min(1.45); + let focal = (size as f64 * 0.5 / half.tan()) as f32; + maps.push(ShadowMap { view: basis(pos, dir, hint), focal, ortho: false, size, offset: 0, bias: 1.5, radius: soft }); + } + _ => { + let fs = size / 2; + for (f, d) in [ + ([1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([0.0, 1.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, -1.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), + ([0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), + ] { + maps.push(ShadowMap { view: basis(pos, f, d), focal: fs as f32 * 0.5, ortho: false, size: fs, offset: 0, bias: 1.5, radius: soft }); + } + } + } + s.lights[li].shadow = first; + } + let texels: Vec> = maps.par_iter().map(|m| shadow_raster(s, m, &casters)).collect(); + for (m, t) in maps.iter_mut().zip(texels) { + m.offset = s.shadow_texels.len() as u32; + s.shadow_texels.extend(t); + } + s.shadows = maps; +} diff --git a/crates/render/src/three_d/adv/scene.rs b/crates/render/src/three_d/adv/scene.rs new file mode 100644 index 00000000..14251745 --- /dev/null +++ b/crates/render/src/three_d/adv/scene.rs @@ -0,0 +1,875 @@ +//! The Advanced 3D scene: everything a rasteriser needs, flattened into world-space triangles +//! and plain arrays (identical input for the CPU rasteriser and the GPU pipeline). +//! +//! - Model and primitive layers: their meshes (glTF/OBJ models are +Y up in model units and +//! are mapped into the layer by `Model Scale` with Y and Z flipped, a 180° turn about X). +//! - Extruded text and shape layers (Geometry Options ▸ Extrusion Depth > 0): bevelled meshes. +//! - Every other 3D layer (footage, solids, comps, flat text and shapes): a textured card of +//! its processed buffer, lit with its Material Options. +//! - Lights from the comp's light layers, the Environment light's image (prefiltered for image +//! based lighting), and shadow maps of the shadow-casting lights. +//! +//! Colours are linear light; textures and layer buffers are linearised from the sRGB curve +//! unless the project already blends in linear light. + +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_effects::Buf; +use effectcraft_geom::{Mat4, Vec3, vec3}; +use effectcraft_model::{self as model, Primitive}; +use effectcraft_project::{ItemKind, Layer, LayerSource, LightKind, PrimitiveKind}; + +use super::shade::srgb_to_linear; +use crate::three_d::camera::{CameraState, NEAR}; +use crate::three_d::compose::camera_for; +use crate::three_d::light::{LightState, Material as ClassicMaterial}; +use crate::{EvalCtx, Renderer}; + +/// One vertex in world space. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[repr(C)] +pub struct Vertex { + pub pos: [f32; 3], + pub normal: [f32; 3], + pub uv: [f32; 2], + pub tangent: [f32; 4], + pub material: u32, +} + +/// A material, resolved for shading. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Material { + /// Linear RGBA factor. + pub base: [f32; 4], + pub metallic: f32, + pub roughness: f32, + pub emissive: [f32; 3], + pub normal_scale: f32, + pub occlusion_strength: f32, + /// 0 opaque, 1 mask, 2 blend. + pub alpha_mode: u32, + pub alpha_cutoff: f32, + /// Layer opacity (multiplies alpha). + pub opacity: f32, + /// Texture indices (−1 = none). + pub tex_base: i32, + pub tex_mr: i32, + pub tex_normal: i32, + pub tex_occlusion: i32, + pub tex_emissive: i32, + pub double_sided: bool, + pub unlit: bool, + pub accepts_lights: bool, + pub receives_shadows: bool, + pub casts_shadows: bool, + /// Drawn (false for "Casts Shadows: Only"). + pub visible: bool, + /// Material Options weights (cards: Diffuse, Specular Intensity / 50%, Ambient). + pub diffuse_k: f32, + pub specular_k: f32, + pub ambient_k: f32, +} + +impl Default for Material { + fn default() -> Self { + Material { + base: [1.0; 4], + metallic: 0.0, + roughness: 0.5, + emissive: [0.0; 3], + normal_scale: 1.0, + occlusion_strength: 1.0, + alpha_mode: 0, + alpha_cutoff: 0.5, + opacity: 1.0, + tex_base: -1, + tex_mr: -1, + tex_normal: -1, + tex_occlusion: -1, + tex_emissive: -1, + double_sided: false, + unlit: false, + accepts_lights: true, + receives_shadows: true, + casts_shadows: false, + visible: true, + diffuse_k: 1.0, + specular_k: 1.0, + ambient_k: 1.0, + } + } +} + +impl Material { + /// Drawn in the transparent (sorted, blended) pass. + pub fn transparent(&self) -> bool { + self.alpha_mode == 2 || self.opacity < 0.999 + } +} + +/// A texture in [`Scene::texels`]: linear straight RGBA. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct TexInfo { + pub offset: u32, + pub width: u32, + pub height: u32, + /// 0 repeat, 1 clamp, 2 mirror. + pub wrap_u: u32, + pub wrap_v: u32, +} + +/// A light (ambient lights are summed into [`Scene::ambient`]). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Light { + /// 0 parallel, 1 spot, 2 point. + pub kind: u32, + pub pos: [f32; 3], + /// Direction the light travels. + pub dir: [f32; 3], + /// Linear colour × intensity. + pub color: [f32; 3], + pub cos_inner: f32, + pub cos_outer: f32, + /// 0 none, 1 smooth, 2 inverse square clamped. + pub falloff: u32, + pub radius: f32, + pub falloff_distance: f32, + /// First shadow map (point lights use six), −1 = none. + pub shadow: i32, + pub shadow_darkness: f32, + /// Shadow Diffusion (pixels). + pub shadow_diffusion: f32, +} + +/// A depth map seen from a light: `view` maps world → light space (+z forward); pixels +/// `u = focal·x/z + size/2` (perspective) or `focal·x + size/2` (orthographic); texels store +/// the light-space depth of the nearest caster. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ShadowMap { + pub view: [[f32; 4]; 3], + pub focal: f32, + pub ortho: bool, + pub size: u32, + pub offset: u32, + /// Depth bias in texels (scaled by the texel footprint at the receiver). + pub bias: f32, + /// PCF spread (texels). + pub radius: f32, +} + +/// The environment image: radiance mip levels `radiance..radiance + mips` and an irradiance +/// map, in [`Scene::textures`]. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EnvInfo { + pub radiance: u32, + pub mips: u32, + pub irradiance: u32, + pub intensity: f32, + /// Radians about the vertical axis. + pub rotation: f32, +} + +#[derive(Clone, Debug, Default)] +pub struct Scene { + /// Raster size (output × `ssaa`). + pub width: u32, + pub height: u32, + pub ssaa: u32, + /// World → clip (row-major; reversed depth: ndc z = near / camera z). + pub clip: [[f32; 4]; 4], + /// World → camera (row-major; camera z = depth for depth of field). + pub view: [[f32; 4]; 4], + pub eye: [f32; 3], + pub cam_fwd: [f32; 3], + pub ortho: bool, + pub vertices: Vec, + /// Triangles: the first `opaque_count` indices are opaque, the rest transparent, sorted + /// back to front. + pub indices: Vec, + pub opaque_count: u32, + pub materials: Vec, + pub textures: Vec, + pub texels: Vec<[f32; 4]>, + pub lights: Vec, + pub shadows: Vec, + pub shadow_texels: Vec, + pub env: Option, + pub ambient: [f32; 3], + /// Shading inputs and outputs are linear (no sRGB conversion needed). + pub linear_io: bool, +} + +/// Row-major f32 copy of a matrix. +pub fn m32(m: &Mat4) -> [[f32; 4]; 4] { + m.0.map(|r| r.map(|v| v as f32)) +} + +/// Builds a [`Scene`]: texture uploads deduplicated by source. +pub(crate) struct Builder { + pub s: Scene, + tex_keys: HashMap<(usize, bool), i32>, + /// Per triangle: transparent and its sort depth. + tri_depth: Vec<(bool, f32)>, + cam: CameraState, +} + +impl Builder { + fn texture_from(&mut self, w: u32, h: u32, wrap: [u32; 2], data: impl Iterator) -> i32 { + let offset = self.s.texels.len() as u32; + self.s.texels.extend(data); + self.s.textures.push(TexInfo { offset, width: w, height: h, wrap_u: wrap[0], wrap_v: wrap[1] }); + (self.s.textures.len() - 1) as i32 + } + + /// Upload a model texture (`srgb`: colour data to linearise). + fn model_texture(&mut self, t: &Arc, srgb: bool) -> i32 { + let key = (Arc::as_ptr(t) as usize, srgb); + if let Some(&i) = self.tex_keys.get(&key) { + return i; + } + let lin = srgb && !self.s.linear_io; + let wrap = t.wrap.map(|w| match w { + model::Wrap::Repeat => 0, + model::Wrap::Clamp => 1, + model::Wrap::Mirror => 2, + }); + let i = self.texture_from( + t.width, + t.height, + wrap, + t.data.iter().map(|p| if lin { [srgb_to_linear(p[0]), srgb_to_linear(p[1]), srgb_to_linear(p[2]), p[3]] } else { *p }), + ); + self.tex_keys.insert(key, i); + i + } + + /// Upload a premultiplied layer buffer as a linear straight-alpha texture (clamped edges). + fn buf_texture(&mut self, b: &Buf) -> (i32, bool) { + let lin = !self.s.linear_io; + let mut opaque = true; + let data: Vec<[f32; 4]> = b + .img + .data + .iter() + .map(|p| { + let a = p[3]; + if a < 0.999 { + opaque = false; + } + if a <= 1e-6 { + return [0.0; 4]; + } + let s = [p[0] / a, p[1] / a, p[2] / a]; + if lin { [srgb_to_linear(s[0]), srgb_to_linear(s[1]), srgb_to_linear(s[2]), a] } else { [s[0], s[1], s[2], a] } + }) + .collect(); + (self.texture_from(b.img.width, b.img.height, [1, 1], data.into_iter()), opaque) + } + + fn material(&mut self, m: Material) -> u32 { + self.s.materials.push(m); + (self.s.materials.len() - 1) as u32 + } + + /// Append a primitive with per-vertex matrices (model/layer space → world). + fn emit(&mut self, p: &Primitive, mat: u32, xf: &dyn Fn(usize) -> Mat4) { + if p.positions.is_empty() || p.indices.len() < 3 { + return; + } + let base = self.s.vertices.len() as u32; + let transparent = self.s.materials[mat as usize].transparent(); + let uniform = xf(usize::MAX); + let normal_of = |m: &Mat4| -> (Mat4, f64) { + let a = &m.0; + let det = a[0][0] * (a[1][1] * a[2][2] - a[1][2] * a[2][1]) - a[0][1] * (a[1][0] * a[2][2] - a[1][2] * a[2][0]) + + a[0][2] * (a[1][0] * a[2][1] - a[1][1] * a[2][0]); + let inv = Mat4([[a[0][0], a[0][1], a[0][2], 0.0], [a[1][0], a[1][1], a[1][2], 0.0], [a[2][0], a[2][1], a[2][2], 0.0], [0.0, 0.0, 0.0, 1.0]]) + .inverse() + .unwrap_or(Mat4::IDENTITY); + (inv.transpose(), det) + }; + let (un, udet) = normal_of(&uniform); + let mut flip_count = 0usize; + for k in 0..p.positions.len() { + let (m, n, det) = if p.joints.is_empty() { + (uniform, un, udet) + } else { + let m = xf(k); + let (n, d) = normal_of(&m); + (m, n, d) + }; + if det < 0.0 { + flip_count += 1; + } + let q = p.positions[k]; + let w = m.apply(vec3(q[0] as f64, q[1] as f64, q[2] as f64)); + let nn = p.normals.get(k).copied().unwrap_or([0.0, 0.0, 1.0]); + let wn = n.apply_vec(vec3(nn[0] as f64, nn[1] as f64, nn[2] as f64)).normalize(); + let t = p.tangents.get(k).copied().unwrap_or([1.0, 0.0, 0.0, 1.0]); + let wt = m.apply_vec(vec3(t[0] as f64, t[1] as f64, t[2] as f64)).normalize(); + self.s.vertices.push(Vertex { + pos: [w.x as f32, w.y as f32, w.z as f32], + normal: [wn.x as f32, wn.y as f32, wn.z as f32], + uv: p.uvs.get(k).copied().unwrap_or([0.0, 0.0]), + tangent: [wt.x as f32, wt.y as f32, wt.z as f32, if det < 0.0 { -t[3] } else { t[3] }], + material: mat, + }); + } + // Mirroring transforms reverse the winding. + let flip = flip_count * 2 > p.positions.len(); + for t in p.indices.chunks_exact(3) { + let (a, b, c) = (t[0] + base, t[1] + base, t[2] + base); + if flip { + self.s.indices.extend([a, c, b]) + } else { + self.s.indices.extend([a, b, c]) + } + let z = [a, b, c].iter().map(|&i| self.cam.depth(Vec3::from(self.s.vertices[i as usize].pos.map(|v| v as f64)))).sum::() / 3.0; + self.tri_depth.push((transparent, z as f32)); + } + } + + /// Opaque triangles first, then transparent ones back to front. + fn sort_triangles(&mut self) { + let tris: Vec<[u32; 3]> = self.s.indices.chunks_exact(3).map(|t| [t[0], t[1], t[2]]).collect(); + let mut order: Vec = (0..tris.len()).collect(); + order.sort_by(|&a, &b| { + let (ta, za) = self.tri_depth[a]; + let (tb, zb) = self.tri_depth[b]; + ta.cmp(&tb).then_with(|| if ta { zb.total_cmp(&za) } else { std::cmp::Ordering::Equal }) + }); + self.s.opaque_count = 3 * self.tri_depth.iter().filter(|t| !t.0).count() as u32; + self.s.indices = order.into_iter().flat_map(|i| tris[i]).collect(); + } +} + +/// The Material Options of a card (classic properties). +fn card_material(ctx: &EvalCtx, layer: &Layer) -> Material { + let c = ClassicMaterial::of(ctx, layer); + Material { + roughness: (1.0 - c.shininess).clamp(0.05, 1.0), + double_sided: true, + accepts_lights: c.accepts_lights, + receives_shadows: c.accepts_shadows != 0, + casts_shadows: c.casts_shadows != 0, + visible: c.casts_shadows != 2, + diffuse_k: c.diffuse, + specular_k: c.specular * 2.0, + ambient_k: c.ambient, + ..Material::default() + } +} + +/// Shadow/light switches and the primitive material of a model or primitive layer. +fn model_options(ctx: &EvalCtx, layer: &Layer, lin: bool) -> (Material, Option) { + let g = layer.props.sub("materialOptions"); + let e = |m: &str, d: u32| g.and_then(|g| g.get(m)).map_or(d, |p| ctx.value(layer, p).as_enum()); + let b = |m: &str| g.and_then(|g| g.get(m)).is_none_or(|p| ctx.value(layer, p).as_bool()); + let casts = e("castsShadows", 0); + let opts = Material { + accepts_lights: b("acceptsLights"), + receives_shadows: e("acceptsShadows", 1) != 0, + casts_shadows: casts != 0, + visible: casts != 2, + ..Material::default() + }; + let prim = match (layer.source.clone(), g) { + (LayerSource::Primitive { .. }, Some(g)) => { + let c = ctx.color(layer, g, "baseColor"); + let em = ctx.color(layer, g, "emissive"); + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + let mut m = model::Material::plain( + [l(c[0]), l(c[1]), l(c[2]), 1.0], + (ctx.f(layer, g, "metallic", 0.0) / 100.0).clamp(0.0, 1.0) as f32, + (ctx.f(layer, g, "roughness", 50.0) / 100.0).clamp(0.0, 1.0) as f32, + ); + m.emissive = [l(em[0]), l(em[1]), l(em[2])]; + Some(m) + } + _ => None, + }; + (opts, prim) +} + +fn prim_kind(k: PrimitiveKind) -> model::prim::PrimitiveKind { + use model::prim::PrimitiveKind as M; + match k { + PrimitiveKind::Cube => M::Cube, + PrimitiveKind::Sphere => M::Sphere, + PrimitiveKind::Plane => M::Plane, + PrimitiveKind::Torus => M::Torus, + PrimitiveKind::Cone => M::Cone, + PrimitiveKind::Cylinder => M::Cylinder, + } +} + +/// The primitive model of a primitive layer at the context time. +pub fn primitive_model(ctx: &EvalCtx, layer: &Layer, kind: PrimitiveKind, mat: model::Material) -> model::Model { + let g = layer.props.sub("geometryOptions"); + let f = |m: &str, d: f64| g.map_or(d, |g| ctx.f(layer, g, m, d)); + let pr = model::prim::PrimParams { + size: [f("width", 300.0) as f32, f("height", 300.0) as f32, f("depth", 300.0) as f32], + radius: f("radius", 150.0) as f32, + tube: f("tubeRadius", 50.0) as f32, + height: f("height", 300.0) as f32, + segments: f("segments", 48.0).round().clamp(3.0, 512.0) as u32, + rings: f("rings", 24.0).round().clamp(2.0, 512.0) as u32, + }; + model::prim::model(prim_kind(kind), &pr, mat) +} + +/// Model space → layer space: model units to pixels, Y up → Y down (a 180° turn about X). +pub fn model_to_layer(unit_scale: f64) -> Mat4 { + Mat4::scale(vec3(unit_scale, -unit_scale, -unit_scale)) +} + +fn model_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) { + let lin = b.s.linear_io; + let (opts, prim_mat) = model_options(ctx, layer, lin); + let opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let geo = layer.props.sub("geometryOptions"); + let (mdl, unit, clip, t): (Arc, f64, Option, f64) = match &layer.source { + LayerSource::Primitive { kind } => (Arc::new(primitive_model(ctx, layer, *kind, prim_mat.unwrap_or_default())), 1.0, None, 0.0), + LayerSource::Model { item } => { + let Some(it) = r.project.item(*item) else { return }; + let ItemKind::Footage(f) = &it.kind else { return }; + let Some(m) = r.footage.model(*item, f) else { return }; + let unit = geo.map_or(1.0, |g| ctx.f(layer, g, "unitScale", 1.0)); + let clip = geo.map_or(0, |g| ctx.e(layer, g, "animation")) as usize; + let clip = clip.checked_sub(1).filter(|c| *c < m.animations.len()); + let speed = geo.map_or(100.0, |g| ctx.f(layer, g, "animationSpeed", 100.0)) / 100.0; + let mut t = layer.layer_time(ctx.time).seconds() * speed; + if let Some(c) = clip { + let d = m.animations[c].duration(); + let looping = geo.is_none_or(|g| ctx.b(layer, g, "loopAnimation")); + if looping && d > 0.0 { + t = t.rem_euclid(d); + } + } + (m, unit, clip, t) + } + _ => return, + }; + let world = ctx.world_matrix(layer) * model_to_layer(unit); + // Materials of the model (one default when it has none). + let mut mats: Vec = vec![]; + let defaults = [model::Material::plain([0.8, 0.8, 0.8, 1.0], 0.0, 0.5)]; + let src: &[model::Material] = if mdl.materials.is_empty() { &defaults } else { &mdl.materials }; + for mm in src { + let tex = |b: &mut Builder, t: Option, srgb: bool| -> i32 { + t.and_then(|t| mdl.textures.get(t.texture)).map_or(-1, |t| b.model_texture(t, srgb)) + }; + let m = Material { + base: mm.base_color, + metallic: mm.metallic, + roughness: mm.roughness, + emissive: mm.emissive, + normal_scale: mm.normal_scale, + occlusion_strength: mm.occlusion_strength, + alpha_mode: match mm.alpha_mode { + model::AlphaMode::Opaque => 0, + model::AlphaMode::Mask => 1, + model::AlphaMode::Blend => 2, + }, + alpha_cutoff: mm.alpha_cutoff, + opacity, + tex_base: tex(b, mm.base_color_tex, true), + tex_mr: tex(b, mm.metallic_roughness_tex, false), + tex_normal: tex(b, mm.normal_tex, false), + tex_occlusion: tex(b, mm.occlusion_tex, false), + tex_emissive: tex(b, mm.emissive_tex, true), + double_sided: mm.double_sided, + unlit: mm.unlit, + ..opts + }; + mats.push(b.material(m)); + } + for inst in mdl.instances(clip, t) { + for p in &inst.mesh.primitives { + let mat = mats[p.material.unwrap_or(0).min(mats.len() - 1)]; + b.emit(p, mat, &|k| if k == usize::MAX { world * inst.world } else { world * model::vertex_matrix(&inst, p, k) }); + } + } +} + +/// Closed polylines of a path (flattened to `tol` pixels). +pub fn contours(path: &kurbo::BezPath, tol: f64) -> Vec> { + let mut out: Vec> = vec![]; + let mut cur: Vec<[f64; 2]> = vec![]; + kurbo::flatten(path, tol, |el| match el { + kurbo::PathEl::MoveTo(p) => { + if cur.len() >= 3 { + out.push(std::mem::take(&mut cur)); + } + cur = vec![[p.x, p.y]]; + } + kurbo::PathEl::LineTo(p) => cur.push([p.x, p.y]), + kurbo::PathEl::ClosePath => { + if cur.len() >= 3 { + out.push(std::mem::take(&mut cur)); + } + cur.clear(); + } + _ => {} + }); + if cur.len() >= 3 { + out.push(cur); + } + out +} + +/// Geometry Options of a text/shape layer when it is extruded. +pub fn extrusion_params(ctx: &EvalCtx, layer: &Layer) -> Option { + let g = layer.props.sub("geometryOptions")?; + let depth = ctx.f(layer, g, "extrusionDepth", 0.0); + (depth > 0.0).then(|| model::extrude::ExtrudeParams { + bevel: model::extrude::BevelStyle::from_index(ctx.e(layer, g, "bevelStyle")), + bevel_depth: ctx.f(layer, g, "bevelDepth", 2.0).max(0.0), + hole_bevel: (ctx.f(layer, g, "holeBevelDepth", 100.0) / 100.0).clamp(0.0, 1.0), + depth, + }) +} + +/// Extruded meshes of a text or shape layer: (mesh, straight colour, mesh space → layer +/// space). Text extrudes each character in its own space, so per-character 3D animation +/// (position, rotation, scale in Z) carries over. +pub fn extruded_meshes(ctx: &EvalCtx, layer: &Layer, p: &model::extrude::ExtrudeParams) -> Vec<(Primitive, [f32; 4], Mat4)> { + let mut groups: Vec<(Vec, [f32; 4], Mat4)> = vec![]; + match layer.source { + LayerSource::Text => { + if let Some(tg) = crate::text::text_geom(ctx, layer) { + for g in tg.glyphs.iter().filter(|g| g.apply_fill && g.fill[3] > 0.0) { + groups.push((vec![g.local.clone()], g.fill, g.m)); + } + } + } + LayerSource::Shape => { + if let Some(c) = layer.props.sub("contents") { + groups = crate::shapes::fill_outlines(ctx, layer, c).into_iter().map(|(ps, c)| (ps, c, Mat4::IDENTITY)).collect(); + } + } + _ => {} + } + groups + .into_iter() + .filter_map(|(paths, color, m)| { + let cs: Vec> = paths.iter().flat_map(|bp| contours(bp, 0.25)).collect(); + let outlines = model::extrude::group_contours(cs); + if outlines.is_empty() { + return None; + } + Some((model::extrude::extrude(&outlines, p), color, m)) + }) + .collect() +} + +fn extruded_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, p: &model::extrude::ExtrudeParams, b: &mut Builder) { + let base = card_material(ctx, layer); + let opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let world = ctx.world_matrix(layer); + let lin = b.s.linear_io; + for (mesh, c, local) in extruded_meshes(ctx, layer, p) { + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + let m = b.material(Material { + base: [l(c[0]), l(c[1]), l(c[2]), c[3]], + alpha_mode: if c[3] < 0.999 { 2 } else { 0 }, + opacity, + double_sided: false, + ..base + }); + let xf = world * local; + b.emit(&mesh, m, &|_| xf); + } +} + +fn card(b: &mut Builder, buf: &Buf, world: Mat4, mut mat: Material) { + if buf.img.width == 0 || buf.img.height == 0 { + return; + } + let (tex, opaque) = b.buf_texture(buf); + mat.tex_base = tex; + mat.alpha_mode = if opaque { 0 } else { 2 }; + let m = b.material(mat); + let s = buf.scale.max(1e-9); + let (x0, y0) = (-buf.offset[0] / s, -buf.offset[1] / s); + let (x1, y1) = ((buf.img.width as f64 - buf.offset[0]) / s, (buf.img.height as f64 - buf.offset[1]) / s); + // Faces the camera (−z) in layer space; uv (0,0) at the buffer's top-left texel corner. + let p = Primitive { + positions: vec![[x0 as f32, y0 as f32, 0.0], [x1 as f32, y0 as f32, 0.0], [x1 as f32, y1 as f32, 0.0], [x0 as f32, y1 as f32, 0.0]], + normals: vec![[0.0, 0.0, -1.0]; 4], + uvs: vec![[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]], + tangents: vec![[1.0, 0.0, 0.0, 1.0]; 4], + indices: vec![0, 2, 1, 0, 3, 2], + material: Some(0), + ..Default::default() + }; + b.emit(&p, m, &|_| world); +} + +fn card_layer(r: &Renderer, ctx: &EvalCtx, layer: &Layer, b: &mut Builder) { + let mut mat = card_material(ctx, layer); + mat.opacity = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + let world = ctx.world_matrix(layer); + if crate::text::per_char_3d(ctx, layer) { + for (buf, m) in crate::text::per_char_planes(ctx, layer, r.opts.scale) { + let buf = r.to_blend(Arc::new(buf), None); + card(b, &buf, world * m, mat); + } + return; + } + if let Some(buf) = r.blend_layer_buf(ctx, layer) { + card(b, &buf, world, mat); + } +} + +/// Evaluated lights: (scene lights, ambient sum, environment light (layer, intensity × colour, +/// rotation)). +fn lights(ctx: &EvalCtx, lin: bool) -> (Vec, [f32; 3], Option<(Option, [f32; 3], f32)>) { + let mut out = vec![]; + let mut ambient = [0.0f32; 3]; + let mut env = None; + let l = |v: f32| if lin { v } else { srgb_to_linear(v) }; + for layer in ctx.comp.layers.iter().filter(|l| l.is_light() && l.switches.video && l.is_active_at(ctx.time)) { + let LayerSource::Light { kind } = layer.source else { continue }; + let Some(g) = layer.props.sub("lightOptions") else { continue }; + let k = (ctx.f(layer, g, "intensity", 100.0) / 100.0) as f32; + let c = ctx.color(layer, g, "color"); + let color = [l(c[0]) * k, l(c[1]) * k, l(c[2]) * k]; + match kind { + LightKind::Ambient => { + for i in 0..3 { + ambient[i] += color[i]; + } + } + LightKind::Environment => { + if env.is_none() { + let src = g.get("source").and_then(|p| ctx.value(layer, p).as_layer()); + let rot = ctx.f(layer, g, "rotation", 0.0).to_radians() as f32; + env = Some((src, color, rot)); + } + } + _ => { + let Some(st) = LightState::from_layer(ctx, layer) else { continue }; + let f = |v: Vec3| [v.x as f32, v.y as f32, v.z as f32]; + out.push(Light { + kind: match kind { + LightKind::Parallel => 0, + LightKind::Spot => 1, + _ => 2, + }, + pos: f(st.pos), + dir: f(st.dir.normalize()), + color, + cos_inner: (st.cone_half * (1.0 - st.feather)).cos() as f32, + cos_outer: st.cone_half.cos() as f32, + falloff: st.falloff, + radius: st.radius as f32, + falloff_distance: st.falloff_distance as f32, + // Marked; `raster::shadow_maps` assigns the maps. + shadow: if st.casts_shadows { 0 } else { -1 }, + shadow_darkness: st.shadow_darkness as f32, + shadow_diffusion: st.shadow_diffusion as f32, + }); + } + } + } + (out, ambient, env) +} + +/// Box-downsample a straight RGBA image by 2 (odd sizes round up). +fn half(w: u32, h: u32, d: &[[f32; 4]]) -> (u32, u32, Vec<[f32; 4]>) { + let (nw, nh) = (w.div_ceil(2).max(1), h.div_ceil(2).max(1)); + let mut out = vec![[0.0f32; 4]; (nw * nh) as usize]; + for y in 0..nh { + for x in 0..nw { + let mut acc = [0.0f32; 4]; + let mut n = 0.0; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (sx, sy) = (2 * x + dx, 2 * y + dy); + if sx < w && sy < h { + let p = d[(sy * w + sx) as usize]; + for k in 0..4 { + acc[k] += p[k]; + } + n += 1.0; + } + } + out[(y * nw + x) as usize] = acc.map(|v| v / n); + } + } + (nw, nh, out) +} + +/// Cosine-weighted irradiance (`w`×`h` equirectangular) of an equirectangular radiance image: +/// each texel is the cosine-weighted mean radiance over the hemisphere around its direction +/// (a constant environment gives the same constant). +pub fn irradiance_map(sw: u32, sh: u32, src: &[[f32; 4]], w: u32, h: u32) -> Vec<[f32; 4]> { + use super::shade::{dot, equirect_dir}; + // Source directions and solid-angle weights (∝ sin θ). + let dirs: Vec<([f32; 3], f32, [f32; 3])> = (0..sh) + .flat_map(|y| (0..sw).map(move |x| (x, y))) + .map(|(x, y)| { + let (u, v) = ((x as f32 + 0.5) / sw as f32, (y as f32 + 0.5) / sh as f32); + let p = src[(y * sw + x) as usize]; + (equirect_dir(u, v, 0.0), (v * std::f32::consts::PI).sin(), [p[0], p[1], p[2]]) + }) + .collect(); + let mut out = vec![[0.0f32; 4]; (w * h) as usize]; + for y in 0..h { + for x in 0..w { + let n = equirect_dir((x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32, 0.0); + let (mut acc, mut wsum) = ([0.0f32; 3], 0.0f32); + for (d, sa, c) in &dirs { + let k = dot(n, *d).max(0.0) * sa; + if k > 0.0 { + for i in 0..3 { + acc[i] += c[i] * k; + } + wsum += k; + } + } + let inv = if wsum > 0.0 { 1.0 / wsum } else { 0.0 }; + out[(y * w + x) as usize] = [acc[0] * inv, acc[1] * inv, acc[2] * inv, 1.0]; + } + } + out +} + +/// Prefilter an environment image into the scene's textures. +pub(crate) fn add_environment(b: &mut Builder, w: u32, h: u32, data: Vec<[f32; 4]>, intensity: [f32; 3], rotation: f32) { + // Radiance level 0 at most 1024 wide. + let (mut w, mut h, mut d) = (w, h, data); + while w > 1024 { + (w, h, d) = half(w, h, &d); + } + let first = b.s.textures.len() as u32; + let mut levels = 0; + let mut small: Option<(u32, u32, Vec<[f32; 4]>)> = None; + loop { + b.texture_from(w, h, [0, 1], d.iter().copied()); + levels += 1; + if w <= 64 && small.is_none() { + small = Some((w, h, d.clone())); + } + if w <= 8 || h <= 4 { + break; + } + (w, h, d) = half(w, h, &d); + } + let (sw, sh, sd) = small.unwrap_or((w, h, d)); + let irr = irradiance_map(sw, sh, &sd, 32, 16); + let ii = b.texture_from(32, 16, [0, 1], irr.into_iter()) as u32; + // Intensity × colour folded into the radiance (one scalar on the GPU): tint the texels. + let k = (intensity[0] + intensity[1] + intensity[2]) / 3.0; + if k > 0.0 && (intensity[0] != intensity[1] || intensity[1] != intensity[2]) { + let tint = intensity.map(|c| c / k); + for t in first..=ii { + let ti = b.s.textures[t as usize]; + for p in &mut b.s.texels[ti.offset as usize..(ti.offset + ti.width * ti.height) as usize] { + for c in 0..3 { + p[c] *= tint[c]; + } + } + } + } + b.s.env = Some(EnvInfo { radiance: first, mips: levels, irradiance: ii, intensity: k, rotation }); +} + +/// The environment layer's pixels as linear straight RGBA. +fn env_pixels(r: &Renderer, ctx: &EvalCtx, src: Option, lin: bool) -> Option<(u32, u32, Vec<[f32; 4]>)> { + let layer = match src { + Some(id) => ctx.comp.layer(effectcraft_project::LayerId(id))?, + None => ctx.comp.layers.iter().find(|l| l.environment && l.is_active_at(ctx.time))?, + }; + let buf = r.blend_layer_buf(ctx, layer)?; + let img = &buf.img; + let data = img + .data + .iter() + .map(|p| { + let a = p[3]; + if a <= 1e-6 { + return [0.0, 0.0, 0.0, 1.0]; + } + let s = [p[0] / a, p[1] / a, p[2] / a]; + if lin { [s[0], s[1], s[2], 1.0] } else { [srgb_to_linear(s[0]), srgb_to_linear(s[1]), srgb_to_linear(s[2]), 1.0] } + }) + .collect(); + Some((img.width, img.height, data)) +} + +/// Supersampling factor: 2×2 unless draft or very large. +fn ssaa_for(r: &Renderer, out: (u32, u32)) -> u32 { + if r.opts.draft || (out.0 as u64 * out.1 as u64) > 8_000_000 { 1 } else { 2 } +} + +/// Build the scene of a run of 3D layers. +pub(crate) fn build(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Scene { + let cam = camera_for(r, ctx); + let lin = r.pipe.linear || r.pipe.linear_blend; + let ssaa = ssaa_for(r, out); + let mut b = Builder { + s: Scene { width: out.0 * ssaa, height: out.1 * ssaa, ssaa, linear_io: lin, ..Scene::default() }, + tex_keys: HashMap::new(), + tri_depth: vec![], + cam, + }; + for layer in run { + if layer.environment || layer.switches.adjustment || ctx.opacity(layer) <= 0.0 || !layer.has_video() { + continue; + } + if layer.source.is_model() { + model_layer(r, ctx, layer, &mut b); + } else if matches!(layer.source, LayerSource::Text | LayerSource::Shape) + && let Some(p) = extrusion_params(ctx, layer) + { + extruded_layer(r, ctx, layer, &p, &mut b); + } else { + card_layer(r, ctx, layer, &mut b); + } + } + b.sort_triangles(); + // Lights, environment, shadows (Draft 3D: unlit, like After Effects). + if !ctx.comp.draft_3d { + let (ls, amb, env) = lights(ctx, lin); + b.s.lights = ls; + b.s.ambient = amb; + if let Some((src, k, rot)) = env + && let Some((w, h, d)) = env_pixels(r, ctx, src, lin) + { + add_environment(&mut b, w, h, d, k, rot); + } + super::raster::shadow_maps(&mut b.s, r.opts.draft); + } + // Camera matrices. + let s = r.opts.scale; + let (cw, ch) = (ctx.comp.width as f64, ctx.comp.height as f64); + let (ow, oh) = (out.0 as f64, out.1 as f64); + let (a, bb) = (2.0 * s * cam.zoom / ow, s * cw / ow - 1.0); + let (c, d) = (2.0 * s * cam.zoom / oh, s * ch / oh - 1.0); + let proj = if cam.ortho { + let (mut zmin, mut zmax) = (f64::INFINITY, f64::NEG_INFINITY); + for v in &b.s.vertices { + let z = cam.depth(Vec3::from(v.pos.map(|x| x as f64))); + zmin = zmin.min(z); + zmax = zmax.max(z); + } + if !zmin.is_finite() { + (zmin, zmax) = (-1.0, 1.0); + } + let pad = (zmax - zmin).abs() * 0.01 + 1.0; + let (zmin, zmax) = (zmin - pad, zmax + pad); + let zr = zmax - zmin; + Mat4([[a, 0.0, 0.0, bb], [0.0, -c, 0.0, -d], [0.0, 0.0, -1.0 / zr, zmax / zr], [0.0, 0.0, 0.0, 1.0]]) + } else { + Mat4([[a, 0.0, bb, 0.0], [0.0, -c, -d, 0.0], [0.0, 0.0, 0.0, NEAR], [0.0, 0.0, 1.0, 0.0]]) + }; + b.s.clip = m32(&(proj * cam.view)); + b.s.view = m32(&cam.view); + let f = cam.forward(); + b.s.cam_fwd = [f.x as f32, f.y as f32, f.z as f32]; + b.s.eye = [cam.eye.x as f32, cam.eye.y as f32, cam.eye.z as f32]; + b.s.ortho = cam.ortho; + b.s +} diff --git a/crates/render/src/three_d/adv/shade.rs b/crates/render/src/three_d/adv/shade.rs new file mode 100644 index 00000000..16d05287 --- /dev/null +++ b/crates/render/src/three_d/adv/shade.rs @@ -0,0 +1,410 @@ +//! Physically based shading for the Advanced 3D renderer (CPU reference; `crates/gpu`'s +//! `advanced3d.wgsl` repeats it step for step). +//! +//! - Direct light: Cook–Torrance microfacet specular with the GGX/Trowbridge–Reitz normal +//! distribution, the Smith–Schlick-GGX geometry term (`k = (r + 1)² / 8`) and Schlick's +//! Fresnel (`F0` = 0.04 for dielectrics, the base colour for metals), plus a Lambertian +//! diffuse lobe weighted by `(1 − F)(1 − metallic)` (Karis 2013, "Real Shading in Unreal +//! Engine 4"; the glTF 2.0 metallic-roughness model). Light colours are the irradiance on a +//! surface facing the light (a white 100% light shows a white diffuse surface at its base +//! colour), i.e. π is folded into the light intensity. +//! - Image-based light: a cosine-convolved irradiance map for diffuse, a box-filtered mip +//! chain of the equirectangular environment sampled by roughness for specular, and the +//! analytic split-sum environment BRDF fit (Karis 2014, "Physically Based Shading on +//! Mobile"). +//! - Shadows: depth maps per light, percentage-closer filtered. + +use super::scene::{EnvInfo, Light, Material, Scene, ShadowMap, TexInfo}; + +pub const PI: f32 = std::f32::consts::PI; + +#[inline] +pub fn dot(a: [f32; 3], b: [f32; 3]) -> f32 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +#[inline] +pub fn cross(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +#[inline] +pub fn norm(a: [f32; 3]) -> [f32; 3] { + let l = dot(a, a).sqrt(); + if l > 1e-20 { [a[0] / l, a[1] / l, a[2] / l] } else { [0.0, 0.0, -1.0] } +} +#[inline] +fn add(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +#[inline] +fn sub(a: [f32; 3], b: [f32; 3]) -> [f32; 3] { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +#[inline] +fn scale(a: [f32; 3], k: f32) -> [f32; 3] { + [a[0] * k, a[1] * k, a[2] * k] +} +#[inline] +fn mix(a: f32, b: f32, t: f32) -> f32 { + a + (b - a) * t +} + +/// sRGB-encoded → linear light. +#[inline] +pub fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } +} +/// Linear light → sRGB-encoded (over-range values continue the curve). +#[inline] +pub fn linear_to_srgb(c: f32) -> f32 { + if c <= 0.0031308 { c * 12.92 } else { 1.055 * c.powf(1.0 / 2.4) - 0.055 } +} + +/// GGX / Trowbridge–Reitz normal distribution, `a = roughness²`. +#[inline] +pub fn d_ggx(n_h: f32, a: f32) -> f32 { + let a2 = a * a; + let d = n_h * n_h * (a2 - 1.0) + 1.0; + a2 / (PI * d * d).max(1e-12) +} + +/// Smith–Schlick-GGX geometry term for direct light (`k = (r + 1)² / 8`). +#[inline] +pub fn g_smith(n_v: f32, n_l: f32, rough: f32) -> f32 { + let k = (rough + 1.0) * (rough + 1.0) / 8.0; + let g1 = |x: f32| x / (x * (1.0 - k) + k); + g1(n_v) * g1(n_l) +} + +/// Schlick's Fresnel approximation. +#[inline] +pub fn f_schlick(f0: [f32; 3], v_h: f32) -> [f32; 3] { + let k = (1.0 - v_h).clamp(0.0, 1.0).powi(5); + [f0[0] + (1.0 - f0[0]) * k, f0[1] + (1.0 - f0[1]) * k, f0[2] + (1.0 - f0[2]) * k] +} + +/// Cook–Torrance BRDF × N·L × π for one light: (diffuse colour factor, specular colour). +/// Returned as the reflected fraction of the light's irradiance. +pub fn brdf(base: [f32; 3], metallic: f32, rough: f32, spec_k: f32, diff_k: f32, n: [f32; 3], v: [f32; 3], l: [f32; 3]) -> [f32; 3] { + let n_l = dot(n, l); + if n_l <= 0.0 { + return [0.0; 3]; + } + let n_v = dot(n, v).max(1e-4); + let h = norm(add(v, l)); + let n_h = dot(n, h).max(0.0); + let v_h = dot(v, h).max(0.0); + let r = rough.clamp(0.03, 1.0); + let f0 = [mix(0.04 * spec_k, base[0], metallic), mix(0.04 * spec_k, base[1], metallic), mix(0.04 * spec_k, base[2], metallic)]; + let f = f_schlick(f0, v_h); + let d = d_ggx(n_h, r * r); + let g = g_smith(n_v, n_l, r); + let spec = d * g / (4.0 * n_v * n_l).max(1e-4); + let mut out = [0.0; 3]; + for c in 0..3 { + let kd = (1.0 - f[c]) * (1.0 - metallic); + out[c] = (kd * base[c] / PI * diff_k + f[c] * spec) * n_l * PI; + } + out +} + +/// Split-sum environment BRDF (scale, bias) for `F0·A + B` (analytic fit). +#[inline] +pub fn env_brdf(n_v: f32, rough: f32) -> (f32, f32) { + let c0 = [-1.0, -0.0275, -0.572, 0.022]; + let c1 = [1.0, 0.0425, 1.04, -0.04]; + let r = [rough * c0[0] + c1[0], rough * c0[1] + c1[1], rough * c0[2] + c1[2], rough * c0[3] + c1[3]]; + let a004 = (r[0] * r[0]).min((-9.28 * n_v).exp2()) * r[0] + r[1]; + (-1.04 * a004 + r[2], 1.04 * a004 + r[3]) +} + +/// Wrap a texel index. +#[inline] +fn wrap(i: i64, n: i64, mode: u32) -> i64 { + match mode { + 1 => i.clamp(0, n - 1), + 2 => { + let m = i.rem_euclid(2 * n); + if m >= n { 2 * n - 1 - m } else { m } + } + _ => i.rem_euclid(n), + } +} + +/// Bilinear texture sample at (u, v) (0..1 across the texture, texel centres at +0.5). +pub fn sample(t: &TexInfo, texels: &[[f32; 4]], u: f32, v: f32) -> [f32; 4] { + let (w, h) = (t.width as i64, t.height as i64); + if w == 0 || h == 0 { + return [1.0; 4]; + } + let x = u * w as f32 - 0.5; + let y = v * h as f32 - 0.5; + let (x0, y0) = (x.floor(), y.floor()); + let (fx, fy) = (x - x0, y - y0); + let (x0, y0) = (x0 as i64, y0 as i64); + let get = |xi: i64, yi: i64| -> [f32; 4] { + let xi = wrap(xi, w, t.wrap_u); + let yi = wrap(yi, h, t.wrap_v); + texels.get(t.offset as usize + (yi * w + xi) as usize).copied().unwrap_or([0.0; 4]) + }; + let (a, b, c, d) = (get(x0, y0), get(x0 + 1, y0), get(x0, y0 + 1), get(x0 + 1, y0 + 1)); + let mut o = [0.0; 4]; + for k in 0..4 { + let top = a[k] + (b[k] - a[k]) * fx; + let bot = c[k] + (d[k] - c[k]) * fx; + o[k] = top + (bot - top) * fy; + } + o +} + +/// Equirectangular coordinates of a world direction (AE world: +Y down; the environment's up +/// is −Y), rotated about the vertical axis by `rot` radians. +#[inline] +pub fn equirect_uv(d: [f32; 3], rot: f32) -> (f32, f32) { + let u = 0.5 + (d[0].atan2(d[2]) + rot) / (2.0 * PI); + let v = (-d[1]).clamp(-1.0, 1.0).acos() / PI; + (u - u.floor(), v) +} + +/// Inverse of [`equirect_uv`] (texel centre → direction). +pub fn equirect_dir(u: f32, v: f32, rot: f32) -> [f32; 3] { + let phi = (u - 0.5) * 2.0 * PI - rot; + let th = v * PI; + [th.sin() * phi.sin(), -th.cos(), th.sin() * phi.cos()] +} + +/// Diffuse (irradiance) and roughness-filtered specular radiance of the environment. +pub fn env_light(s: &Scene, e: &EnvInfo, n: [f32; 3], r: [f32; 3], rough: f32) -> ([f32; 3], [f32; 3]) { + let (u, v) = equirect_uv(n, e.rotation); + let irr = s.textures.get(e.irradiance as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let (u, v) = equirect_uv(r, e.rotation); + let lv = rough.clamp(0.0, 1.0) * (e.mips.saturating_sub(1)) as f32; + let l0 = lv.floor() as u32; + let l1 = (l0 + 1).min(e.mips.saturating_sub(1)); + let f = lv - l0 as f32; + let a = s.textures.get((e.radiance + l0) as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let b = s.textures.get((e.radiance + l1) as usize).map_or([0.0; 4], |t| sample(t, &s.texels, u, v)); + let spec = [mix(a[0], b[0], f), mix(a[1], b[1], f), mix(a[2], b[2], f)]; + (scale([irr[0], irr[1], irr[2]], e.intensity), scale(spec, e.intensity)) +} + +/// Light-space position (u, v in shadow-map pixels, depth) of a world point. +#[inline] +pub fn shadow_coord(m: &ShadowMap, p: [f32; 3]) -> Option<(f32, f32, f32)> { + let v = &m.view; + let q = [ + v[0][0] * p[0] + v[0][1] * p[1] + v[0][2] * p[2] + v[0][3], + v[1][0] * p[0] + v[1][1] * p[1] + v[1][2] * p[2] + v[1][3], + v[2][0] * p[0] + v[2][1] * p[1] + v[2][2] * p[2] + v[2][3], + ]; + let c = m.size as f32 * 0.5; + if m.ortho { + Some((q[0] * m.focal + c, q[1] * m.focal + c, q[2])) + } else if q[2] > 1e-3 { + Some((q[0] * m.focal / q[2] + c, q[1] * m.focal / q[2] + c, q[2])) + } else { + None + } +} + +/// Fraction of light reaching `p` through shadow map `m` (1 = lit), PCF over a 5×5 grid +/// spread over `radius` texels. +pub fn shadow_factor(m: &ShadowMap, texels: &[f32], p: [f32; 3], n_l: f32) -> f32 { + let Some((u, v, z)) = shadow_coord(m, p) else { return 1.0 }; + let size = m.size as i64; + // Slope-scaled bias (world units are pixels). + // Texel footprint (world units) × PCF reach, steeper for grazing light. + let texel = if m.ortho { 1.0 / m.focal } else { z.abs() / m.focal }; + let bias = m.bias * texel * (1.0 + m.radius) * (1.0 + 3.0 * (1.0 - n_l.clamp(0.0, 1.0))); + let step = m.radius / 2.0; + let mut lit = 0.0; + for j in -2..=2 { + for i in -2..=2 { + let x = (u + i as f32 * step).floor() as i64; + let y = (v + j as f32 * step).floor() as i64; + if x < 0 || y < 0 || x >= size || y >= size { + lit += 1.0; + continue; + } + let d = texels[m.offset as usize + (y * size + x) as usize]; + if z - bias <= d { + lit += 1.0; + } + } + } + lit / 25.0 +} + +/// The visibility of light `li` at `p` (1 = unshadowed), darkened by Shadow Darkness. +pub fn light_shadow(s: &Scene, l: &Light, p: [f32; 3], n_l: f32) -> f32 { + if l.shadow < 0 { + return 1.0; + } + let first = l.shadow as usize; + let m = if l.kind == 2 { + // Point light: the cube face of the major axis of the light → point direction. + let d = sub(p, l.pos); + let (ax, ay, az) = (d[0].abs(), d[1].abs(), d[2].abs()); + let face = if ax >= ay && ax >= az { + if d[0] > 0.0 { 0 } else { 1 } + } else if ay >= az { + if d[1] > 0.0 { 2 } else { 3 } + } else if d[2] > 0.0 { + 4 + } else { + 5 + }; + s.shadows.get(first + face) + } else { + s.shadows.get(first) + }; + let Some(m) = m else { return 1.0 }; + let f = shadow_factor(m, &s.shadow_texels, p, n_l); + 1.0 - l.shadow_darkness * (1.0 - f) +} + +/// Distance attenuation × spot cone of a light at `p` (AE falloff modes). +pub fn attenuation(l: &Light, p: [f32; 3], to_l: [f32; 3], dist: f32) -> f32 { + let mut k = match l.falloff { + 1 if l.kind != 0 => { + if dist <= l.radius { + 1.0 + } else if l.falloff_distance <= 0.0 { + 0.0 + } else { + let t = ((dist - l.radius) / l.falloff_distance).clamp(0.0, 1.0); + 1.0 - t * t * (3.0 - 2.0 * t) + } + } + 2 if l.kind != 0 => { + if dist <= l.radius || (l.radius <= 0.0 && dist <= 1.0) { + 1.0 + } else { + (l.radius.max(1.0) / dist).powi(2) + } + } + _ => 1.0, + }; + let _ = p; + if l.kind == 1 { + let cos_a = dot(scale(to_l, -1.0), l.dir).clamp(-1.0, 1.0); + k *= if cos_a >= l.cos_inner { + 1.0 + } else if cos_a <= l.cos_outer { + 0.0 + } else { + // Smoothstep across the feather in angle. + let a = cos_a.acos(); + let (ai, ao) = (l.cos_inner.acos(), l.cos_outer.acos()); + let t = ((a - ai) / (ao - ai).max(1e-6)).clamp(0.0, 1.0); + 1.0 - t * t * (3.0 - 2.0 * t) + }; + } + k +} + +/// Inputs of one fragment. +#[derive(Clone, Copy, Debug)] +pub struct Frag { + pub pos: [f32; 3], + /// Interpolated (unnormalised) vertex normal. + pub normal: [f32; 3], + pub uv: [f32; 2], + pub tangent: [f32; 4], + pub material: u32, + pub front: bool, +} + +/// Shade a fragment: premultiplied linear RGBA, or `None` when discarded (alpha test, back +/// face of a single-sided material). +pub fn shade(s: &Scene, f: &Frag) -> Option<[f32; 4]> { + let m: &Material = s.materials.get(f.material as usize)?; + if !f.front && !m.double_sided { + return None; + } + let tex = |i: i32| -> Option<[f32; 4]> { (i >= 0).then(|| s.textures.get(i as usize).map(|t| sample(t, &s.texels, f.uv[0], f.uv[1]))).flatten() }; + let mut base = m.base; + if let Some(t) = tex(m.tex_base) { + for k in 0..4 { + base[k] *= t[k]; + } + } + let alpha = base[3] * m.opacity; + if m.alpha_mode == 1 && alpha < m.alpha_cutoff { + return None; + } + if alpha <= 1e-5 && m.alpha_mode != 0 { + return None; + } + let alpha = if m.alpha_mode == 0 { m.opacity } else { alpha }; + let rgb = [base[0], base[1], base[2]]; + let mut emissive = m.emissive; + if let Some(t) = tex(m.tex_emissive) { + emissive = [emissive[0] * t[0], emissive[1] * t[1], emissive[2] * t[2]]; + } + if m.unlit || !m.accepts_lights || (s.lights.is_empty() && s.env.is_none() && s.ambient == [0.0; 3]) { + let c = add(rgb, emissive); + return Some([c[0] * alpha, c[1] * alpha, c[2] * alpha, alpha]); + } + let (mut metallic, mut rough) = (m.metallic, m.roughness); + if let Some(t) = tex(m.tex_mr) { + rough *= t[1]; + metallic *= t[2]; + } + let mut n = norm(f.normal); + if !f.front { + n = scale(n, -1.0); + } + if let Some(t) = tex(m.tex_normal) { + let tn = [(t[0] * 2.0 - 1.0) * m.normal_scale, (t[1] * 2.0 - 1.0) * m.normal_scale, t[2] * 2.0 - 1.0]; + let tg = [f.tangent[0], f.tangent[1], f.tangent[2]]; + // Gram-Schmidt against the shading normal. + let tg = norm(sub(tg, scale(n, dot(n, tg)))); + let sign = if f.front { f.tangent[3] } else { -f.tangent[3] }; + let bt = scale(cross(n, tg), sign); + n = norm(add(add(scale(tg, tn[0]), scale(bt, tn[1])), scale(n, tn[2]))); + } + let ao = match tex(m.tex_occlusion) { + Some(t) => 1.0 + m.occlusion_strength * (t[0] - 1.0), + None => 1.0, + }; + let v = if s.ortho { scale(s.cam_fwd, -1.0) } else { norm(sub(s.eye, f.pos)) }; + let mut out = [0.0f32; 3]; + for l in &s.lights { + let (to_l, dist) = if l.kind == 0 { + (scale(l.dir, -1.0), f32::INFINITY) + } else { + let d = sub(l.pos, f.pos); + let len = dot(d, d).sqrt(); + if len < 1e-6 { ([0.0, 0.0, -1.0], 0.0) } else { (scale(d, 1.0 / len), len) } + }; + let k = attenuation(l, f.pos, to_l, dist); + if k <= 0.0 { + continue; + } + let n_l = dot(n, to_l); + if n_l <= 0.0 { + continue; + } + let sh = if m.receives_shadows { light_shadow(s, l, f.pos, n_l) } else { 1.0 }; + let b = brdf(rgb, metallic, rough, m.specular_k, m.diffuse_k, n, v, to_l); + for c in 0..3 { + out[c] += b[c] * l.color[c] * k * sh; + } + } + for c in 0..3 { + out[c] += s.ambient[c] * rgb[c] * m.ambient_k * ao; + } + if let Some(e) = &s.env { + let n_v = dot(n, v).clamp(1e-4, 1.0); + let r = sub(scale(n, 2.0 * dot(n, v)), v); + let (irr, spec) = env_light(s, e, n, r, rough); + let f0 = [mix(0.04 * m.specular_k, rgb[0], metallic), mix(0.04 * m.specular_k, rgb[1], metallic), mix(0.04 * m.specular_k, rgb[2], metallic)]; + let (a, b) = env_brdf(n_v, rough); + for c in 0..3 { + let fs = f0[c] * a + b; + out[c] += (irr[c] * rgb[c] * (1.0 - fs) * (1.0 - metallic) * m.diffuse_k + spec[c] * fs) * ao; + } + } + let c = add(out, emissive); + Some([c[0] * alpha, c[1] * alpha, c[2] * alpha, alpha]) +} diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs new file mode 100644 index 00000000..47ad6178 --- /dev/null +++ b/crates/render/src/three_d/adv/tests.rs @@ -0,0 +1,314 @@ +use effectcraft_keyframe::Value; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, LightKind, PrimitiveKind, Project, Renderer as R3, build}; +use effectcraft_time::{FrameRate, Tick}; + +use super::scene::{Material, Scene, TexInfo, Vertex, irradiance_map}; +use super::shade::{self, PI}; +use super::*; +use crate::{NoFootage, RenderOpts, Renderer}; + +// ---------------------------------------------------------------- shading + +#[test] +fn brdf_matches_cook_torrance_formula() { + // Reference: f = kd·c/π + D·G·F/(4·NV·NL), returned × NL × π. + let n = [0.0, 0.0, 1.0]; + let v = shade::norm([0.3, 0.1, 1.0]); + let l = shade::norm([-0.4, 0.2, 1.0]); + let (base, metal, rough) = ([0.8, 0.4, 0.2], 0.3f32, 0.45f32); + let got = shade::brdf(base, metal, rough, 1.0, 1.0, n, v, l); + let h = shade::norm([v[0] + l[0], v[1] + l[1], v[2] + l[2]]); + let (nv, nl, nh, vh) = (shade::dot(n, v), shade::dot(n, l), shade::dot(n, h), shade::dot(v, h)); + let a2 = (rough * rough).powi(2); + let d = a2 / (PI * ((nh * nh) * (a2 - 1.0) + 1.0).powi(2)); + let k = (rough + 1.0).powi(2) / 8.0; + let g = nv / (nv * (1.0 - k) + k) * nl / (nl * (1.0 - k) + k); + for c in 0..3 { + let f0 = 0.04 + (base[c] - 0.04) * metal; + let f = f0 + (1.0 - f0) * (1.0 - vh).powi(5); + let expect = ((1.0 - f) * (1.0 - metal) * base[c] / PI + d * g * f / (4.0 * nv * nl)) * nl * PI; + assert!((got[c] - expect).abs() < 1e-5, "channel {c}: {} vs {expect}", got[c]); + } + // No light from behind the surface. + assert_eq!(shade::brdf(base, metal, rough, 1.0, 1.0, n, v, [0.0, 0.0, -1.0]), [0.0; 3]); + // A white rough dielectric lit head-on reflects about its base colour (π folded in). + let w = shade::brdf([1.0; 3], 0.0, 1.0, 1.0, 1.0, n, n, n); + assert!(w[0] > 0.95 && w[0] < 1.05, "{w:?}"); +} + +#[test] +fn ggx_distribution_is_normalised() { + // ∫ D(h)·(n·h) dω = 1 over the hemisphere. + for &rough in &[0.2f32, 0.5, 0.9] { + let a = rough * rough; + let n = 4000; + let mut sum = 0.0f64; + for i in 0..n { + let th = (i as f64 + 0.5) / n as f64 * std::f64::consts::FRAC_PI_2; + let c = th.cos() as f32; + sum += shade::d_ggx(c, a) as f64 * c as f64 * th.sin() * std::f64::consts::FRAC_PI_2 / n as f64 * 2.0 * std::f64::consts::PI; + } + assert!((sum - 1.0).abs() < 0.02, "roughness {rough}: {sum}"); + } +} + +#[test] +fn constant_environment_has_constant_irradiance() { + let src = vec![[0.25, 0.5, 0.75, 1.0]; 64 * 32]; + let irr = irradiance_map(64, 32, &src, 32, 16); + for p in &irr { + for c in 0..3 { + assert!((p[c] - src[0][c]).abs() < 1e-4, "{p:?}"); + } + } + // And a sky brighter than the ground lights upward-facing normals more. + let sky: Vec<[f32; 4]> = (0..32).flat_map(|y| (0..64).map(move |_| if y < 16 { [1.0, 1.0, 1.0, 1.0] } else { [0.0, 0.0, 0.0, 1.0] })).collect(); + let irr = irradiance_map(64, 32, &sky, 32, 16); + assert!(irr[16][0] > 0.9 && irr[15 * 32 + 16][0] < 0.1, "{:?} {:?}", irr[16], irr[15 * 32 + 16]); +} + +#[test] +fn equirect_round_trip_and_env_brdf() { + for (u, v) in [(0.1f32, 0.2f32), (0.5, 0.5), (0.9, 0.7)] { + let d = shade::equirect_dir(u, v, 0.3); + let (u2, v2) = shade::equirect_uv(d, 0.3); + assert!((u - u2).abs() < 1e-4 && (v - v2).abs() < 1e-4); + } + // Up (−Y in AE space) is the top row. + assert!(shade::equirect_uv([0.0, -1.0, 0.0], 0.0).1 < 1e-3); + let (a, b) = shade::env_brdf(1.0, 0.0); + assert!(a + b > 0.9 && a + b <= 1.05); + let (a, b) = shade::env_brdf(0.1, 1.0); + assert!(a >= 0.0 && b > -0.01 && a + b < 1.0); +} + +// ---------------------------------------------------------------- rasteriser + +/// A scene of axis-aligned quads (z = camera depth) seen by an orthographic identity camera +/// mapping world x/y straight to raster pixels. +fn quad_scene(w: u32, h: u32, quads: &[([f32; 4], f32, [f32; 4])]) -> Scene { + let mut s = Scene { width: w, height: h, ssaa: 1, linear_io: true, ..Scene::default() }; + // clip: x → 2x/w − 1, y → 1 − 2y/h, z: depth 0..1000 → 1..0 (nearer = larger). + s.clip = [[2.0 / w as f32, 0.0, 0.0, -1.0], [0.0, -2.0 / h as f32, 0.0, 1.0], [0.0, 0.0, -0.001, 1.0], [0.0, 0.0, 0.0, 1.0]]; + s.view = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]; + s.ortho = true; + s.cam_fwd = [0.0, 0.0, 1.0]; + for (rect, z, color) in quads { + let m = s.materials.len() as u32; + s.materials.push(Material { base: *color, unlit: true, double_sided: true, alpha_mode: if color[3] < 1.0 { 2 } else { 0 }, ..Material::default() }); + let b = s.vertices.len() as u32; + for (x, y) in [(rect[0], rect[1]), (rect[2], rect[1]), (rect[2], rect[3]), (rect[0], rect[3])] { + s.vertices.push(Vertex { pos: [x, y, *z], normal: [0.0, 0.0, -1.0], uv: [0.0; 2], tangent: [1.0, 0.0, 0.0, 1.0], material: m }); + } + s.indices.extend([b, b + 2, b + 1, b, b + 3, b + 2]); + } + // Opaque first, transparent (in given order) after. + let tris: Vec<[u32; 3]> = s.indices.chunks(3).map(|t| [t[0], t[1], t[2]]).collect(); + let (o, t): (Vec<_>, Vec<_>) = tris.into_iter().partition(|t| !s.materials[s.vertices[t[0] as usize].material as usize].transparent()); + s.opaque_count = 3 * o.len() as u32; + s.indices = o.into_iter().chain(t).flatten().collect(); + s +} + +#[test] +fn depth_buffer_keeps_the_nearest_surface() { + let red = [1.0, 0.0, 0.0, 1.0]; + let blue = [0.0, 0.0, 1.0, 1.0]; + // Blue is listed first but is farther away. + let s = quad_scene(40, 30, &[([0.0, 0.0, 30.0, 20.0], 500.0, blue), ([10.0, 10.0, 40.0, 30.0], 100.0, red)]); + let t = raster::render(&s); + let px = |x: usize, y: usize| t.color[y * 40 + x]; + assert_eq!(px(5, 5), blue); + assert_eq!(px(15, 15), red, "overlap: the nearer quad wins"); + assert_eq!(px(35, 25), red); + assert_eq!(px(35, 5), [0.0; 4]); + assert_eq!(t.depth[15 * 40 + 15], 100.0); + assert!(t.depth[5 * 40 + 35].is_infinite()); +} + +#[test] +fn coverage_is_exact_for_pixel_aligned_quads() { + // Shared diagonal edges are filled once (top-left rule): exact pixel counts. + let s = quad_scene(32, 32, &[([4.0, 4.0, 20.0, 12.0], 10.0, [1.0, 1.0, 1.0, 0.5])]); + let t = raster::render(&s); + let covered = t.color.iter().filter(|c| c[3] > 0.0).count(); + assert_eq!(covered, 16 * 8); + // Blended once (no double coverage along the diagonal). + assert!(t.color.iter().all(|c| c[3] == 0.0 || (c[3] - 0.5).abs() < 1e-6)); + // Two transparent layers composite in order. + let s = quad_scene(8, 8, &[([0.0, 0.0, 8.0, 8.0], 100.0, [1.0, 0.0, 0.0, 0.5]), ([0.0, 0.0, 8.0, 8.0], 50.0, [0.0, 0.0, 1.0, 0.5])]); + let c = raster::render(&s).color[0]; + assert!((c[0] - 0.25).abs() < 1e-6 && (c[2] - 0.5).abs() < 1e-6 && (c[3] - 0.75).abs() < 1e-6, "{c:?}"); +} + +#[test] +fn textures_sample_bilinear_with_wrap_modes() { + let t = TexInfo { offset: 0, width: 2, height: 1, wrap_u: 0, wrap_v: 1 }; + let texels = vec![[0.0, 0.0, 0.0, 1.0], [1.0, 1.0, 1.0, 1.0]]; + assert_eq!(shade::sample(&t, &texels, 0.25, 0.5)[0], 0.0); + assert_eq!(shade::sample(&t, &texels, 0.5, 0.5)[0], 0.5); + // Repeat wraps the right edge back to texel 0. + assert!((shade::sample(&t, &texels, 1.0, 0.5)[0] - 0.5).abs() < 1e-6); + let c = TexInfo { wrap_u: 1, ..t }; + assert_eq!(shade::sample(&c, &texels, 1.0, 0.5)[0], 1.0); +} + +// ---------------------------------------------------------------- full renderer + +fn project(w: u32, h: u32) -> (Project, ItemId) { + let mut p = Project::default(); + let mut c = Comp::new(w, h, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + c.renderer = R3::Advanced3D; + let id = p.add_item("C", Default::default(), None, ItemKind::Comp(c.into())); + (p, id) +} + +fn add(p: &mut Project, cid: ItemId, src: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let comp = p.comp(cid).unwrap().clone(); + let mut l = build::layer(p, &comp, "L", src, (comp.width, comp.height), None); + edit(&mut l); + p.comp_mut(cid).unwrap().layers.insert(0, l); +} + +fn set(l: &mut effectcraft_project::Layer, path: &str, v: Value) { + l.props.prop_mut(path).unwrap_or_else(|| panic!("{path}")).value = v; +} + +fn render(p: &Project, cid: ItemId) -> crate::Image { + Renderer::new(p, &NoFootage, RenderOpts::default()).comp_frame(cid, Tick::ZERO) +} + +#[test] +fn lit_cube_renders_with_depth_and_shading() { + let (mut p, cid) = project(160, 120); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Cube }, |l| { + set(l, "geometryOptions/width", Value::Scalar(60.0)); + set(l, "geometryOptions/height", Value::Scalar(60.0)); + set(l, "geometryOptions/depth", Value::Scalar(60.0)); + set(l, "transform/rotationY", Value::Scalar(35.0)); + set(l, "transform/rotationX", Value::Scalar(-25.0)); + set(l, "materialOptions/baseColor", Value::Color([1.0, 1.0, 1.0, 1.0])); + }); + let img = render(&p, cid); + let centre = img.data[60 * 160 + 80]; + assert!(centre[3] > 0.99, "cube covers the centre: {centre:?}"); + assert_eq!(img.data[2 * 160 + 2][3], 0.0, "corner is empty"); + // No lights: unlit (base colour) — add a light and faces get different shades. + add(&mut p, cid, LayerSource::Light { kind: LightKind::Parallel }, |_| {}); + let lit = render(&p, cid); + let mut shades: Vec = lit.data.iter().filter(|q| q[3] > 0.99).map(|q| (q[0] * 20.0) as i32).collect(); + shades.sort(); + shades.dedup(); + assert!(shades.len() >= 3, "faces are shaded differently: {shades:?}"); + // Classic 3D draws no meshes. + p.comp_mut(cid).unwrap().renderer = R3::Classic3D; + assert!(render(&p, cid).data.iter().all(|q| q[3] == 0.0)); +} + +#[test] +fn shadows_darken_the_receiver() { + let (mut p, cid) = project(200, 150); + // A floor plane facing the camera, a sphere in front of it, a spot light behind the camera. + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Plane }, |l| { + set(l, "geometryOptions/width", Value::Scalar(400.0)); + set(l, "geometryOptions/height", Value::Scalar(400.0)); + set(l, "transform/position", Value::Vec3([100.0, 75.0, 200.0])); + set(l, "materialOptions/baseColor", Value::Color([1.0, 1.0, 1.0, 1.0])); + }); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Sphere }, |l| { + set(l, "geometryOptions/radius", Value::Scalar(30.0)); + set(l, "transform/position", Value::Vec3([100.0, 75.0, 50.0])); + set(l, "materialOptions/castsShadows", Value::Enum(2)); + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Point }, |l| { + set(l, "transform/position", Value::Vec3([100.0, 75.0, -300.0])); + set(l, "lightOptions/castsShadows", Value::Bool(true)); + }); + let img = render(&p, cid); + // The sphere casts only (invisible); its shadow lands on the plane behind it. + let mid = img.data[75 * 200 + 100]; + let side = img.data[75 * 200 + 20]; + assert!(mid[3] > 0.99 && side[3] > 0.99); + assert!(mid[0] < side[0] * 0.5, "shadowed centre {mid:?} vs lit side {side:?}"); + // Without shadow casting the centre is lit. + let l = &mut p.comp_mut(cid).unwrap().layers[0]; + set(l, "lightOptions/castsShadows", Value::Bool(false)); + let img = render(&p, cid); + assert!(img.data[75 * 200 + 100][0] > side[0] * 0.8); +} + +#[test] +fn cards_and_environment_light() { + let (mut p, cid) = project(120, 90); + let sid = p.add_item( + "S", + Default::default(), + None, + ItemKind::Solid(effectcraft_project::Solid { color: [0.5, 0.5, 0.5], width: 60, height: 40, pixel_aspect: 1.0 }), + ); + add(&mut p, cid, LayerSource::Solid { item: sid }, |l| l.switches.three_d = true); + let unlit = render(&p, cid); + let c = unlit.data[20 * 120 + 30]; + assert!((c[0] - 0.5).abs() < 0.01 && c[3] > 0.99, "unlit card keeps its colour: {c:?}"); + assert_eq!(unlit.data[80 * 120 + 110][3], 0.0); + // An Environment light sourcing a white environment layer brightens the card. + let env = p.add_item( + "E", + Default::default(), + None, + ItemKind::Solid(effectcraft_project::Solid { color: [1.0, 1.0, 1.0], width: 64, height: 32, pixel_aspect: 1.0 }), + ); + add(&mut p, cid, LayerSource::Solid { item: env }, |l| { + l.environment = true; + l.switches.three_d = true; + }); + add(&mut p, cid, LayerSource::Light { kind: LightKind::Environment }, |_| {}); + let lit = render(&p, cid); + let c2 = lit.data[20 * 120 + 30]; + assert!(c2[0] > 0.3, "{c2:?}"); + // The environment layer itself is not drawn. + assert_eq!(lit.data[80 * 120 + 110][3], 0.0); +} + +#[test] +fn depth_of_field_blurs_out_of_focus_surfaces() { + let mut img = Image::new(64, 64); + for y in 0..64 { + for x in 0..64 { + let v = if (x / 4 + y / 4) % 2 == 0 { 1.0 } else { 0.0 }; + img.data[y * 64 + x] = [v, v, v, 1.0]; + } + } + let var = |im: &Image| { + let m: f32 = im.data.iter().map(|p| p[0]).sum::() / im.data.len() as f32; + im.data.iter().map(|p| (p[0] - m).powi(2)).sum::() / im.data.len() as f32 + }; + let dof = crate::three_d::camera::Dof { focus: 1000.0, aperture: 40.0, blur_level: 1.0 }; + let sharp = depth_of_field(&img, &vec![1000.0; 64 * 64], &dof, 1.0); + assert!((var(&sharp) - var(&img)).abs() < 1e-6, "in focus: unchanged"); + let blurred = depth_of_field(&img, &vec![3000.0; 64 * 64], &dof, 1.0); + assert!(var(&blurred) < var(&img) * 0.3, "out of focus: blurred"); +} + +#[test] +fn extruded_text_renders_as_a_mesh() { + let (mut p, cid) = project(200, 100); + add(&mut p, cid, LayerSource::Text, |l| { + l.switches.three_d = true; + let doc = effectcraft_keyframe::TextDoc { text: "IO".into(), size: 60.0, fill: [1.0, 1.0, 1.0, 1.0], apply_fill: true, ..Default::default() }; + set(l, "text/sourceText", Value::Text(Box::new(doc))); + let mut next = 10_000u64; + l.props.children.push(build::extrusion_geometry_options(&mut build::Ids(&mut next)).into()); + set(l, "geometryOptions/extrusionDepth", Value::Scalar(30.0)); + set(l, "geometryOptions/bevelStyle", Value::Enum(1)); + set(l, "transform/rotationY", Value::Scalar(40.0)); + }); + let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let l = &ctx.comp.layers[0]; + let params = scene::extrusion_params(&ctx, l).unwrap(); + let meshes = scene::extruded_meshes(&ctx, l, ¶ms); + assert!(!meshes.is_empty()); + assert!(meshes.iter().all(|(m, _, _)| m.positions.iter().any(|q| (q[2] - 30.0).abs() < 1e-3))); + let img = render(&p, cid); + assert!(img.data.iter().filter(|q| q[3] > 0.5).count() > 200); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 1ef7412b..2647fadd 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -323,6 +323,12 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut if run.is_empty() { return; } + // Advanced 3D comps: meshes, physically based lights, shadow maps (collapsed precomps keep + // the parent's Classic 3D planes). + if ctx.comp.renderer == effectcraft_project::Renderer::Advanced3D && r.collapse3d.is_none() { + super::adv::draw_run(r, ctx, run, canvas); + return; + } let out = (canvas.width, canvas.height); // A collapsed precomp's 3D layers are lit by the outer comp's lights. let lights = scene_lights(&r.collapse3d.map_or(*ctx, |c| c.parent)); @@ -583,3 +589,182 @@ fn occlusion(c: &Item, p: Vec3, l: &LightState) -> [f32; 3] { } out } + +// ------------------------------------------------------------------ auxiliary render pass + +/// One surface seen through an output pixel (aux pass). +#[derive(Clone, Copy)] +struct AuxFrag { + z: f32, + a: f32, + obj: f32, + mat: f32, + n: [f32; 3], + uv: [f32; 2], + obj_hash: u32, + mat_hash: u32, +} + +/// The comp's auxiliary channels at the context time, at output resolution: camera-space depth +/// (`Z`), `ObjectID` (layer number), `MaterialID` (source item id), `Coverage`, camera-space +/// normals (`N.X/Y/Z`), texture coordinates (`UV.U/V`) and Cryptomatte `CryptoObject` (layer +/// names) / `CryptoMaterial` (source names) with two ranks each, plus their manifests. +/// +/// Layers are gathered in stack order (top first); a run of 3D layers contributes its planes +/// sorted near → far at each pixel, a 2D layer its coverage on the plane of the comp. The +/// front-most surface with ≥ 50 % opacity (else the strongest one) provides depth, IDs, normal +/// and UV; Cryptomatte coverage is each surface's visible share (`α · Π(1 − α_front)`). +pub(crate) fn aux_pass(r: &Renderer, ctx: &EvalCtx) -> effectcraft_raster::AuxChannels { + use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_float, crypto_hash}; + let s = r.opts.scale; + let (w, h) = (((ctx.comp.width as f64 * s).round() as u32).max(1), ((ctx.comp.height as f64 * s).round() as u32).max(1)); + let out = (w, h); + let (wu, n) = (w as usize, w as usize * h as usize); + let cam = camera_for(r, ctx); + let visible: Vec<&Layer> = r.visible_layers(ctx).into_iter().filter(|l| !l.switches.adjustment).collect(); + let number = |l: &Layer| ctx.comp.layers.iter().position(|x| x.id == l.id).map_or(0.0, |i| i as f32 + 1.0); + let source_name = |l: &Layer| l.source.item().and_then(|i| r.project.item(i)).map(|it| it.name.clone()).unwrap_or_else(|| l.name.clone()); + let source_id = |l: &Layer| l.source.item().map_or(0.0, |i| i.0 as f32); + let mut manifest_obj: Vec<(String, u32)> = vec![]; + let mut manifest_mat: Vec<(String, u32)> = vec![]; + for l in &visible { + let (on, mn) = (l.name.clone(), source_name(l)); + if !manifest_obj.iter().any(|(x, _)| *x == on) { + manifest_obj.push((on.clone(), crypto_hash(&on))); + } + if !manifest_mat.iter().any(|(x, _)| *x == mn) { + manifest_mat.push((mn.clone(), crypto_hash(&mn))); + } + } + // Groups in stack order: runs of 3D layers, single 2D layers. + let mut groups: Vec<&[&Layer]> = vec![]; + let mut i = 0; + while i < visible.len() { + let mut j = i + 1; + if visible[i].is_3d() { + while j < visible.len() && visible[j].is_3d() { + j += 1; + } + } + groups.push(&visible[i..j]); + i = j; + } + let mut frags: Vec> = vec![Vec::new(); n]; + let plane_z = cam.depth(vec3(ctx.comp.width as f64 * 0.5, ctx.comp.height as f64 * 0.5, 0.0)) as f32; + for g in groups.iter().rev() { + if g[0].is_3d() { + let items: Vec = g.iter().enumerate().flat_map(|(k, l)| prepare_all(r, ctx, l, k, out, true)).collect(); + let meta: Vec<(f32, f32, u32, u32, [f32; 3], [f64; 2])> = items + .iter() + .map(|it| { + let nrm = it.geos.first().map_or(vec3(0.0, 0.0, -1.0), |gm| cam.view.apply_vec(gm.normal)); + let mut nv = [nrm.x as f32, nrm.y as f32, nrm.z as f32]; + if nv[2] > 0.0 { + nv = nv.map(|v| -v); + } + let (sw, sh) = crate::source_size(r.project, it.layer); + let size = if sw == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [sw as f64, sh as f64] }; + (number(it.layer), source_id(it.layer), crypto_hash(&it.layer.name), crypto_hash(&source_name(it.layer)), nv, size) + }) + .collect(); + frags.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + let yi = y as i64; + let mut local: Vec = Vec::new(); + for (x, cell) in row.iter_mut().enumerate() { + let xi = x as i64; + local.clear(); + for (it, m) in items.iter().zip(&meta) { + if xi < it.bbox[0] || xi >= it.bbox[2] || yi < it.bbox[1] || yi >= it.bbox[3] { + continue; + } + let Some(gm) = it.geos.first() else { continue }; + let Some((u, v, z)) = unproject(gm, x as f64 + 0.5, y as f64 + 0.5) else { continue }; + let a = it.texel(u, v)[3] * it.opacity; + if a > 1e-4 { + let uv = [(u / m.5[0]) as f32, (v / m.5[1]) as f32]; + local.push(AuxFrag { z: z as f32, a: a.min(1.0), obj: m.0, mat: m.1, n: m.4, uv, obj_hash: m.2, mat_hash: m.3 }); + } + } + local.sort_by(|p, q| p.z.total_cmp(&q.z)); + cell.extend_from_slice(&local); + } + }); + } else { + let l = g[0]; + let mut img = Image::new(w, h); + if let Some(b) = r.layer_buf(ctx, l) { + r.place(ctx, l, &b, &mut img, BlendMode::Normal, ctx.opacity(l) as f32); + } + let (obj, mat, oh, mh) = (number(l), source_id(l), crypto_hash(&l.name), crypto_hash(&source_name(l))); + frags.par_chunks_mut(wu).zip(img.data.par_chunks(wu)).enumerate().for_each(|(y, (row, px))| { + for (x, (cell, p)) in row.iter_mut().zip(px).enumerate() { + if p[3] > 1e-4 { + let uv = [(x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32]; + cell.push(AuxFrag { z: plane_z, a: p[3].min(1.0), obj, mat, n: [0.0, 0.0, -1.0], uv, obj_hash: oh, mat_hash: mh }); + } + } + }); + } + } + // Resolve per pixel. + const NAMES: [&str; 9] = ["Z", "ObjectID", "MaterialID", "Coverage", "N.X", "N.Y", "N.Z", "UV.U", "UV.V"]; + const CRYPTO: usize = 16; + let resolved: Vec<([f32; 9], [f32; CRYPTO])> = frags + .par_iter() + .map(|fs| { + let mut o = [BACKGROUND_DEPTH, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let mut crypto = [0.0f32; CRYPTO]; + if fs.is_empty() { + return (o, crypto); + } + let mut t = 1.0f32; + let mut obj: Vec<(u32, f32)> = vec![]; + let mut mat: Vec<(u32, f32)> = vec![]; + let mut best: Option<(usize, f32)> = None; + let mut front: Option = None; + for (k, f) in fs.iter().enumerate() { + let c = f.a * t; + if front.is_none() && f.a >= 0.5 { + front = Some(k); + } + if best.is_none_or(|(_, bc)| c > bc) { + best = Some((k, c)); + } + for (list, hsh) in [(&mut obj, f.obj_hash), (&mut mat, f.mat_hash)] { + match list.iter_mut().find(|(x, _)| *x == hsh) { + Some(e) => e.1 += c, + None => list.push((hsh, c)), + } + } + t *= 1.0 - f.a; + if t <= 1e-5 { + break; + } + } + let f = fs[front.or(best.map(|b| b.0)).unwrap_or(0)]; + o = [f.z, f.obj, f.mat, 1.0 - t, f.n[0], f.n[1], f.n[2], f.uv[0], f.uv[1]]; + for (base, list) in [(0usize, &mut obj), (8usize, &mut mat)] { + list.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + for (rank, (hsh, c)) in list.iter().take(4).enumerate() { + crypto[base + rank * 2] = crypto_float(*hsh); + crypto[base + rank * 2 + 1] = *c; + } + } + (o, crypto) + }) + .collect(); + let mut aux = effectcraft_raster::AuxChannels::new(w, h, s); + for (k, name) in NAMES.iter().enumerate() { + aux.channels.push((name.to_string(), resolved.iter().map(|r| r.0[k]).collect())); + } + for (base, layer) in [(0usize, "CryptoObject"), (8usize, "CryptoMaterial")] { + for rank_pair in 0..2 { + for (c, ch) in ["R", "G", "B", "A"].iter().enumerate() { + let k = base + rank_pair * 4 + c; + aux.channels.push((format!("{layer}{rank_pair:02}.{ch}"), resolved.iter().map(|r| r.1[k]).collect())); + } + } + } + aux.manifests = vec![("CryptoObject".into(), manifest_obj), ("CryptoMaterial".into(), manifest_mat)]; + aux +} diff --git a/crates/render/src/three_d/light.rs b/crates/render/src/three_d/light.rs index 9476f98d..5a7963b4 100644 --- a/crates/render/src/three_d/light.rs +++ b/crates/render/src/three_d/light.rs @@ -117,7 +117,18 @@ impl LightState { /// All lights switched on and active at the context time. pub fn lights_at(ctx: &EvalCtx) -> Vec { - ctx.comp.layers.iter().filter(|l| l.is_light() && l.switches.video && l.is_active_at(ctx.time)).filter_map(|l| LightState::from_layer(ctx, l)).collect() + // Environment lights only light Advanced 3D comps (image-based light). + ctx.comp + .layers + .iter() + .filter(|l| { + l.is_light() + && l.switches.video + && l.is_active_at(ctx.time) + && l.source != (effectcraft_project::LayerSource::Light { kind: LightKind::Environment }) + }) + .filter_map(|l| LightState::from_layer(ctx, l)) + .collect() } /// Material Options of a 3D layer. diff --git a/crates/render/src/three_d/mod.rs b/crates/render/src/three_d/mod.rs index 8bef7360..e97c67fe 100644 --- a/crates/render/src/three_d/mod.rs +++ b/crates/render/src/three_d/mod.rs @@ -5,6 +5,7 @@ //! consecutive 3D layers is handed to [`compose::draw_run`] as a group (2D layers break 3D //! groups, as in After Effects). +pub mod adv; pub mod camera; pub(crate) mod compose; pub mod light; diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 32d471b3..8e345d18 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -716,7 +716,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index e9a40e61..3254a878 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -22,6 +22,7 @@ fn item_icon(it: &Item) -> Icon { ItemKind::Footage(f) => match f.kind { effectcraft_engine::project::FootageKind::Still | effectcraft_engine::project::FootageKind::Sequence => Icon::Image, effectcraft_engine::project::FootageKind::Audio => Icon::Audio, + effectcraft_engine::project::FootageKind::Model => Icon::Cube, _ => Icon::Footage, }, } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5b6c085f..c9a42698 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -433,6 +433,7 @@ fn layer_icon(l: &Layer, project: &effectcraft_engine::project::Project) -> Icon LayerSource::Null => Icon::Null, LayerSource::Camera => Icon::Camera, LayerSource::Light { .. } => Icon::Light, + LayerSource::Model { .. } | LayerSource::Primitive { .. } => Icon::Cube, LayerSource::Comp { .. } => Icon::Comp, LayerSource::Solid { .. } => { if l.switches.adjustment { diff --git a/docs/architecture.md b/docs/architecture.md index dceb0e0c..039bce3b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,6 +22,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `model` | 3D models for Advanced 3D: glTF 2.0 (`.gltf`/`.glb`) and OBJ/MTL import (meshes, PBR metallic-roughness materials and textures, node hierarchy, skins, animations), parametric primitives, extruded/bevelled outline meshes and polygon triangulation | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | | L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | @@ -70,7 +71,35 @@ the **source** (solid, footage frame, text, shape contents, or a nested comp), a then **effects** in order, then **layer styles**, then the **transform** into comp space (with motion blur sub-samples when enabled), the **track matte**, and finally **blends** into the accumulator. Adjustment layers apply their effects to the accumulator. Runs of 3D layers are composited per pixel through the -active camera, with lights, shadows and depth of field. +active camera, with lights, shadows and depth of field (Classic 3D), or rasterised as meshes by +the Advanced 3D renderer (below). + +**Advanced 3D** (Composition Settings ▸ 3D Renderer ▸ Advanced 3D, `comp.renderer`; +`crates/render/src/three_d/adv`, `crates/model`). A run of 3D layers becomes one scene of +world-space triangles: 3D model layers (glTF 2.0 / OBJ footage, mapped from +Y-up model units +into the layer by Geometry Options ▸ Model Scale; skins and animation clips played along layer +time), primitive layers (Layer ▸ New ▸ Cube/Sphere/Plane/Torus/Cone/Cylinder, parametric in +Geometry Options), text and shape layers with Geometry Options ▸ Extrusion Depth (glyph and +fill outlines flattened, triangulated with holes and extruded with Angular/Concave/Convex +bevels, per character so per-character 3D carries over), and a textured card for every other +3D layer. Shading is physically based in linear light (glTF metallic-roughness; Cook–Torrance +GGX specular, Smith–Schlick geometry, Schlick Fresnel, Lambert diffuse, after Karis 2013): +parallel, spot and point lights with After Effects' falloffs, ambient lights, image-based +light from an Environment light whose Source (or the comp's Environment Layer, Layer ▸ +Environment Layer) is an equirectangular image (cosine irradiance map; roughness-indexed mip +chain with the split-sum environment BRDF fit), and percentage-closer-filtered shadow maps +(spot: perspective, parallel: orthographic over the scene, point: six cube faces). Cards keep +Material Options (Accepts Lights/Shadows, Casts Shadows On/Only, Ambient, Diffuse, Specular); +models and primitives add Base Color, Metallic, Roughness, Emissive. Rasterisation runs at 2×2 +supersampling with a reversed-Z depth buffer; opaque triangles resolve visibility first, the +transparent ones are sorted back to front and blended. The software rasteriser +(`adv::raster`) is the reference and the headless/CI path; `effectcraft-gpu` implements +`Accelerator::raster_3d` with a wgpu render pipeline (`advanced3d.wgsl`) that repeats the +shading step for step, and tests compare the two. Resolve, depth of field (a gather blur by +each pixel's circle of confusion from the camera's Depth of Field settings) and the conversion +back to the working encoding run on the CPU for both. Not yet: motion blur, track mattes and +blend modes inside an Advanced 3D run, collapsed 3D precomps (drawn flattened), morph targets, +extruded strokes. A **layer cache** keeps each layer's finished pixels (source, masks and effects) keyed by a hash of its evaluated inputs, excluding the transform. Static and transform-only layers render once; diff --git a/docs/parity.md b/docs/parity.md index 02fa2175..f468ba0f 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **297 of 298** After Effects 2026 effects by name (99.7%, after M9.11 and Warp Stabilizer; was 257); only the 3D Camera Tracker is left | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -74,10 +74,10 @@ library groundwork for mask tracking and Warp Stabilizer. | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, -Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of +the Warp Stabilizer and mask-tracking UI on top of the new tracking library, SVG/PSD import, WebM, a disk cache, -the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and -wasm threads. +native macOS menus, and wasm threads. (The missing effect categories, 3D Channel, Immersive Video +and OCIO, landed in M9.11; the scripting object model in M14.4.) ## By area @@ -91,25 +91,37 @@ wasm threads. | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | -| Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | +| Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | -| Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | +| 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | +| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing -Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five -OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, -Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +The 3D Camera Tracker (built on the tracking library by other work). Boris FX Mocha and Cineware are third-party and not counted. +### Added in M9.11 (39 effects) + +| Category | Effects | Notes | +|---|---|---| +| 3D Channel | 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, Fog 3D, ID Matte, IDentifier | Read auxiliary channels: multi-layer OpenEXR (any `layer.channel`, Cryptomatte manifests from the header) or the compositor's own depth / layer-ID / normal / UV / Cryptomatte pass for precomps of 3D comps. | +| Immersive Video | VR Blur, VR Chromatic Aberrations, VR Color Gradients, VR Converter, VR De-Noise, VR Digital Glitch, VR Fractal Noise, VR Glow, VR Plane to Sphere, VR Rotate Sphere, VR Sharpen, VR Sphere To Plane | Seam-aware equirectangular processing (wrap-around, pole continuation, latitude-scaled filters), mono / over-under / side-by-side layouts; VR Converter between equirectangular, cube maps (4:3, 3:2, 6:1), sphere map and 2D. | +| Color Correction | OCIO CDL / Color Space / Display / File / Look Transform, Color Stabilizer | Built-in minimal OCIO-style config (ACES2065-1, ACEScg, ACEScct, sRGB, Rec.709, Rec.2020, Display P3, linear variants, XYZ) from published matrices and transfer functions; `.cube` / `.3dl` / `.csp` LUTs and ASC `.cc` / `.ccc` / `.cdl`. Custom `.ocio` config files are not read yet. | +| Distort | CC Flo Motion, Liquify, Rolling Shutter Repair | Liquify strokes are effect data replayed into a displacement mesh (`liquify.stroke` / `liquify.clear` commands; no interactive viewer brush yet). Rolling Shutter Repair uses the KLT tracker. | +| Blur & Sharpen | Camera-Shake Deblur, CC Radial Blur | Deblur substitutes aligned patches from sharper neighbouring frames. | +| Audio | Compressor, Distortion, Gate | Applied in the mixdown like the other audio effects. | +| Simulation | CC Hair, Particle Playground | Particle Playground: cannon, grid, layer exploder, layer map, gravity, repel, wall, persistent property mapper (no Particle Exploder, text particles or ephemeral mapper yet). | +| Keying | Key Light | The full Keylight 1.2 control set under a generic name ("Keylight" is a vendor trademark); `lookup("Keylight (1.2)")` finds it. | +| Utility | Color Profile Converter | Our colour spaces and ACES; rendering intents (perceptual gamut compression, relative / absolute colorimetric, saturation). | +| Obsolete | mocha shape | Mocha's export format is not public: reads a documented JSON shape format instead. | + ## Highest-value gaps, in order 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10). @@ -132,4 +144,4 @@ Mocha and Cineware are third-party and not counted. The engine underneath (keyframes, expressions, shape operators, text animators, Classic 3D, all 38 blend modes, the render queue) is deep. Most of what is missing is interactive tooling in the viewer, settings that are stored but not yet rendered, about 69 menu entries that are still -disabled, and the large systems: tracking, puppet, paint, roto, Advanced 3D and GPU rendering. +disabled, and the large systems: tracking, puppet, paint and roto (Advanced 3D landed in M7.4–M7.6). diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2532f2a2..2e2172e1 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -31,6 +31,7 @@ const LAYERS: &[(&str, u8)] = &[ ("text", 2), ("effects", 2), ("track", 2), + ("model", 2), ("render", 3), ("media", 3), ("expr", 3),