From 485f562e195926125fadf58264dd59409d0610ff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 10:59:08 -0400 Subject: [PATCH 1/3] M12.2: gpu compositor and gpu effects New L3 crate effectcraft-gpu: a wgpu compute backend for the per-frame compositing walk (layer transforms with the CPU's nearest/bilinear/bicubic sampling and minification pre-filter, motion-blur accumulation, track mattes, Preserve Transparency, layer style passes and knockout, all 38 blend modes as a WGSL port of color::blend, 8/16 bpc clamp+quantise, colour-space conversions). 3D runs, adjustment and wireframe layers run on the CPU between GPU steps. GPU effects with the CPU semantics: Gaussian, Fast Box and Directional Blur, Glow, Levels, Curves, Hue/Saturation, Tint, Fill, Gradient Ramp, Fractal Noise, Drop Shadow, Brightness & Contrast, Exposure, Invert, Transform (registry gpu badge from effects::GPU_EFFECTS). render: Backend (Cpu/Gpu/Auto) in RenderOpts, the Accelerator trait and Renderer::accel, GPU effect chains in the effect stack, hooks for the GPU walk. Project setting gpu_acceleration (Mercury GPU Acceleration / Software Only), render.backend command, file.projectSettings renderer. Session and Render Queue exports carry the accelerator. CLI --gpu and bench --gpu. Tests compare GPU vs CPU renders (blend modes x bit depths, transforms, masks, mattes, motion blur, colour management, styles, collapse, adjustment fallback, every GPU effect) and skip without an adapter. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 21 + Cargo.toml | 9 + apps/effectcraft-cli/Cargo.toml | 1 + apps/effectcraft-cli/src/main.rs | 97 ++- crates/effects/src/color2.rs | 6 + crates/effects/src/lib.rs | 27 + crates/engine/src/commands/comp_more.rs | 2 +- crates/engine/src/commands/file.rs | 49 +- crates/engine/src/commands/query.rs | 1 + crates/engine/src/commands/render_queue.rs | 3 +- crates/engine/src/lib.rs | 8 +- crates/engine/src/render_queue.rs | 7 +- crates/export/src/lib.rs | 4 + crates/export/tests/roundtrip.rs | 6 +- crates/gpu/Cargo.toml | 29 + crates/gpu/src/context.rs | 528 +++++++++++++++ crates/gpu/src/effects.rs | 347 ++++++++++ crates/gpu/src/lib.rs | 127 ++++ crates/gpu/src/ops.rs | 167 +++++ crates/gpu/src/shaders/common.wgsl | 452 +++++++++++++ crates/gpu/src/shaders/display.wgsl | 19 + crates/gpu/src/shaders/kernels.wgsl | 750 +++++++++++++++++++++ crates/gpu/src/tests.rs | 517 ++++++++++++++ crates/gpu/src/walk.rs | 188 ++++++ crates/host/src/lib.rs | 1 + crates/project/src/lib.rs | 5 + crates/render/src/lib.rs | 276 ++++++-- crates/ui-egui/src/lib.rs | 9 +- docs/architecture.md | 24 +- docs/parity.md | 7 +- 30 files changed, 3625 insertions(+), 62 deletions(-) create mode 100644 crates/gpu/Cargo.toml create mode 100644 crates/gpu/src/context.rs create mode 100644 crates/gpu/src/effects.rs create mode 100644 crates/gpu/src/lib.rs create mode 100644 crates/gpu/src/ops.rs create mode 100644 crates/gpu/src/shaders/common.wgsl create mode 100644 crates/gpu/src/shaders/display.wgsl create mode 100644 crates/gpu/src/shaders/kernels.wgsl create mode 100644 crates/gpu/src/tests.rs create mode 100644 crates/gpu/src/walk.rs diff --git a/Cargo.lock b/Cargo.lock index 6f31e68dd..3def0e94d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1157,6 +1157,7 @@ dependencies = [ "cpu-time", "effectcraft-automation", "effectcraft-engine", + "effectcraft-gpu", "effectcraft-host", "effectcraft-render", "effectcraft-time", @@ -1256,6 +1257,25 @@ dependencies = [ "serde", ] +[[package]] +name = "effectcraft-gpu" +version = "0.1.0" +dependencies = [ + "bytemuck", + "effectcraft-color", + "effectcraft-effects", + "effectcraft-geom", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-raster", + "effectcraft-render", + "effectcraft-text", + "effectcraft-time", + "log", + "pollster 0.4.0", + "wgpu", +] + [[package]] name = "effectcraft-host" version = "0.1.0" @@ -1406,6 +1426,7 @@ dependencies = [ "effectcraft-color", "effectcraft-engine", "effectcraft-geom", + "effectcraft-gpu", "effectcraft-host", "effectcraft-keyframe", "effectcraft-path", diff --git a/Cargo.toml b/Cargo.toml index 8212d74a4..a1941cf45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } effectcraft-track = { path = "crates/track" } effectcraft-render = { path = "crates/render" } +effectcraft-gpu = { path = "crates/gpu" } effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } @@ -66,6 +67,11 @@ filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cf filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# gpu (the same wgpu eframe / egui-wgpu use) +wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } +bytemuck = "1" +pollster = "0.4" + # ui egui = "0.36" eframe = { version = "0.36", default-features = false } @@ -108,6 +114,9 @@ opt-level = 3 [profile.dev.package.effectcraft-render] opt-level = 3 +[profile.dev.package.effectcraft-gpu] +opt-level = 3 + [profile.dev.package.effectcraft-effects] opt-level = 3 diff --git a/apps/effectcraft-cli/Cargo.toml b/apps/effectcraft-cli/Cargo.toml index 2adf08732..ff8842d92 100644 --- a/apps/effectcraft-cli/Cargo.toml +++ b/apps/effectcraft-cli/Cargo.toml @@ -17,6 +17,7 @@ effectcraft-engine = { workspace = true } effectcraft-host = { workspace = true } effectcraft-automation = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-gpu = { workspace = true } effectcraft-time = { workspace = true } serde_json = { workspace = true } image = { workspace = true } diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 43e758378..92e5c1fca 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -13,11 +13,14 @@ //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue -//! effectcraft-cli bench [--comp C] [--time S] [--scale K] [--n N] [--play N] render timings +//! effectcraft-cli bench [--comp C] [--time S] [--scale K] [--n N] [--play N] [--gpu] render timings +//! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half) //! effectcraft-cli mcp [--bridge PORT] MCP server on stdio //! //! Project: --project F.ecproj (or a positional *.ecproj) | --demo | --empty (default: demo; mcp: empty) //! Saving: --save (back to --project) | --save-as F.ecproj +//! GPU: --gpu renders on the GPU compositor (Mercury GPU Acceleration) when an adapter exists; +//! the default is the CPU (Mercury Software Only) //! Bridge: --bridge PORT drives a running `effectcraft --control PORT` instead of a headless session //! Output: --json for one compact JSON document on stdout (errors: {"error": …}, exit 1) //! ``` @@ -41,10 +44,11 @@ const USAGE: &str = "usage: effectcraft-cli : id or name, '-' = active comp; : id, '#n' or name; : JSON or bare string"; /// Options that take a value. @@ -196,7 +200,7 @@ fn backend(args: &Args, default_demo: bool) -> Result { } return Ok(Backend::bridge(addr)?); } - let mut b = Backend::headless(effectcraft_host::session()); + let mut b = Backend::headless(session(args)?); if let Some(p) = &args.project { b.exec("file.open", json!({"path": p}))?; } else if args.flag("--demo") || (default_demo && !args.flag("--empty")) { @@ -205,6 +209,17 @@ fn backend(args: &Args, default_demo: bool) -> Result { Ok(b) } +/// A wired session; `--gpu` attaches the GPU compositor (renders then follow the project's +/// renderer setting, Mercury GPU Acceleration by default). +fn session(args: &Args) -> Result { + let mut s = effectcraft_host::session(); + if args.flag("--gpu") { + let g = effectcraft_gpu::Gpu::headless().ok_or_else(|| Failure::Error("--gpu: no usable GPU adapter".into()))?; + s.accel = Some(std::sync::Arc::new(g)); + } + Ok(s) +} + /// `--save` / `--save-as` after a mutating command. fn maybe_save(b: &mut Backend, args: &Args) -> Result, Failure> { let r = if let Some(p) = args.opt("--save-as") { @@ -436,7 +451,7 @@ fn render(args: &Args, json_out: bool) -> Result<(), Failure> { if args.opt("--bridge").is_some() { return usage_err("render runs headless; use `exec renderQueue.add` / `renderQueue.render` with --bridge"); } - let mut s = effectcraft_host::session(); + let mut s = session(args)?; match &args.project { Some(p) => s.execute("file.open", json!({"path": p})).map_err(|e| Failure::Error(e.to_string()))?, None => s.execute("file.openDemoProject", json!({})).map_err(|e| Failure::Error(e.to_string()))?, @@ -549,6 +564,9 @@ fn bench_cmd(args: &Args) -> Result<(), Failure> { if let Some(c) = args.opt("--comp") { s.execute("comp.open", json!({"comp": c})).map_err(|e| Failure::Error(e.to_string()))?; } + if args.flag("--gpu") { + return bench_gpu(&s, args); + } let cid = s.active_comp_id().ok_or_else(|| Failure::Error("no composition".into()))?; let t = Tick::from_seconds_f64(args.num("--time")?.unwrap_or(3.0)); let opts = RenderOpts { scale: args.num("--scale")?.unwrap_or(1.0), ..Default::default() }; @@ -664,3 +682,70 @@ fn bench_play(s: &Session, cid: ItemId, t: Tick, opts: RenderOpts, n: usize) { let st = cache.stats(); eprintln!(" layer cache: {} hits, {} misses, {} entries, {:.1} MB", st.hits, st.misses, st.entries, st.bytes as f64 / 1e6); } + +/// Median wall time (ms) of `n` runs of `f` after one warm-up run. +fn time_ms(n: usize, mut f: impl FnMut()) -> f64 { + f(); + let mut v: Vec = (0..n) + .map(|_| { + let t0 = std::time::Instant::now(); + f(); + t0.elapsed().as_secs_f64() * 1e3 + }) + .collect(); + median(&mut v) +} + +/// `bench --gpu`: CPU vs GPU ms/frame for every comp (or `--comp`) at Full and Half. "cold" +/// renders everything (no layer cache); "warm" reuses the layer cache (playback / scrubbing of +/// unchanged layers: compositing cost); "viewer" is the GPU display path without readback. +fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> { + use effectcraft_render::Backend; + let gpu = effectcraft_gpu::Gpu::headless().ok_or_else(|| Failure::Error("--gpu: no usable GPU adapter".into()))?; + let n = args.num("--n")?.unwrap_or(10.0).max(1.0) as usize; + let comps: Vec = match args.opt("--comp") { + Some(_) => s.active_comp_id().into_iter().collect(), + None => s.project.comps().map(|(id, _)| *id).collect(), + }; + eprintln!("GPU: {} — median of {n} runs (ms/frame)", effectcraft_render::Accelerator::name(&gpu)); + eprintln!("{:<28} {:>5} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9}", "comp", "res", "size", "cpu cold", "gpu cold", "cpu warm", "gpu warm", "gpu view"); + for cid in comps { + let Some(comp) = s.project.comp(cid) else { continue }; + let name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let t = Tick::from_seconds_f64(args.num("--time")?.unwrap_or(3.0).min(comp.duration.seconds() * 0.5)); + for (label, scale) in [("Full", 1.0), ("Half", 0.5)] { + let mk = |backend: Backend, cache: Option<&LayerCache>| { + let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts { scale, backend, ..Default::default() }); + r.expr = s.expr.as_deref(); + r.cache = cache; + r.accel = Some(&gpu); + r.comp_frame(cid, t) + }; + let cpu_cold = time_ms(n, || { + std::hint::black_box(mk(Backend::Cpu, None)); + }); + let gpu_cold = time_ms(n, || { + std::hint::black_box(mk(Backend::Gpu, None)); + }); + let cache = LayerCache::default(); + let cpu_warm = time_ms(n, || { + std::hint::black_box(mk(Backend::Cpu, Some(&cache))); + }); + let gcache = LayerCache::default(); + let gpu_warm = time_ms(n, || { + std::hint::black_box(mk(Backend::Gpu, Some(&gcache))); + }); + let view = time_ms(n, || { + let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts { scale, backend: Backend::Gpu, ..Default::default() }); + r.expr = s.expr.as_deref(); + r.cache = Some(&gcache); + r.accel = Some(&gpu); + std::hint::black_box(gpu.render_display(&r, cid, t)); + gpu.wait(); + }); + let size = format!("{}x{}", (comp.width as f64 * scale).round(), (comp.height as f64 * scale).round()); + eprintln!("{name:<28} {label:>5} {size:>10} {cpu_cold:>9.2} {gpu_cold:>9.2} {cpu_warm:>9.2} {gpu_warm:>9.2} {view:>9.2}"); + } + } + Ok(()) +} diff --git a/crates/effects/src/color2.rs b/crates/effects/src/color2.rs index 8d707c685..8ed52d2b3 100644 --- a/crates/effects/src/color2.rs +++ b/crates/effects/src/color2.rs @@ -119,6 +119,12 @@ impl Curve { h00 * self.ys[k] + h10 * h * self.ms[k] + h01 * self.ys[k + 1] + h11 * h * self.ms[k + 1] } + /// Control points (x, y), Hermite tangents and the 1025-entry table over 0..1 that + /// [`Curve::eval`] interpolates (the GPU path uploads them). + pub fn tables(&self) -> (&[f32], &[f32], &[f32], &[f32]) { + (&self.xs, &self.ys, &self.ms, &self.lut) + } + pub fn eval(&self, x: f32) -> f32 { if !(0.0..=1.0).contains(&x) { return self.eval_exact(x); diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 7c6946220..2e29061a1 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -353,6 +353,11 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); + for s in v.iter_mut() { + if GPU_EFFECTS.contains(&s.id) { + s.gpu = true; + } + } v }) } @@ -411,6 +416,28 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ g } +/// Effects the GPU compositor (`effectcraft-gpu`) implements with the CPU effect's semantics +/// (Mercury GPU Acceleration; the Effects & Presets GPU badge). Expression controls are +/// pass-throughs and count as GPU effects too. +pub const GPU_EFFECTS: &[&str] = &[ + "ec.blur.gaussian", + "ec.blur.fastbox", + "ec.blur.directional", + "ec.stylize.glow", + "ec.color.levels", + "ec.color.curves", + "ec.color.huesaturation", + "ec.color.tint", + "ec.generate.fill", + "ec.generate.gradientramp", + "ec.noise.fractal", + "ec.perspective.dropshadow", + "ec.color.brightnesscontrast", + "ec.color.exposure", + "ec.channel.invert", + "ec.distort.transform", +]; + /// Effects whose output depends on [`EffectCtx::time`] directly (not only through animated /// parameters). The renderer's layer cache folds the layer time into the key for these. /// The `time_dependence_is_declared` test checks the list against every registered effect. diff --git a/crates/engine/src/commands/comp_more.rs b/crates/engine/src/commands/comp_more.rs index 6a066b5ac..e3057f67d 100644 --- a/crates/engine/src/commands/comp_more.rs +++ b/crates/engine/src/commands/comp_more.rs @@ -89,7 +89,7 @@ fn save_frame(s: &mut Session, p: &Value) -> Result { let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(s.time()); let scale = f_p(p, "scale").unwrap_or(1.0).clamp(0.01, 4.0); let bg = comp.background; - let img = s.render(cid, t, effectcraft_render::RenderOpts { scale, ..Default::default() }); + let img = s.render(cid, t, effectcraft_render::RenderOpts { scale, backend: effectcraft_render::Backend::Auto, ..Default::default() }); let rgba = img.to_rgba8_over(bg); let png = encode_png(&rgba, img.width, img.height).map_err(|e| EngineError::Other(format!("PNG encoding failed: {e}")))?; s.services.write_file(&path, &png).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index ee61f9007..a85a391fe 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -152,6 +152,17 @@ fn project_settings(s: &mut Session, p: &Value) -> Result { ), }; } + if let Some(r) = p.get("renderer").or_else(|| p.get("gpuAcceleration")) { + proj.settings.gpu_acceleration = match r { + Value::Bool(b) => *b, + Value::String(s) => match effectcraft_render::Backend::parse(s) { + Some(effectcraft_render::Backend::Cpu) => false, + Some(_) => true, + None => return Err(bad("file.projectSettings", format!("renderer: gpu|software, not `{s}`"))), + }, + _ => return Err(bad("file.projectSettings", "renderer: gpu|software")), + }; + } if let Some(t) = str_p(p, "timeDisplay") { proj.settings.time_display = if t.eq_ignore_ascii_case("frames") { effectcraft_project::TimeDisplayStyle::Frames } else { effectcraft_project::TimeDisplayStyle::Timecode }; @@ -161,6 +172,41 @@ fn project_settings(s: &mut Session, p: &Value) -> Result { Ok(serde_json::to_value(&s.project.settings).unwrap_or_default()) } +/// Video Rendering and Effects: report or set the renderer (Mercury GPU Acceleration when a GPU +/// adapter exists, else Mercury Software Only). +fn render_backend(s: &mut Session, p: &Value) -> Result { + if let Some(b) = p.get("backend").or_else(|| p.get("renderer")) { + let gpu = match b { + Value::Bool(b) => *b, + Value::String(v) => match effectcraft_render::Backend::parse(v) { + Some(effectcraft_render::Backend::Cpu) => false, + Some(_) => true, + None => return Err(bad("render.backend", format!("backend: gpu|cpu, not `{v}`"))), + }, + _ => return Err(bad("render.backend", "backend: gpu|cpu")), + }; + if gpu != s.project.settings.gpu_acceleration { + s.edit("Project Settings", None, |proj, _| { + proj.settings.gpu_acceleration = gpu; + Ok(()) + })?; + } + } + Ok(backend_status(s)) +} + +/// The renderer setting and what renders with it. +pub fn backend_status(s: &Session) -> Value { + let adapter = s.accel.as_ref().map(|a| a.name()); + let gpu = s.project.settings.gpu_acceleration; + json!({ + "renderer": if gpu { "Mercury GPU Acceleration" } else { "Mercury Software Only" }, + "gpuAcceleration": gpu, + "adapter": adapter, + "active": if gpu && adapter.is_some() { "gpu" } else { "cpu" }, + }) +} + fn cycle_depth(s: &mut Session, _: &Value) -> Result { s.edit("Project Bit Depth", None, |proj, _| { proj.settings.bit_depth = proj.settings.bit_depth.next(); @@ -184,10 +230,11 @@ pub fn specs() -> Vec { "Project Settings...", ["File"], Some("Cmd+Alt+Shift+K"), - "{bitDepth?: 8|16|32, workingSpace?: none|srgb|rec709|rec2020|p3, linearize?, blendLinear?, timeDisplay?: timecode|frames}", + "{bitDepth?: 8|16|32, workingSpace?: none|srgb|rec709|rec2020|p3, linearize?, blendLinear?, renderer?: gpu|software, timeDisplay?: timecode|frames}", always, project_settings ), cmd!("file.cycleBitDepth", "Cycle Project Bit Depth", [], None, "{}", always, cycle_depth), + cmd!("render.backend", "Video Rendering and Effects", [], None, "{backend?: gpu|cpu}", always, render_backend), ] } diff --git a/crates/engine/src/commands/query.rs b/crates/engine/src/commands/query.rs index f38231501..69a9dc3be 100644 --- a/crates/engine/src/commands/query.rs +++ b/crates/engine/src/commands/query.rs @@ -47,6 +47,7 @@ fn summary(s: &mut Session, _: &Value) -> Result { "path": s.path, "dirty": s.is_dirty(), "bitDepth": s.project.settings.bit_depth.label(), + "renderer": super::file::backend_status(s), "items": items, "activeComp": s.state.active_comp.map(|c| c.0), "time": s.time().seconds(), diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index d22384dd3..c905ce6c2 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -495,7 +495,8 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { let mut item = RenderQueueItem::new(0, cid); item.output = OutputModule::for_format(format); item.output.output = path.clone(); - let job = crate::ExportJob { project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), item: &item, path: &path }; + let job = + crate::ExportJob { project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), accel: s.accel.as_deref(), item: &item, path: &path }; let r = exporter.export(&job, &mut |_, _| true).map_err(EngineError::Other)?; s.toast(format!("Saved preview to {}", r.path)); Ok(serde_json::to_value(&r).unwrap_or_default()) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 564dbc1bd..adb510a47 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -225,6 +225,10 @@ pub struct Session { pub services: Arc, pub footage: Arc, pub expr: Option>, + /// GPU compositor (Mercury GPU Acceleration), when the frontend has one: used by renders + /// that ask for [`effectcraft_render::Backend::Gpu`]/`Auto`, and by Render Queue exports + /// when the project's renderer is the GPU. + pub accel: Option>, /// Expression syntax checker (set by the host that links the expression engine). pub expr_check: Option std::result::Result<(), String>>, pub importer: Option>, @@ -268,6 +272,7 @@ impl Default for Session { services: Arc::new(FsServices), footage: Arc::new(NoFootage), expr: None, + accel: None, expr_check: None, importer: None, exporter: None, @@ -464,6 +469,7 @@ impl Session { let mut r = Renderer::new(&self.project, self.footage.as_ref(), opts); r.expr = self.expr.as_deref(); r.cache = Some(&self.layer_cache); + r.accel = self.accel.as_deref(); r.comp_frame(comp, t) } @@ -482,7 +488,7 @@ impl Session { let c = self.project.comp(comp).ok_or(EngineError::NoComp)?; let long = c.width.max(c.height).max(1) as f64; let scale = if max_side == 0 { 1.0 } else { (max_side as f64 / long).min(1.0) }; - let img = self.render(comp, t, RenderOpts { scale, ..Default::default() }); + let img = self.render(comp, t, RenderOpts { scale, backend: effectcraft_render::Backend::Auto, ..Default::default() }); Ok((img.width, img.height, img.to_rgba8_over(c.background))) } diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index 95c908242..bc490f472 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -23,6 +23,8 @@ pub struct ExportJob<'a> { pub footage: &'a dyn FootageSource, pub expr: Option<&'a dyn ExprHost>, pub item: &'a RenderQueueItem, + /// GPU compositor to render with when the project's renderer is Mercury GPU Acceleration. + pub accel: Option<&'a dyn effectcraft_render::Accelerator>, /// Resolved output path (templates expanded except the `#` frame-number run). pub path: &'a str, } @@ -120,6 +122,7 @@ struct Work { project: Arc, footage: Arc, expr: Option>, + accel: Option>, exporter: Arc, /// Per queue item: one (item with that output module, resolved path) per output module. items: Vec>, @@ -155,7 +158,7 @@ fn run_work(w: Work, shared: &JobShared) { // first module's file is the item's output. let mut r: Result = Err("no output module".into()); for (mi, (mitem, path)) in modules.iter().enumerate() { - let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), item: mitem, path }; + let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), accel: w.accel.as_deref(), item: mitem, path }; let rr = w.exporter.export(&job, &mut |done, total| { let mut s = lock(&shared.state); s.done = done; @@ -288,7 +291,7 @@ impl Session { return Err("nothing is queued: add a composition (Composition ▸ Add to Render Queue) and tick Render".into()); } let ids: Vec = items.iter().filter_map(|m| m.first().map(|(i, _)| i.id)).collect(); - let work = Work { project: self.project.clone(), footage: self.footage.clone(), expr: self.expr.clone(), exporter, items }; + let work = Work { project: self.project.clone(), footage: self.footage.clone(), expr: self.expr.clone(), accel: self.accel.clone(), exporter, items }; let shared = Arc::new(JobShared::default()); if wait { run_work(work, &shared); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 1616926d8..87c9f80b4 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -56,6 +56,8 @@ pub struct Job<'a> { pub project: &'a Project, pub footage: &'a dyn FootageSource, pub expr: Option<&'a dyn ExprHost>, + /// GPU compositor: used when the project's renderer is Mercury GPU Acceleration. + pub accel: Option<&'a dyn effectcraft_render::Accelerator>, pub comp: ItemId, pub settings: &'a RenderSettings, pub output: &'a OutputModule, @@ -172,9 +174,11 @@ pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { draft: job.settings.quality == RenderQuality::Draft, // Output renders always look through the comp's active camera. view: None, + backend: effectcraft_render::Backend::Auto, }; let mut r = Renderer::new(job.project, job.footage, opts); r.expr = job.expr; + r.accel = job.accel; r.comp_frame(job.comp, t) } diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 761b2c67f..186aa1a97 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -55,7 +55,7 @@ fn settings() -> RenderSettings { fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); - let job = Job { project: p, footage, expr: None, comp: cid, settings: &s, output: om, path: &path, sink: None }; + let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None }; let mut last = Progress::default(); let mut calls = 0; let r = export(&job, &mut |pr| { @@ -167,7 +167,7 @@ fn h264_odd_size_is_cropped_even() { let comp = p.comp(cid).expect("comp"); assert_eq!(s.output_size(comp).0 % 2, 1); let path = d.join("odd.mp4").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; let r = export(&job, &mut |_| true).expect("export"); assert_eq!((r.width % 2, r.height % 2), (0, 0)); } @@ -287,7 +287,7 @@ fn cancel_stops_early() { let om = OutputModule::for_format(OutputFormat::PngSequence); let s = settings(); let path = d.join("c_[#####].png").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; let r = export(&job, &mut |_| false); assert!(matches!(r, Err(effectcraft_export::ExportError::Cancelled))); let written = std::fs::read_dir(&d).map(|r| r.count()).unwrap_or(0); diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml new file mode 100644 index 000000000..36c751df6 --- /dev/null +++ b/crates/gpu/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "effectcraft-gpu" +description = "The GPU compositor (wgpu compute): layer transforms, masks/mattes, the 38 blend modes, motion blur, bit-depth quantisation and GPU effects, with the CPU renderer as the reference" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-time = { workspace = true } +effectcraft-geom = { workspace = true } +effectcraft-color = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-effects = { workspace = true } +effectcraft-render = { workspace = true } +effectcraft-keyframe = { workspace = true } +wgpu = { workspace = true } +bytemuck = { workspace = true } +log = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +pollster = { workspace = true } + +[dev-dependencies] +effectcraft-text = { workspace = true } diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs new file mode 100644 index 000000000..9f814377e --- /dev/null +++ b/crates/gpu/src/context.rs @@ -0,0 +1,528 @@ +//! Device, pipelines, textures, uploads and readback. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, Weak}; + +use effectcraft_effects::Buf; +use effectcraft_raster::Image; +use wgpu::util::DeviceExt; + +/// Compute entry points in `kernels.wgsl` (one pipeline each). +const ENTRIES: &[&str] = &[ + "warp_blend", + "blend_full", + "matte", + "preserve", + "knockout", + "channel_mix", + "quantize", + "convert", + "half", + "box_h", + "box_v", + "directional", + "glow_bright", + "glow_combine", + "shadow_make", + "shadow_combine", + "pointwise", +]; + +/// Pixel format of every working texture: premultiplied RGBA, 32-bit float (32 bpc headroom; +/// 8/16 bpc are emulated by clamping and quantising, as on the CPU). +pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba32Float; + +/// Uploaded layer buffers kept on the GPU (bytes). +const UPLOAD_BUDGET: usize = 1536 << 20; + +/// A GPU image: premultiplied RGBA f32 in a texture (cheap to clone; immutable once written). +#[derive(Clone, Debug)] +pub struct GpuImage { + pub texture: wgpu::Texture, + pub width: u32, + pub height: u32, +} + +/// Uniform parameter block shared by every kernel (see `shaders/common.wgsl`). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Params { + pub u: [[u32; 4]; 4], + pub f: [[f32; 4]; 12], +} + +impl Params { + fn bytes(&self) -> Vec { + let mut v = Vec::with_capacity(256); + for r in &self.u { + for x in r { + v.extend_from_slice(&x.to_le_bytes()); + } + } + for r in &self.f { + for x in r { + v.extend_from_slice(&x.to_le_bytes()); + } + } + v + } +} + +struct Upload { + buf: Weak, + img: GpuImage, + bytes: usize, + last_use: u64, +} + +#[derive(Default)] +struct Uploads { + map: HashMap, + bytes: usize, + clock: u64, +} + +/// The device and everything compiled for it. +pub struct GpuContext { + pub device: wgpu::Device, + pub queue: wgpu::Queue, + pub(crate) name: String, + pub(crate) max_dim: u32, + bgl: wgpu::BindGroupLayout, + pipelines: HashMap<&'static str, wgpu::ComputePipeline>, + display_bgl: wgpu::BindGroupLayout, + display: wgpu::ComputePipeline, + dummy_tex: wgpu::TextureView, + dummy_buf: wgpu::Buffer, + uploads: Mutex, +} + +fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + } +} + +fn storage_tex_entry(binding: u32, format: wgpu::TextureFormat) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { access: wgpu::StorageTextureAccess::WriteOnly, format, view_dimension: wgpu::TextureViewDimension::D2 }, + count: None, + } +} + +impl GpuContext { + /// Build on an existing device (the desktop app shares egui-wgpu's). `Err` when the + /// adapter cannot run the compositor (no compute shaders, e.g. WebGL2; no float storage + /// textures). + pub fn new(adapter: &wgpu::Adapter, device: wgpu::Device, queue: wgpu::Queue) -> Result { + let info = adapter.get_info(); + if !adapter.get_downlevel_capabilities().flags.contains(wgpu::DownlevelFlags::COMPUTE_SHADERS) { + return Err(format!("{} ({:?}): no compute shaders", info.name, info.backend)); + } + for f in [FORMAT, wgpu::TextureFormat::Rgba8Unorm] { + if !adapter.get_texture_format_features(f).allowed_usages.contains(wgpu::TextureUsages::STORAGE_BINDING) { + return Err(format!("{} ({:?}): {f:?} storage textures unsupported", info.name, info.backend)); + } + } + let name = format!("{} ({:?})", info.name, info.backend); + let src = [include_str!("shaders/common.wgsl"), include_str!("shaders/kernels.wgsl")].concat(); + let module = + device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("effectcraft kernels"), source: wgpu::ShaderSource::Wgsl(src.into()) }); + let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effectcraft kernels"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }, + tex_entry(1), + tex_entry(2), + storage_tex_entry(3, FORMAT), + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Storage { read_only: true }, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effectcraft kernels"), + bind_group_layouts: &[Some(&bgl)], + immediate_size: 0, + }); + let pipelines = ENTRIES + .iter() + .map(|e| { + let p = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some(e), + layout: Some(&layout), + module: &module, + entry_point: Some(e), + compilation_options: Default::default(), + cache: None, + }); + (*e, p) + }) + .collect(); + let dmodule = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effectcraft display"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/display.wgsl").into()), + }); + let display_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effectcraft display"), + entries: &[tex_entry(1), storage_tex_entry(3, wgpu::TextureFormat::Rgba8Unorm)], + }); + let dlayout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effectcraft display"), + bind_group_layouts: &[Some(&display_bgl)], + immediate_size: 0, + }); + let display = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("display"), + layout: Some(&dlayout), + module: &dmodule, + entry_point: Some("display"), + compilation_options: Default::default(), + cache: None, + }); + let dummy = device.create_texture(&wgpu::TextureDescriptor { + label: Some("dummy"), + size: wgpu::Extent3d { width: 1, height: 1, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + let dummy_buf = + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("dummy"), contents: &[0u8; 16], usage: wgpu::BufferUsages::STORAGE }); + let max_dim = device.limits().max_texture_dimension_2d; + Ok(GpuContext { + device, + queue, + name, + max_dim, + bgl, + pipelines, + display_bgl, + display, + dummy_tex: dummy.create_view(&Default::default()), + dummy_buf, + uploads: Mutex::new(Uploads::default()), + }) + } + + /// A device of its own on the best adapter (CLI, tests, benchmarks). `None` when no + /// adapter qualifies. + #[cfg(not(target_arch = "wasm32"))] + pub fn headless() -> Option { + let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); + let adapter = pollster::block_on( + instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }), + ) + .ok()?; + let limits = adapter.limits(); + let required_limits = wgpu::Limits { + max_texture_dimension_2d: limits.max_texture_dimension_2d.min(16384), + max_buffer_size: limits.max_buffer_size, + ..wgpu::Limits::default() + }; + let (device, queue) = + pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { label: Some("effectcraft gpu"), required_limits, ..Default::default() })) + .ok()?; + device.on_uncaptured_error(Arc::new(|e| log::error!("wgpu: {e}"))); + GpuContext::new(&adapter, device, queue).map_err(|e| log::info!("gpu: {e}")).ok() + } + + /// Readback (GPU → CPU) waits for the device; impossible on the browser's main thread. + pub fn can_readback(&self) -> bool { + cfg!(not(target_arch = "wasm32")) + } + + pub(crate) fn fits(&self, w: u32, h: u32) -> bool { + w > 0 && h > 0 && w <= self.max_dim && h <= self.max_dim + } + + /// A new (zeroed) working texture. + pub(crate) fn image(&self, w: u32, h: u32) -> GpuImage { + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: None, + size: wgpu::Extent3d { width: w.max(1), height: h.max(1), depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: FORMAT, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + GpuImage { texture, width: w, height: h } + } + + /// Upload a CPU image (`None` when it does not fit the device). + pub fn upload_image(&self, img: &Image) -> Option { + if !self.fits(img.width, img.height) { + return None; + } + let g = self.image(img.width, img.height); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { texture: &g.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + bytemuck::cast_slice(&img.data), + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(img.width * 16), rows_per_image: Some(img.height) }, + wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 }, + ); + Some(g) + } + + /// Upload a (layer cache) buffer, reusing the texture while the same `Arc` is alive: static + /// layers upload once, not every frame. + pub(crate) fn upload_buf(&self, buf: &Arc) -> Option { + let key = Arc::as_ptr(buf) as usize; + if let Ok(mut u) = self.uploads.lock() { + u.clock += 1; + let now = u.clock; + if let Some(e) = u.map.get_mut(&key) + && e.buf.upgrade().is_some_and(|b| Arc::ptr_eq(&b, buf)) + { + e.last_use = now; + return Some(e.img.clone()); + } + } + let img = self.upload_image(&buf.img)?; + if let Ok(mut u) = self.uploads.lock() { + let bytes = buf.img.data.len() * 16; + let now = u.clock; + if let Some(old) = u.map.insert(key, Upload { buf: Arc::downgrade(buf), img: img.clone(), bytes, last_use: now }) { + u.bytes -= old.bytes; + } + u.bytes += bytes; + // Drop textures of freed buffers, then the least recently used over budget. + if u.bytes > UPLOAD_BUDGET { + let dead: Vec = u.map.iter().filter(|(_, e)| e.buf.strong_count() == 0).map(|(k, _)| *k).collect(); + for k in dead { + if let Some(e) = u.map.remove(&k) { + u.bytes -= e.bytes; + } + } + } + while u.bytes > UPLOAD_BUDGET { + let Some(k) = u.map.iter().min_by_key(|(_, e)| e.last_use).map(|(k, _)| *k) else { break }; + if let Some(e) = u.map.remove(&k) { + u.bytes -= e.bytes; + } + } + } + Some(img) + } + + /// Forget uploaded layer textures. + pub fn clear_uploads(&self) { + if let Ok(mut u) = self.uploads.lock() { + u.map.clear(); + u.bytes = 0; + } + } +} + +/// Records GPU work for one render (a command encoder submitted at readback or the end). +pub(crate) struct Enc<'g> { + pub g: &'g GpuContext, + enc: Option, + pending: usize, +} + +impl<'g> Enc<'g> { + pub fn new(g: &'g GpuContext) -> Enc<'g> { + Enc { g, enc: None, pending: 0 } + } + + fn encoder(&mut self) -> &mut wgpu::CommandEncoder { + let g = self.g; + self.enc.get_or_insert_with(|| g.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("effectcraft") })) + } + + /// A new zeroed image. + pub fn image(&self, w: u32, h: u32) -> GpuImage { + self.g.image(w, h) + } + + /// Run kernel `entry` writing `out` (dispatched over `groups` workgroups). + pub fn dispatch( + &mut self, + entry: &str, + p: &Params, + src: &GpuImage, + aux: Option<&GpuImage>, + out: &GpuImage, + data: Option<&wgpu::Buffer>, + groups: (u32, u32), + ) { + let g = self.g; + let Some(pipe) = g.pipelines.get(entry) else { + log::error!("gpu: no kernel {entry}"); + return; + }; + let ub = g.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: &p.bytes(), usage: wgpu::BufferUsages::UNIFORM }); + let sv = src.texture.create_view(&Default::default()); + let av = aux.map(|a| a.texture.create_view(&Default::default())); + let ov = out.texture.create_view(&Default::default()); + let bg = g.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &g.bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: ub.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&sv) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(av.as_ref().unwrap_or(&g.dummy_tex)) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&ov) }, + wgpu::BindGroupEntry { binding: 4, resource: data.unwrap_or(&g.dummy_buf).as_entire_binding() }, + ], + }); + let enc = self.encoder(); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some(entry), timestamp_writes: None }); + pass.set_pipeline(pipe); + pass.set_bind_group(0, &bg, &[]); + pass.dispatch_workgroups(groups.0.max(1), groups.1.max(1), 1); + } + self.pending += 1; + // Keep command buffers (and the transient textures they hold) bounded. + if self.pending >= 256 { + self.submit(); + } + } + + /// Per-pixel kernel over `out`'s size (16×16 workgroups). + pub fn pixels(&mut self, entry: &str, p: &Params, src: &GpuImage, aux: Option<&GpuImage>, out: &GpuImage, data: Option<&wgpu::Buffer>) { + let groups = (out.width.div_ceil(16), out.height.div_ceil(16)); + self.dispatch(entry, p, src, aux, out, data, groups); + } + + /// Copy `src` into a larger zeroed image at (`x`, `y`) (Buf::pad). + pub fn copy_into(&mut self, src: &GpuImage, dst: &GpuImage, x: u32, y: u32) { + let enc = self.encoder(); + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { texture: &src.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyTextureInfo { texture: &dst.texture, mip_level: 0, origin: wgpu::Origin3d { x, y, z: 0 }, aspect: wgpu::TextureAspect::All }, + wgpu::Extent3d { width: src.width, height: src.height, depth_or_array_layers: 1 }, + ); + } + + /// Submit recorded work. + pub fn submit(&mut self) { + if let Some(enc) = self.enc.take() { + self.g.queue.submit([enc.finish()]); + } + self.pending = 0; + } + + /// Read an image back to the CPU (submits and waits). `None` where readback is impossible. + pub fn download(&mut self, img: &GpuImage) -> Option { + let bytes = self.read_texture(&img.texture, img.width, img.height, 16)?; + let mut out = Image::new(img.width, img.height); + for (p, c) in out.data.iter_mut().zip(bytes.chunks_exact(16)) { + for (k, v) in p.iter_mut().enumerate() { + *v = f32::from_le_bytes([c[4 * k], c[4 * k + 1], c[4 * k + 2], c[4 * k + 3]]); + } + } + Some(out) + } + + /// Tightly packed texel bytes of a texture (`bpp` bytes per pixel). + pub fn read_texture(&mut self, texture: &wgpu::Texture, w: u32, h: u32, bpp: u32) -> Option> { + if !self.g.can_readback() { + return None; + } + let row = w * bpp; + let padded = row.div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let size = padded as u64 * h as u64; + let buf = self.g.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + self.encoder().copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(padded), rows_per_image: Some(h) } }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + self.submit(); + let (tx, rx) = std::sync::mpsc::channel(); + buf.map_async(wgpu::MapMode::Read, .., move |r| { + let _ = tx.send(r); + }); + if let Err(e) = self.g.device.poll(wgpu::PollType::wait_indefinitely()) { + log::error!("gpu: poll: {e}"); + return None; + } + rx.recv().ok()?.ok()?; + let view = buf.get_mapped_range(..).ok()?; + let mut out = Vec::with_capacity((row * h) as usize); + for y in 0..h as usize { + out.extend_from_slice(&view[y * padded as usize..y * padded as usize + row as usize]); + } + drop(view); + buf.unmap(); + Some(out) + } + + /// A data buffer for kernels that read tables (curves). + pub fn data(&self, v: &[f32]) -> wgpu::Buffer { + let mut bytes = Vec::with_capacity(v.len().max(4) * 4); + for x in v { + bytes.extend_from_slice(&x.to_le_bytes()); + } + while bytes.len() < 16 { + bytes.push(0); + } + self.g.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("data"), contents: &bytes, usage: wgpu::BufferUsages::STORAGE }) + } + + /// Convert to the viewer's premultiplied RGBA8 texture (egui-wgpu's native texture format). + pub fn display(&mut self, img: &GpuImage) -> wgpu::Texture { + let g = self.g; + let tex = g.device.create_texture(&wgpu::TextureDescriptor { + label: Some("viewer frame"), + size: wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let sv = img.texture.create_view(&Default::default()); + let ov = tex.create_view(&Default::default()); + let bg = g.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &g.display_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&sv) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&ov) }, + ], + }); + let enc = self.encoder(); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("display"), timestamp_writes: None }); + pass.set_pipeline(&g.display); + pass.set_bind_group(0, &bg, &[]); + pass.dispatch_workgroups(img.width.div_ceil(16), img.height.div_ceil(16), 1); + } + tex + } +} diff --git a/crates/gpu/src/effects.rs b/crates/gpu/src/effects.rs new file mode 100644 index 000000000..4addbaa85 --- /dev/null +++ b/crates/gpu/src/effects.rs @@ -0,0 +1,347 @@ +//! GPU effects: the same steps as the CPU effects in `effectcraft-effects` (padding, blur +//! radii, parameter conversions), with the pixel loops as compute kernels. + +use effectcraft_effects::{Buf, EffectCtx}; +use effectcraft_geom::{Mat3, vec2}; +use effectcraft_raster::Sampling; +use effectcraft_render::FxStep; + +use crate::context::{Enc, GpuImage, Params}; +use crate::ops; + +/// Effects with a GPU implementation (must equal `effectcraft_effects::GPU_EFFECTS`), plus the +/// expression controls (pass-throughs). +pub fn supports(id: &str) -> bool { + effectcraft_effects::GPU_EFFECTS.contains(&id) || id.starts_with("ec.control.") +} + +/// A layer buffer on the GPU (see [`Buf`]). +struct GBuf { + img: GpuImage, + offset: [f64; 2], + scale: f64, +} + +impl GBuf { + fn to_px(&self, p: [f64; 2]) -> (f64, f64) { + (p[0] * self.scale + self.offset[0], p[1] * self.scale + self.offset[1]) + } + + /// Buf::pad: grow by `pad` transparent pixels on every side. + fn pad(&mut self, e: &mut Enc, pad: u32) -> Option<()> { + if pad == 0 { + return Some(()); + } + let (w, h) = (self.img.width + 2 * pad, self.img.height + 2 * pad); + if !e.g.fits(w, h) { + return None; + } + let out = e.image(w, h); + e.copy_into(&self.img, &out, pad, pad); + self.img = out; + self.offset[0] += pad as f64; + self.offset[1] += pad as f64; + Some(()) + } +} + +/// Run a chain of GPU effects (one upload, one readback), quantising after each when `levels` +/// is set. `None` when anything cannot run here. +pub(crate) fn run_chain(e: &mut Enc, chain: &[FxStep], buf: &Buf, levels: Option) -> Option { + if !e.g.can_readback() || buf.img.is_empty() { + return None; + } + let mut b = GBuf { img: e.g.upload_image(&buf.img)?, offset: buf.offset, scale: buf.scale }; + for step in chain { + b = apply(e, step.spec.id, &step.ctx, b)?; + if let Some(l) = levels { + b.img = ops::quantize(e, &b.img, l); + } + } + let img = e.download(&b.img)?; + Some(Buf { img, offset: b.offset, scale: b.scale }) +} + +fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + match id { + "ec.blur.gaussian" => gaussian(e, ctx, b), + "ec.blur.fastbox" => box_blur(e, ctx, b), + "ec.blur.directional" => directional(e, ctx, b), + "ec.stylize.glow" => glow(e, ctx, b), + "ec.perspective.dropshadow" => drop_shadow(e, ctx, b), + "ec.distort.transform" => transform(e, ctx, b), + "ec.color.curves" => curves(e, ctx, b), + _ if id.starts_with("ec.control.") => Some(b), + _ => pointwise(e, id, ctx, b), + } +} + +// ---------------------------------------------------------------- blurs + +/// raster::blur::box_radii. +fn box_radii(sigma: f64, n: usize) -> Vec { + if sigma <= 0.0 { + return vec![0; n]; + } + let w_ideal = (12.0 * sigma * sigma / n as f64 + 1.0).sqrt(); + let mut wl = w_ideal.floor() as i64; + if wl % 2 == 0 { + wl -= 1; + } + let wu = wl + 2; + let m_ideal = (12.0 * sigma * sigma - (n as i64 * wl * wl) as f64 - 4.0 * n as f64 * wl as f64 - 3.0 * n as f64) / (-4.0 * wl as f64 - 4.0); + let m = m_ideal.round() as i64; + (0..n as i64).map(|i| (((if i < m { wl } else { wu }) - 1) / 2).max(0) as usize).collect() +} + +/// Horizontal passes with radii `rx`, then vertical passes with radii `ry` (raster::blur). +fn box_passes(e: &mut Enc, img: &GpuImage, rx: &[usize], ry: &[usize], repeat: bool) -> GpuImage { + let mut cur = img.clone(); + let passes = rx.iter().map(|&r| (false, r)).chain(ry.iter().map(|&r| (true, r))).filter(|p| p.1 > 0); + for (vertical, r) in passes { + let block = if vertical { (4 * r).clamp(32, 256) } else { 256 } as u32; + let (lines, n) = if vertical { (cur.width, cur.height) } else { (cur.height, cur.width) }; + let mut p = Params::default(); + p.u[0] = [r as u32, repeat as u32, block, 0]; + let out = e.image(cur.width, cur.height); + e.dispatch(if vertical { "box_v" } else { "box_h" }, &p, &cur, None, &out, None, (lines.div_ceil(64), n.div_ceil(block))); + cur = out; + } + cur +} + +/// raster::gaussian_blur (3 box passes per axis). +fn gaussian_blur(e: &mut Enc, img: &GpuImage, sx: f64, sy: f64, repeat: bool) -> GpuImage { + let rx = if sx > 0.05 { box_radii(sx, 3) } else { vec![] }; + let ry = if sy > 0.05 { box_radii(sy, 3) } else { vec![] }; + box_passes(e, img, &rx, &ry, repeat) +} + +fn dims_xy(dim: u32) -> (f64, f64) { + match dim { + 1 => (1.0, 0.0), + 2 => (0.0, 1.0), + _ => (1.0, 1.0), + } +} + +fn gaussian(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let s = ctx.params.f("blurriness").max(0.0) * 0.5 * b.scale; + if s <= 0.0 { + return Some(b); + } + let (kx, ky) = dims_xy(ctx.params.e("dimensions")); + let repeat = ctx.params.b("repeatEdge"); + if !repeat && !ctx.adjustment { + b.pad(e, (s * 3.0).ceil() as u32)?; + } + b.img = gaussian_blur(e, &b.img, s * kx, s * ky, repeat || ctx.adjustment); + Some(b) +} + +fn box_blur(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let r = (ctx.params.f("radius") * b.scale).round().max(0.0) as usize; + if r == 0 { + return Some(b); + } + let it = ctx.params.f("iterations").round().clamp(1.0, 50.0) as usize; + let (kx, ky) = dims_xy(ctx.params.e("dimensions")); + let repeat = ctx.params.b("repeatEdge"); + if !repeat && !ctx.adjustment { + b.pad(e, (r * it) as u32 + 1)?; + } + b.img = box_passes(e, &b.img, &vec![r * kx as usize; it], &vec![r * ky as usize; it], repeat || ctx.adjustment); + Some(b) +} + +fn directional(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let len = ctx.params.f("length") * b.scale; + if len < 0.5 { + return Some(b); + } + if !ctx.adjustment { + b.pad(e, len.ceil() as u32 + 1)?; + } + let a = ctx.params.f("direction").to_radians(); + let n = (len.ceil() as usize * 2 + 1).clamp(3, 257); + let mut p = Params::default(); + p.f[0] = [a.sin() as f32, -a.cos() as f32, len as f32, n as f32]; + let out = e.image(b.img.width, b.img.height); + e.pixels("directional", &p, &b.img, None, &out, None); + b.img = out; + Some(b) +} + +fn glow(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let thr = ctx.params.f("threshold") as f32 / 100.0; + let radius = ctx.params.f("radius") * b.scale; + let intensity = ctx.params.f("intensity") as f32; + let use_colors = ctx.params.e("colors") == 1; + let ca = ctx.params.color("colorA"); + let cb = ctx.params.color("colorB"); + if !ctx.adjustment { + b.pad(e, (radius * 1.5).ceil() as u32 + 2)?; + } + let mut p = Params::default(); + p.u[0][0] = use_colors as u32; + p.f[0][0] = thr; + p.f[1] = ca; + p.f[2] = cb; + let bright = e.image(b.img.width, b.img.height); + e.pixels("glow_bright", &p, &b.img, None, &bright, None); + let s = (radius / 2.0).max(0.5); + let blurred = gaussian_blur(e, &bright, s, s, false); + let mut p = Params::default(); + p.u[0][0] = (ctx.params.e("operation") == 1) as u32; + p.f[0][0] = intensity; + let out = e.image(b.img.width, b.img.height); + e.pixels("glow_combine", &p, &b.img, Some(&blurred), &out, None); + b.img = out; + Some(b) +} + +fn drop_shadow(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let color = ctx.params.color("color"); + let opacity = ctx.params.f("opacity") as f32 / 100.0; + let dir = ctx.params.f("direction").to_radians(); + let dist = ctx.params.f("distance") * b.scale; + let soft = ctx.params.f("softness") * b.scale; + let only = ctx.params.b("shadowOnly"); + let pad = (dist + soft * 1.5).ceil() as u32 + 2; + if !ctx.adjustment { + b.pad(e, pad)?; + } + let (dx, dy) = (dir.sin() * dist, -dir.cos() * dist); + let mut p = Params::default(); + p.f[0] = color; + p.f[1] = [opacity, dx as f32, dy as f32, 0.0]; + let mut sh = e.image(b.img.width, b.img.height); + e.pixels("shadow_make", &p, &b.img, None, &sh, None); + if soft > 0.0 { + sh = gaussian_blur(e, &sh, soft / 2.0, soft / 2.0, false); + } + if !only { + let out = e.image(b.img.width, b.img.height); + e.pixels("shadow_combine", &Params::default(), &sh, Some(&b.img), &out, None); + sh = out; + } + b.img = sh; + Some(b) +} + +fn transform(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let anchor = b.to_px(ctx.params.v2("anchor")); + let pos = b.to_px(ctx.params.v2("position")); + let sh = ctx.params.f("scaleHeight"); + let sw = if ctx.params.b("uniform") { sh } else { ctx.params.f("scaleWidth") }; + let opacity = ctx.params.f("opacity") as f32 / 100.0; + let m = Mat3::translate(vec2(pos.0, pos.1)) + * Mat3::rotate_deg(ctx.params.f("rotation")) + * Mat3::skew_deg(ctx.params.f("skew"), ctx.params.f("skewAxis")) + * Mat3::scale(vec2(sw / 100.0, sh / 100.0)) + * Mat3::translate(vec2(-anchor.0, -anchor.1)); + let empty = e.image(b.img.width, b.img.height); + b.img = ops::warp(e, &empty, &b.img, &m, Sampling::Bilinear, effectcraft_color::BlendMode::Normal, opacity, 0, None); + Some(b) +} + +// ---------------------------------------------------------------- per-pixel + +fn curves(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let parsed = ["rgb", "red", "green", "blue", "alpha"].map(|id| effectcraft_effects::Curve::parse(ctx.params.s(id))); + if parsed.iter().all(Option::is_none) { + return Some(b); + } + let mut data = vec![]; + let mut offs = [-1.0f32; 5]; + for (i, c) in parsed.iter().enumerate() { + let Some(c) = c else { continue }; + let (xs, ys, ms, lut) = c.tables(); + offs[i] = data.len() as f32; + data.push(xs.len() as f32); + data.extend_from_slice(xs); + data.extend_from_slice(ys); + data.extend_from_slice(ms); + data.extend_from_slice(lut); + } + let buf = e.data(&data); + let mut p = Params::default(); + p.u[0][0] = 8; + p.f[0] = [offs[0], offs[1], offs[2], offs[3]]; + p.f[1][0] = offs[4]; + let out = e.image(b.img.width, b.img.height); + e.pixels("pointwise", &p, &b.img, None, &out, Some(&buf)); + Some(GBuf { img: out, ..b }) +} + +fn rgb(c: [f32; 4]) -> [f32; 4] { + [c[0], c[1], c[2], 0.0] +} + +fn pointwise(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + let f = |k: &str| ctx.params.f(k); + let mut p = Params::default(); + match id { + "ec.color.tint" => { + p.u[0][0] = 1; + p.f[0] = rgb(ctx.params.color("black")); + p.f[1] = rgb(ctx.params.color("white")); + p.f[2][0] = f("amount") as f32 / 100.0; + } + "ec.color.brightnesscontrast" => { + let br = f("brightness") as f32 / 100.0; + let ct = f("contrast") as f32 / 100.0; + let k = if ct >= 0.0 { 1.0 / (1.0 - ct * 0.99) } else { 1.0 + ct }; + p.u[0][0] = 2; + p.f[0] = [br, k, 0.0, 0.0]; + } + "ec.color.huesaturation" => { + p.u[0] = [3, ctx.params.b("colorize") as u32, 0, 0]; + p.f[0] = [f("hue") as f32 / 360.0, f("saturation") as f32 / 100.0, f("lightness") as f32 / 100.0, 0.0]; + p.f[1] = [f("colorizeHue") as f32 / 360.0, f("colorizeSaturation") as f32 / 100.0, f("colorizeLightness") as f32 / 100.0, 0.0]; + } + "ec.color.levels" => { + p.u[0] = [4, !ctx.params.b("noClip") as u32, 0, 0]; + p.f[0] = [f("inBlack") as f32, f("inWhite") as f32, f("gamma").max(0.01) as f32, f("outBlack") as f32]; + p.f[1][0] = f("outWhite") as f32; + } + "ec.color.exposure" => { + p.u[0][0] = 5; + p.f[0] = [2f32.powf(f("exposure") as f32), f("offset") as f32, f("gamma").max(0.01) as f32, 0.0]; + } + "ec.channel.invert" => { + p.u[0] = [6, ctx.params.e("channel"), 0, 0]; + p.f[0][0] = 1.0 - f("blend") as f32 / 100.0; + } + "ec.generate.fill" => { + p.u[0] = [7, ctx.params.b("invert") as u32, 0, 0]; + p.f[0] = ctx.params.color("color"); + p.f[1][0] = f("opacity") as f32 / 100.0; + } + "ec.generate.gradientramp" => { + let s = b.to_px(ctx.params.v2("start")); + let en = b.to_px(ctx.params.v2("end")); + let (dx, dy) = (en.0 - s.0, en.1 - s.1); + let len2 = (dx * dx + dy * dy).max(1e-9); + p.u[0] = [9, (ctx.params.e("shape") == 1) as u32, ctx.seed, 0]; + p.f[0] = [s.0 as f32, s.1 as f32, en.0 as f32, en.1 as f32]; + p.f[1] = rgb(ctx.params.color("startColor")); + p.f[2] = rgb(ctx.params.color("endColor")); + p.f[3] = [len2 as f32, f("scatter") as f32 / 512.0, f("blend") as f32 / 100.0, 0.0]; + } + "ec.noise.fractal" => { + let octaves = f("complexity").clamp(1.0, 20.0); + let (sr, cr) = (f("rotation") as f32).to_radians().sin_cos(); + let (ox, oy) = b.to_px(ctx.params.v2("offset")); + p.u[0] = [10, ctx.params.e("fractalType"), octaves.ceil() as u32, f("seed") as u32 ^ 0x51ed]; + p.u[1][0] = ctx.params.b("invert") as u32; + p.f[0] = [f("contrast") as f32 / 100.0, f("brightness") as f32 / 100.0, (f("scale") * b.scale).max(1.0) as f32, (octaves - octaves.floor()) as f32]; + p.f[1] = [ox as f32, oy as f32, sr, cr]; + p.f[2] = [f("evolution") as f32 / 360.0, f("blend") as f32 / 100.0, f("opacity") as f32 / 100.0, 0.0]; + } + _ => return None, + } + let out = e.image(b.img.width, b.img.height); + e.pixels("pointwise", &p, &b.img, None, &out, None); + Some(GBuf { img: out, ..b }) +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs new file mode 100644 index 000000000..e7b54ab27 --- /dev/null +++ b/crates/gpu/src/lib.rs @@ -0,0 +1,127 @@ +//! The EffectCraft GPU compositor (Mercury GPU Acceleration's counterpart), on wgpu compute +//! shaders: Metal, Vulkan, Direct3D 12 and WebGPU. +//! +//! The CPU [`Renderer`] stays the reference and keeps rendering layer *content* (sources, masks, +//! CPU effects, layer styles) into its layer cache. [`Gpu`] composites: it uploads the cached +//! layer buffers (once per buffer), applies the layer transforms with the CPU's sampling +//! (nearest / bilinear / bicubic, minification pre-filter), motion-blur sub-samples, track +//! mattes, Preserve Transparency, layer styles' passes, all 38 blend modes and the 8/16 bpc +//! clamping and quantisation, and converts colour spaces. 3D runs, adjustment layers and +//! wireframes run on the CPU between GPU steps (read back, draw, upload). +//! +//! GPU effects ([`effectcraft_effects::GPU_EFFECTS`]) run as compute kernels with the CPU +//! effect's exact steps (padding, box-blur radii, parameter conversions); chains of them are +//! uploaded and read back once. +//! +//! Plug a [`Gpu`] into [`Renderer::accel`] (it implements [`Accelerator`]); renders then use it +//! when [`RenderOpts::backend`](effectcraft_render::RenderOpts) asks for it. The viewer can +//! skip readback entirely with [`Gpu::render_display`], which leaves an RGBA8 texture for +//! egui-wgpu to draw. + +mod context; +mod effects; +mod ops; +mod walk; + +use std::sync::Arc; + +pub use context::{GpuContext, GpuImage}; +use effectcraft_effects::Buf; +use effectcraft_project::ItemId; +use effectcraft_raster::Image; +use effectcraft_render::{Accelerator, FxStep, Renderer}; +use effectcraft_time::Tick; +pub use wgpu; + +use crate::context::Enc; + +/// The GPU compositor (cheap to clone; one device shared by all clones). +#[derive(Clone)] +pub struct Gpu { + ctx: Arc, +} + +/// A viewer frame left on the GPU: premultiplied RGBA8 (`wgpu::TextureFormat::Rgba8Unorm`), +/// ready for `egui_wgpu::Renderer::register_native_texture`. +#[derive(Clone, Debug)] +pub struct DisplayFrame { + pub texture: wgpu::Texture, + pub width: u32, + pub height: u32, +} + +impl Gpu { + /// On a device of its own (CLI, tests, benchmarks); `None` without a usable adapter. + #[cfg(not(target_arch = "wasm32"))] + pub fn headless() -> Option { + GpuContext::headless().map(Gpu::from_context) + } + + /// On an existing device (the desktop app shares egui-wgpu's). + pub fn new(adapter: &wgpu::Adapter, device: wgpu::Device, queue: wgpu::Queue) -> Result { + GpuContext::new(adapter, device, queue).map(Gpu::from_context) + } + + pub fn from_context(ctx: GpuContext) -> Gpu { + Gpu { ctx: Arc::new(ctx) } + } + + pub fn context(&self) -> &GpuContext { + &self.ctx + } + + pub fn device(&self) -> &wgpu::Device { + &self.ctx.device + } + + /// Render a top-level comp frame and leave it on the GPU as a display texture (no + /// readback unless a CPU fallback step needs one). `None` = render on the CPU. + pub fn render_display(&self, r: &Renderer, comp: ItemId, t: Tick) -> Option { + let mut e = Enc::new(&self.ctx); + let img = walk::render(&mut e, r, comp, t)?; + let texture = e.display(&img); + e.submit(); + Some(DisplayFrame { texture, width: img.width, height: img.height }) + } + + /// Read a display frame back as premultiplied RGBA8 (Info panel sampling, eyedroppers). + pub fn read_display(&self, f: &DisplayFrame) -> Option> { + Enc::new(&self.ctx).read_texture(&f.texture, f.width, f.height, 4) + } + + /// Render a frame on the GPU and read it back (`None` = not handled). + pub fn render(&self, r: &Renderer, comp: ItemId, t: Tick) -> Option { + if !self.ctx.can_readback() { + return None; + } + let mut e = Enc::new(&self.ctx); + let img = walk::render(&mut e, r, comp, t)?; + e.download(&img) + } + + /// Wait for all submitted GPU work (benchmarks). + pub fn wait(&self) { + let _ = self.ctx.device.poll(wgpu::PollType::wait_indefinitely()); + } +} + +impl Accelerator for Gpu { + fn name(&self) -> String { + self.ctx.name.clone() + } + + fn comp_frame(&self, r: &Renderer, comp: ItemId, t: Tick) -> Option { + self.render(r, comp, t) + } + + fn supports_effect(&self, id: &str) -> bool { + self.ctx.can_readback() && effects::supports(id) + } + + fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option { + effects::run_chain(&mut Enc::new(&self.ctx), chain, buf, levels) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/gpu/src/ops.rs b/crates/gpu/src/ops.rs new file mode 100644 index 000000000..775e86c13 --- /dev/null +++ b/crates/gpu/src/ops.rs @@ -0,0 +1,167 @@ +//! Compositing primitives: each mirrors a CPU operation in `effectcraft-raster` / +//! `effectcraft-render` and returns a new image (GPU images are immutable). + +use effectcraft_color::{BlendMode, Conversion}; +use effectcraft_geom::{Mat3, Rect, vec2}; +use effectcraft_project::MatteKind; +use effectcraft_raster::Sampling; + +use crate::context::{Enc, GpuImage, Params}; + +/// Index of a blend mode in `BlendMode::ALL` (the kernels' mode numbers). +pub fn mode_id(m: BlendMode) -> u32 { + BlendMode::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32 +} + +fn sampling_id(s: Sampling) -> u32 { + match s { + Sampling::Nearest => 0, + Sampling::Bilinear => 1, + Sampling::Bicubic => 2, + } +} + +fn rows(m: &Mat3) -> [[f32; 4]; 3] { + m.0.map(|r| [r[0] as f32, r[1] as f32, r[2] as f32, 0.0]) +} + +/// Box-filter halving (minification pre-filter). +pub fn half(e: &mut Enc, src: &GpuImage) -> GpuImage { + let out = e.image(src.width.div_ceil(2).max(1), src.height.div_ceil(2).max(1)); + e.pixels("half", &Params::default(), src, None, &out, None); + out +} + +/// `raster::composite_warp` (and, with `accumulate = Some(weight)`, `accumulate_warp`): draw +/// `src` transformed by `m` (src pixels → dst pixels) onto `dst`. +pub fn warp( + e: &mut Enc, + dst: &GpuImage, + src: &GpuImage, + m: &Mat3, + sampling: Sampling, + mode: BlendMode, + opacity: f32, + seed: u32, + accumulate: Option, +) -> GpuImage { + if src.width == 0 || src.height == 0 || opacity <= 0.0 { + return dst.clone(); + } + let Some(inv0) = m.inverse() else { return dst.clone() }; + // Minification: pick a pre-filtered level so sampling doesn't alias. + let scale = m.mean_scale(); + let mut img = src.clone(); + let mut inv = inv0; + if scale < 0.5 && m.is_affine() { + let mut s = scale * 2.0; + let mut f = 0.5; + img = half(e, src); + while s < 0.5 && img.width > 1 && img.height > 1 { + img = half(e, &img); + f *= 0.5; + s *= 2.0; + } + inv = Mat3::scale(vec2(f, f)) * inv; + } + let bounds = m.map_rect(&Rect::from_size(src.width as f64, src.height as f64)).inflate(1.0).round_out(); + let area = bounds.intersect(&Rect::from_size(dst.width as f64, dst.height as f64)); + if area.is_empty() { + return dst.clone(); + } + let (x0, x1) = (area.x0.max(0.0) as u32, area.x1.min(dst.width as f64) as u32); + let (y0, y1) = (area.y0.max(0.0) as u32, area.y1.min(dst.height as f64) as u32); + let mut p = Params::default(); + let clear = matches!(mode, BlendMode::StencilAlpha | BlendMode::StencilLuma); + let flags = u32::from(accumulate.is_some()) | if clear && accumulate.is_none() { 2 } else { 0 }; + p.u[0] = [mode_id(mode), sampling_id(sampling), seed, flags]; + let r = rows(&inv); + p.f[0] = r[0]; + p.f[1] = r[1]; + p.f[2] = r[2]; + p.f[3] = [opacity, accumulate.unwrap_or(1.0), 0.0, 0.0]; + p.f[4] = [x0 as f32, y0 as f32, x1 as f32, y1 as f32]; + let out = e.image(dst.width, dst.height); + e.pixels("warp_blend", &p, &img, Some(dst), &out, None); + out +} + +/// `Image::blend_from`: blend `src` (same size) onto `dst`. +pub fn blend_full(e: &mut Enc, dst: &GpuImage, src: &GpuImage, mode: BlendMode, opacity: f32, seed: u32) -> GpuImage { + let mut p = Params::default(); + p.u[0] = [mode_id(mode), 0, seed, 0]; + p.f[0] = [opacity, 0.0, 0.0, 0.0]; + let out = e.image(dst.width, dst.height); + e.pixels("blend_full", &p, src, Some(dst), &out, None); + out +} + +/// Multiply an isolated layer by its track matte. +pub fn matte(e: &mut Enc, iso: &GpuImage, matte: &GpuImage, kind: MatteKind) -> GpuImage { + let mut p = Params::default(); + p.u[0][0] = match kind { + MatteKind::Alpha => 0, + MatteKind::AlphaInverted => 1, + MatteKind::Luma => 2, + MatteKind::LumaInverted => 3, + }; + let out = e.image(iso.width, iso.height); + e.pixels("matte", &p, iso, Some(matte), &out, None); + out +} + +/// Preserve Transparency: multiply by the alpha below. +pub fn preserve(e: &mut Enc, iso: &GpuImage, canvas: &GpuImage) -> GpuImage { + let out = e.image(iso.width, iso.height); + e.pixels("preserve", &Params::default(), iso, Some(canvas), &out, None); + out +} + +/// Knockout: clear what lies below the layer's content. +pub fn knockout(e: &mut Enc, canvas: &GpuImage, shape: &GpuImage) -> GpuImage { + let out = e.image(canvas.width, canvas.height); + e.pixels("knockout", &Params::default(), canvas, Some(shape), &out, None); + out +} + +/// Styled-layer finish: move `canvas` toward `tmp` by `opacity` on the enabled channels. +pub fn channel_mix(e: &mut Enc, canvas: &GpuImage, tmp: &GpuImage, channels: [bool; 3], opacity: f32) -> GpuImage { + let mut p = Params::default(); + p.u[0] = [channels[0] as u32, channels[1] as u32, channels[2] as u32, 0]; + p.f[0][0] = opacity; + let out = e.image(canvas.width, canvas.height); + e.pixels("channel_mix", &p, canvas, Some(tmp), &out, None); + out +} + +/// Clamp and quantise to `levels` steps (8/16 bpc). +pub fn quantize(e: &mut Enc, img: &GpuImage, levels: f32) -> GpuImage { + let mut p = Params::default(); + p.f[0][0] = levels; + let out = e.image(img.width, img.height); + e.pixels("quantize", &p, img, None, &out, None); + out +} + +fn curve_id(s: Option) -> u32 { + use effectcraft_color::ColorSpace::*; + match s { + None => 0, + Some(Srgb | DisplayP3) => 1, + Some(Rec709 | Rec2020) => 2, + } +} + +/// Colour conversion on straight colour (`render::color::convert`). +pub fn convert(e: &mut Enc, img: &GpuImage, c: &Conversion) -> GpuImage { + let mut p = Params::default(); + p.u[0] = [curve_id(c.decode), c.matrix.is_some() as u32, curve_id(c.encode), 0]; + if let Some(m) = c.matrix { + for i in 0..3 { + p.f[i] = [m[i][0], m[i][1], m[i][2], 0.0]; + } + } + let out = e.image(img.width, img.height); + e.pixels("convert", &p, img, None, &out, None); + out +} diff --git a/crates/gpu/src/shaders/common.wgsl b/crates/gpu/src/shaders/common.wgsl new file mode 100644 index 000000000..dee86b164 --- /dev/null +++ b/crates/gpu/src/shaders/common.wgsl @@ -0,0 +1,452 @@ +// EffectCraft GPU compositor: shared bindings and helpers. +// +// Every kernel uses the same bind group: a uniform block of parameters, two input textures +// (premultiplied RGBA f32, read with textureLoad), one output storage texture and a read-only +// storage buffer of extra data (curve tables). Each helper mirrors a CPU function in +// effectcraft-raster / effectcraft-color operation for operation, so results match the CPU +// reference to float rounding. + +struct Params { + u: array, 4>, + f: array, 12>, +} + +@group(0) @binding(0) var P: Params; +@group(0) @binding(1) var src: texture_2d; +@group(0) @binding(2) var aux: texture_2d; +@group(0) @binding(4) var data: array; + +// ---------------------------------------------------------------- sampling (raster::Image) + +fn tex_get(t: texture_2d, x: i32, y: i32) -> vec4 { + let d = vec2(textureDimensions(t)); + if (x < 0 || y < 0 || x >= d.x || y >= d.y) { + return vec4(0.0); + } + return textureLoad(t, vec2(x, y), 0); +} + +fn tex_get_clamped(t: texture_2d, x: i32, y: i32) -> vec4 { + let d = vec2(textureDimensions(t)); + return textureLoad(t, vec2(clamp(x, 0, d.x - 1), clamp(y, 0, d.y - 1)), 0); +} + +// Image::sample_bilinear (pixel centres at +0.5; transparent outside). +fn sample_bilinear(t: texture_2d, x: f32, y: f32) -> vec4 { + let fx = x - 0.5; + let fy = y - 0.5; + let x0 = floor(fx); + let y0 = floor(fy); + let tx = fx - x0; + let ty = fy - y0; + let xi = i32(x0); + let yi = i32(y0); + let a = tex_get(t, xi, yi); + let b = tex_get(t, xi + 1, yi); + let c = tex_get(t, xi, yi + 1); + let d = tex_get(t, xi + 1, yi + 1); + let top = a + (b - a) * tx; + let bot = c + (d - c) * tx; + return top + (bot - top) * ty; +} + +// Image::sample_bilinear_clamped (edge pixels repeat). +fn sample_bilinear_clamped(t: texture_2d, x: f32, y: f32) -> vec4 { + let fx = x - 0.5; + let fy = y - 0.5; + let x0 = floor(fx); + let y0 = floor(fy); + let tx = fx - x0; + let ty = fy - y0; + let xi = i32(x0); + let yi = i32(y0); + let a = tex_get_clamped(t, xi, yi); + let b = tex_get_clamped(t, xi + 1, yi); + let c = tex_get_clamped(t, xi, yi + 1); + let d = tex_get_clamped(t, xi + 1, yi + 1); + let top = a + (b - a) * tx; + let bot = c + (d - c) * tx; + return top + (bot - top) * ty; +} + +fn cubic_w(t: f32) -> vec4 { + let t2 = t * t; + let t3 = t2 * t; + return vec4(-0.5 * t3 + t2 - 0.5 * t, 1.5 * t3 - 2.5 * t2 + 1.0, -1.5 * t3 + 2.0 * t2 + 0.5 * t, 0.5 * t3 - 0.5 * t2); +} + +// Image::sample_bicubic (Catmull-Rom; negatives clamped after ringing). +fn sample_bicubic(t: texture_2d, x: f32, y: f32) -> vec4 { + let fx = x - 0.5; + let fy = y - 0.5; + let x0 = floor(fx); + let y0 = floor(fy); + let wx = cubic_w(fx - x0); + let wy = cubic_w(fy - y0); + let xi = i32(x0); + let yi = i32(y0); + var o = vec4(0.0); + for (var j = 0; j < 4; j++) { + for (var i = 0; i < 4; i++) { + let p = tex_get(t, xi - 1 + i, yi - 1 + j); + o += p * (wx[i] * wy[j]); + } + } + return max(o, vec4(0.0)); +} + +// 0 nearest, 1 bilinear, 2 bicubic (raster::Sampling). +fn sample(t: texture_2d, sampling: u32, x: f32, y: f32) -> vec4 { + if (sampling == 0u) { + return tex_get(t, i32(floor(x)), i32(floor(y))); + } + if (sampling == 2u) { + return sample_bicubic(t, x, y); + } + return sample_bilinear(t, x, y); +} + +// raster::hash_noise. +fn hash_noise(x: u32, y: u32, seed: u32) -> f32 { + var h = (x * 0x8da6b343u) ^ (y * 0xd8163841u) ^ (seed * 0xcb1ab31fu); + h = h ^ (h >> 13u); + h = h * 0x5bd1e995u; + h = h ^ (h >> 15u); + return f32(h & 0x00ffffffu) / 16777216.0; +} + +// powf for x >= 0 (0^y = 0 for y > 0; WGSL leaves pow(0, y) to the implementation). +fn powz(x: f32, y: f32) -> f32 { + if (x <= 0.0) { + return select(0.0, 1.0, y == 0.0); + } + return pow(x, y); +} + +// Rust's f32::round (half away from zero; WGSL's round() is half to even). +fn round_away(x: f32) -> f32 { + let t = trunc(x); + if (abs(x - t) >= 0.5) { + return t + sign(x); + } + return t; +} + +// f32::rem_euclid(1.0). +fn fract_euclid(x: f32) -> f32 { + return x - floor(x); +} + +fn luminance(c: vec3) -> f32 { + return 0.2126 * c.x + 0.7152 * c.y + 0.0722 * c.z; +} + +// ---------------------------------------------------------------- blend modes (color::blend) +// Mode numbers are indices into BlendMode::ALL. + +fn burn(cb: f32, cs: f32) -> f32 { + if (cb >= 1.0) { + return 1.0; + } + if (cs <= 0.0) { + return 0.0; + } + return 1.0 - min((1.0 - cb) / cs, 1.0); +} + +fn dodge(cb: f32, cs: f32) -> f32 { + if (cb <= 0.0) { + return 0.0; + } + if (cs >= 1.0) { + return 1.0; + } + return min(cb / (1.0 - cs), 1.0); +} + +fn hard_light(cb: f32, cs: f32) -> f32 { + if (cs <= 0.5) { + return cb * 2.0 * cs; + } + let s = 2.0 * cs - 1.0; + return cb + s - cb * s; +} + +fn soft_light(cb: f32, cs: f32) -> f32 { + if (cs <= 0.5) { + return cb - (1.0 - 2.0 * cs) * cb * (1.0 - cb); + } + var d: f32; + if (cb <= 0.25) { + d = ((16.0 * cb - 12.0) * cb + 4.0) * cb; + } else { + d = sqrt(max(cb, 0.0)); + } + return cb + (2.0 * cs - 1.0) * (d - cb); +} + +fn vivid(cb: f32, cs: f32) -> f32 { + if (cs <= 0.5) { + return burn(cb, 2.0 * cs); + } + return dodge(cb, 2.0 * (cs - 0.5)); +} + +fn separable(mode: u32, cb: f32, cs: f32) -> f32 { + switch mode { + case 3u: { return min(cb, cs); } + case 4u: { return cb * cs; } + case 5u: { return burn(cb, cs); } + case 6u: { + if (cs <= 0.0) { + return 0.0; + } + return max(1.0 - (1.0 - cb) / cs, 0.0); + } + case 7u: { return max(cb + cs - 1.0, 0.0); } + case 9u, 14u: { return cb + cs; } + case 10u: { return max(cb, cs); } + case 11u: { return cb + cs - cb * cs; } + case 12u: { return dodge(cb, cs); } + case 13u: { + if (cs >= 1.0) { + return 1.0; + } + return min(cb / (1.0 - cs), 1.0); + } + case 16u: { return hard_light(cs, cb); } + case 17u: { return soft_light(cb, cs); } + case 18u: { return hard_light(cb, cs); } + case 19u: { return clamp(cb + 2.0 * cs - 1.0, 0.0, 1.0); } + case 20u: { return vivid(cb, cs); } + case 21u: { + if (cs <= 0.5) { + return min(cb, 2.0 * cs); + } + return max(cb, 2.0 * cs - 1.0); + } + case 22u: { + if (cb + cs >= 1.0) { + return 1.0; + } + return 0.0; + } + case 23u, 24u: { return abs(cb - cs); } + case 25u: { return cb + cs - 2.0 * cb * cs; } + case 26u: { return max(cb - cs, 0.0); } + case 27u: { + if (cs <= 0.0) { + if (cb > 0.0) { + return 1.0; + } + return 0.0; + } + return cb / cs; + } + default: { return cs; } + } +} + +fn blum(c: vec3) -> f32 { + return 0.3 * c.x + 0.59 * c.y + 0.11 * c.z; +} + +fn clip_color(c: vec3) -> vec3 { + let l = blum(c); + let n = min(min(c.x, c.y), c.z); + let x = max(max(c.x, c.y), c.z); + var o = c; + if (n < 0.0) { + o = l + (o - l) * l / max(l - n, 1e-9); + } + if (x > 1.0) { + o = l + (o - l) * (1.0 - l) / max(x - l, 1e-9); + } + return o; +} + +fn set_lum(c: vec3, l: f32) -> vec3 { + let d = l - blum(c); + return clip_color(c + d); +} + +fn bsat(c: vec3) -> f32 { + return max(max(c.x, c.y), c.z) - min(min(c.x, c.y), c.z); +} + +fn set_sat(c: vec3, s: f32) -> vec3 { + let mx = max(max(c.x, c.y), c.z); + let mn = min(min(c.x, c.y), c.z); + if (mx - mn <= 1e-9) { + return vec3(0.0); + } + return (c - mn) * s / (mx - mn); +} + +fn non_separable(mode: u32, cb: vec3, cs: vec3) -> vec3 { + switch mode { + case 28u: { return set_lum(set_sat(cs, bsat(cb)), blum(cb)); } + case 29u: { return set_lum(set_sat(cb, bsat(cs)), blum(cb)); } + case 30u: { return set_lum(cs, blum(cb)); } + case 31u: { return set_lum(cb, blum(cs)); } + case 8u: { + if (luminance(cs) < luminance(cb)) { + return cs; + } + return cb; + } + case 15u: { + if (luminance(cs) > luminance(cb)) { + return cs; + } + return cb; + } + default: { + return vec3(separable(mode, cb.x, cs.x), separable(mode, cb.y, cs.y), separable(mode, cb.z, cs.z)); + } + } +} + +fn supports_hdr(mode: u32) -> bool { + switch mode { + case 0u, 1u, 2u, 3u, 4u, 7u, 8u, 9u, 10u, 11u, 14u, 15u, 23u, 24u, 26u, 27u, 32u, 33u, 34u, 35u, 36u, 37u: { return true; } + default: { return false; } + } +} + +fn is_stencil(mode: u32) -> bool { + return mode >= 32u && mode <= 35u; +} + +// color::blend_pixel. +fn blend_pixel(mode: u32, dst: vec4, src: vec4, noise: f32) -> vec4 { + let sa = src.w; + let da = dst.w; + switch mode { + case 0u: { + let k = 1.0 - sa; + return vec4(src.xyz + dst.xyz * k, sa + da * k); + } + case 1u, 2u: { + if (sa <= 0.0 || noise >= sa) { + return dst; + } + let inv = 1.0 / sa; + return vec4(src.xyz * inv, 1.0); + } + case 32u: { return dst * sa; } + case 34u: { return dst * (1.0 - sa); } + case 33u: { return dst * luminance(src.xyz); } + case 35u: { return dst * (1.0 - luminance(src.xyz)); } + case 36u: { return vec4(src.xyz + dst.xyz * (1.0 - sa), min(sa + da, 1.0)); } + case 37u: { return vec4(src.xyz + dst.xyz * (1.0 - sa), sa + da - sa * da); } + default: {} + } + if (sa <= 0.0) { + return dst; + } + if (da <= 0.0) { + return src; + } + var cs = src.xyz / sa; + var cb = dst.xyz / da; + if (!supports_hdr(mode)) { + cs = clamp(cs, vec3(0.0), vec3(1.0)); + cb = clamp(cb, vec3(0.0), vec3(1.0)); + } + let b = non_separable(mode, cb, cs); + let ao = sa + da - sa * da; + return vec4((1.0 - da) * src.xyz + (1.0 - sa) * dst.xyz + sa * da * b, ao); +} + +// ---------------------------------------------------------------- colour (color crate) + +fn srgb_to_linear(v: f32) -> f32 { + if (v <= 0.04045) { + return v / 12.92; + } + return pow((v + 0.055) / 1.055, 2.4); +} + +fn linear_to_srgb(v: f32) -> f32 { + if (v <= 0.0031308) { + return v * 12.92; + } + return 1.055 * pow(max(v, 0.0), 1.0 / 2.4) - 0.055; +} + +fn rgb_to_hsl(c: vec3) -> vec3 { + let mx = max(max(c.x, c.y), c.z); + let mn = min(min(c.x, c.y), c.z); + let l = (mx + mn) * 0.5; + if (abs(mx - mn) < 1e-7) { + return vec3(0.0, 0.0, l); + } + let d = mx - mn; + var s: f32; + if (l > 0.5) { + s = d / (2.0 - mx - mn); + } else { + s = d / (mx + mn); + } + var h: f32; + if (mx == c.x) { + h = ((c.y - c.z) / d + select(0.0, 6.0, c.y < c.z)) / 6.0; + } else if (mx == c.y) { + h = ((c.z - c.x) / d + 2.0) / 6.0; + } else { + h = ((c.x - c.y) / d + 4.0) / 6.0; + } + return vec3(h, s, l); +} + +fn hsl_channel(t0: f32, p: f32, q: f32) -> f32 { + let t = fract_euclid(t0); + if (t < 1.0 / 6.0) { + return p + (q - p) * 6.0 * t; + } + if (t < 0.5) { + return q; + } + if (t < 2.0 / 3.0) { + return p + (q - p) * (2.0 / 3.0 - t) * 6.0; + } + return p; +} + +fn hsl_to_rgb(h: f32, s: f32, l: f32) -> vec3 { + if (s <= 0.0) { + return vec3(l); + } + var q: f32; + if (l < 0.5) { + q = l * (1.0 + s); + } else { + q = l + s - l * s; + } + let p = 2.0 * l - q; + return vec3(hsl_channel(h + 1.0 / 3.0, p, q), hsl_channel(h, p, q), hsl_channel(h - 1.0 / 3.0, p, q)); +} + +// ColorSpace transfer curves: 1 = sRGB (sRGB, Display P3), 2 = 2.4 power (Rec. 709 / 2020). +fn curve_decode(curve: u32, v: f32) -> f32 { + let a = abs(v); + var l = a; + if (curve == 1u) { + l = srgb_to_linear(a); + } else if (curve == 2u) { + l = powz(a, 2.4); + } + return select(l, -l, v < 0.0); +} + +fn curve_encode(curve: u32, v: f32) -> f32 { + let a = abs(v); + var e = a; + if (curve == 1u) { + e = linear_to_srgb(a); + } else if (curve == 2u) { + e = powz(a, 1.0 / 2.4); + } + return select(e, -e, v < 0.0); +} diff --git a/crates/gpu/src/shaders/display.wgsl b/crates/gpu/src/shaders/display.wgsl new file mode 100644 index 000000000..682618759 --- /dev/null +++ b/crates/gpu/src/shaders/display.wgsl @@ -0,0 +1,19 @@ +// Viewer display: premultiplied f32 → premultiplied RGBA8 (ui-egui's frames::to_color_image), +// written straight into a texture egui samples. + +@group(0) @binding(1) var dsrc: texture_2d; +@group(0) @binding(3) var dout: texture_storage_2d; + +@compute @workgroup_size(16, 16) +fn display(@builtin(global_invocation_id) gid: vec3) { + let dims = vec2(textureDimensions(dout)); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let s = textureLoad(dsrc, p, 0); + let a = clamp(s.w, 0.0, 1.0); + // (v · 255 + 0.5) truncated, as the CPU conversion. + let q = floor(vec4(clamp(s.xyz, vec3(0.0), vec3(a)), a) * 255.0 + 0.5) / 255.0; + textureStore(dout, p, q); +} diff --git a/crates/gpu/src/shaders/kernels.wgsl b/crates/gpu/src/shaders/kernels.wgsl new file mode 100644 index 000000000..98ead5493 --- /dev/null +++ b/crates/gpu/src/shaders/kernels.wgsl @@ -0,0 +1,750 @@ +// Compositing and effect kernels (rgba32float output). Parameter layouts are documented next to +// each entry point and mirrored by the Rust dispatch code (src/ops.rs, src/effects.rs). + +@group(0) @binding(3) var out: texture_storage_2d; + +fn out_dims() -> vec2 { + return vec2(textureDimensions(out)); +} + +// ---------------------------------------------------------------- compositing + +// Warp `src` by the inverse matrix (output pixel → source pixel) and blend over `aux` +// (raster::composite_warp / accumulate_warp). +// u[0] = (mode, sampling, seed, flags: 1 accumulate, 2 clear outside bounds) +// f[0..3] = inverse rows; f[3] = (opacity, weight); f[4] = bounds (x0, y0, x1, y1) +@compute @workgroup_size(16, 16) +fn warp_blend(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let x = i32(gid.x); + let y = i32(gid.y); + if (x >= dims.x || y >= dims.y) { + return; + } + let mode = P.u[0].x; + let sampling = P.u[0].y; + let seed = P.u[0].z; + let flags = P.u[0].w; + let b = P.f[4]; + var d = textureLoad(aux, vec2(x, y), 0); + let fx = f32(x) + 0.5; + let fy = f32(y) + 0.5; + if (fx < b.x || fy < b.y || fx > b.z || fy > b.w) { + if ((flags & 2u) != 0u) { + d = vec4(0.0); + } + textureStore(out, vec2(x, y), d); + return; + } + let r0 = P.f[0]; + let r1 = P.f[1]; + let r2 = P.f[2]; + var px = r0.x * fx + r0.y * fy + r0.z; + var py = r1.x * fx + r1.y * fy + r1.z; + let w = r2.x * fx + r2.y * fy + r2.z; + if (w != 1.0 && w != 0.0) { + px = px / w; + py = py / w; + } + let sz = vec2(textureDimensions(src)); + let stencil = is_stencil(mode); + if (px < -1.0 || py < -1.0 || px > sz.x + 1.0 || py > sz.y + 1.0) { + if (stencil && (flags & 1u) == 0u) { + d = blend_pixel(mode, d, vec4(0.0), 0.5); + } + textureStore(out, vec2(x, y), d); + return; + } + var s = sample(src, sampling, px, py); + if ((flags & 1u) != 0u) { + if (s.w > 0.0) { + d = d + s * P.f[3].y; + } + textureStore(out, vec2(x, y), d); + return; + } + if (s.w <= 0.0 && !stencil) { + textureStore(out, vec2(x, y), d); + return; + } + let op = P.f[3].x; + if (op < 1.0) { + s = s * op; + } + var n = 0.5; + if (mode == 1u || mode == 2u) { + n = hash_noise(u32(x), u32(y), seed); + } + textureStore(out, vec2(x, y), blend_pixel(mode, d, s, n)); +} + +// Blend `src` (same size) onto `aux` (Image::blend_from). +// u[0] = (mode, _, seed); f[0].x = opacity +@compute @workgroup_size(16, 16) +fn blend_full(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let mode = P.u[0].x; + let d = textureLoad(aux, p, 0); + var s = textureLoad(src, p, 0); + if (s.w <= 0.0 && !is_stencil(mode)) { + textureStore(out, p, d); + return; + } + s = s * P.f[0].x; + var n = 0.5; + if (mode == 1u || mode == 2u) { + n = hash_noise(gid.x, gid.y, P.u[0].z); + } + textureStore(out, p, blend_pixel(mode, d, s, n)); +} + +// Track matte: `src` × matte factor of `aux`. u[0].x = 0 alpha, 1 alpha inverted, 2 luma, +// 3 luma inverted. +@compute @workgroup_size(16, 16) +fn matte(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let q = textureLoad(aux, p, 0); + var k: f32; + switch P.u[0].x { + case 0u: { k = q.w; } + case 1u: { k = 1.0 - q.w; } + case 2u: { k = luminance(q.xyz); } + default: { k = 1.0 - luminance(q.xyz); } + } + k = clamp(k, 0.0, 1.0); + textureStore(out, p, textureLoad(src, p, 0) * k); +} + +// Preserve Transparency: `src` × alpha of `aux`. +@compute @workgroup_size(16, 16) +fn preserve(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + textureStore(out, p, textureLoad(src, p, 0) * textureLoad(aux, p, 0).w); +} + +// Knockout: `src` × (1 − alpha of `aux`). +@compute @workgroup_size(16, 16) +fn knockout(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let keep = 1.0 - clamp(textureLoad(aux, p, 0).w, 0.0, 1.0); + textureStore(out, p, textureLoad(src, p, 0) * keep); +} + +// Styled layer finish: canvas `src` moves toward `aux` by the opacity, per enabled channel. +// u[0].xyz = R/G/B switches; f[0].x = opacity. +@compute @workgroup_size(16, 16) +fn channel_mix(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let c = textureLoad(src, p, 0); + let t = textureLoad(aux, p, 0); + let op = P.f[0].x; + var v = t; + if (P.u[0].x == 0u) { + v.x = c.x; + } + if (P.u[0].y == 0u) { + v.y = c.y; + } + if (P.u[0].z == 0u) { + v.z = c.z; + } + textureStore(out, p, c + (v - c) * op); +} + +// 8/16 bpc: clamp premultiplied pixels to 0..1 (colour ≤ alpha) and round. f[0].x = levels. +@compute @workgroup_size(16, 16) +fn quantize(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let levels = P.f[0].x; + let inv = 1.0 / levels; + let s = textureLoad(src, p, 0); + let a = round_away(clamp(s.w, 0.0, 1.0) * levels) * inv; + var o = vec4(0.0, 0.0, 0.0, a); + o.x = round_away(clamp(s.x, 0.0, a) * levels) * inv; + o.y = round_away(clamp(s.y, 0.0, a) * levels) * inv; + o.z = round_away(clamp(s.z, 0.0, a) * levels) * inv; + textureStore(out, p, o); +} + +// Colour conversion on straight colour (render::color::convert). +// u[0] = (decode curve, has matrix, encode curve); f[0..3] = matrix rows. +@compute @workgroup_size(16, 16) +fn convert(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let s = textureLoad(src, p, 0); + let a = s.w; + if (a <= 0.0) { + textureStore(out, p, s); + return; + } + var c = s.xyz / a; + let dc = P.u[0].x; + if (dc != 0u) { + c = vec3(curve_decode(dc, c.x), curve_decode(dc, c.y), curve_decode(dc, c.z)); + } + if (P.u[0].y != 0u) { + let m0 = P.f[0].xyz; + let m1 = P.f[1].xyz; + let m2 = P.f[2].xyz; + c = vec3(m0.x * c.x + m0.y * c.y + m0.z * c.z, m1.x * c.x + m1.y * c.y + m1.z * c.z, m2.x * c.x + m2.y * c.y + m2.z * c.z); + } + let ec = P.u[0].z; + if (ec != 0u) { + c = vec3(curve_encode(ec, c.x), curve_encode(ec, c.y), curve_encode(ec, c.z)); + } + textureStore(out, p, vec4(c * a, a)); +} + +// Box-filter halving for minification (raster::warp::half). +@compute @workgroup_size(16, 16) +fn half(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let sx = 2 * p.x; + let sy = 2 * p.y; + var acc = vec4(0.0); + acc += tex_get_clamped(src, sx, sy); + acc += tex_get_clamped(src, sx + 1, sy); + acc += tex_get_clamped(src, sx, sy + 1); + acc += tex_get_clamped(src, sx + 1, sy + 1); + textureStore(out, p, acc * 0.25); +} + +// ---------------------------------------------------------------- blurs (raster::blur) + +fn line_get(horizontal: bool, line: i32, i: i32, n: i32, repeat: bool) -> vec4 { + var j = i; + if (i < 0 || i >= n) { + if (!repeat) { + return vec4(0.0); + } + j = clamp(i, 0, n - 1); + } + if (horizontal) { + return textureLoad(src, vec2(j, line), 0); + } + return textureLoad(src, vec2(line, j), 0); +} + +// One box pass of radius r along rows (horizontal) or columns, as running sums over blocks of +// `block` pixels (each block restarts its sum, as the CPU's vertical pass does). +fn box_pass(gid: vec3, horizontal: bool) { + let dims = out_dims(); + let r = i32(P.u[0].x); + let repeat = P.u[0].y != 0u; + let block = i32(P.u[0].z); + var lines = dims.x; + var n = dims.y; + if (horizontal) { + lines = dims.y; + n = dims.x; + } + let line = i32(gid.x); + let b0 = i32(gid.y) * block; + if (line >= lines || b0 >= n) { + return; + } + let norm = 1.0 / f32(2 * r + 1); + var acc = vec4(0.0); + for (var i = b0 - r; i <= b0 + r; i++) { + acc += line_get(horizontal, line, i, n, repeat); + } + let b1 = min(b0 + block, n); + for (var i = b0; i < b1; i++) { + if (horizontal) { + textureStore(out, vec2(i, line), acc * norm); + } else { + textureStore(out, vec2(line, i), acc * norm); + } + let add = line_get(horizontal, line, i + r + 1, n, repeat); + let sub = line_get(horizontal, line, i - r, n, repeat); + acc += add - sub; + } +} + +// u[0] = (radius, repeat edges, block); dispatched over (line, block). +@compute @workgroup_size(64, 1) +fn box_h(@builtin(global_invocation_id) gid: vec3) { + box_pass(gid, true); +} + +@compute @workgroup_size(64, 1) +fn box_v(@builtin(global_invocation_id) gid: vec3) { + box_pass(gid, false); +} + +// Directional blur (raster::directional_blur). f[0] = (dx, dy, length, samples). +@compute @workgroup_size(16, 16) +fn directional(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let dx = P.f[0].x; + let dy = P.f[0].y; + let len = P.f[0].z; + let n = i32(P.f[0].w); + var acc = vec4(0.0); + for (var i = 0; i < n; i++) { + let t = (f32(i) / f32(n - 1) - 0.5) * len; + acc += sample_bilinear(src, f32(p.x) + 0.5 + dx * t, f32(p.y) + 0.5 + dy * t); + } + textureStore(out, p, acc / f32(n)); +} + +// ---------------------------------------------------------------- Glow / Drop Shadow + +// Glow bright pass. u[0].x = A & B colours; f[0].x = threshold; f[1] = colour A; f[2] = colour B. +@compute @workgroup_size(16, 16) +fn glow_bright(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let px = textureLoad(src, p, 0); + let a = px.w; + if (a <= 0.0) { + textureStore(out, p, vec4(0.0)); + return; + } + let thr = P.f[0].x; + let l = luminance(px.xyz / a); + let k = clamp((l - thr) / max(1.0 - thr, 1e-3), 0.0, 1.0); + if (P.u[0].x != 0u) { + let t = clamp(l, 0.0, 1.0); + let ca = P.f[1].xyz; + let cb = P.f[2].xyz; + let c = ca + (cb - ca) * t; + textureStore(out, p, vec4(c * k * a, k * a)); + } else { + textureStore(out, p, px * k); + } +} + +// Glow composite: original `src` with the blurred glow `aux`. u[0].x = behind; f[0].x = intensity. +@compute @workgroup_size(16, 16) +fn glow_combine(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + var o = textureLoad(src, p, 0); + let g = textureLoad(aux, p, 0) * P.f[0].x; + if (P.u[0].x != 0u) { + let k = 1.0 - o.w; + o = o + g * k; + } else { + o = vec4(o.xyz + g.xyz, min(o.w + g.w * (1.0 - o.w), 1.0)); + } + o.w = clamp(o.w, 0.0, 1.0); + textureStore(out, p, o); +} + +// Drop shadow silhouette. f[0] = shadow colour; f[1] = (opacity, dx, dy). +@compute @workgroup_size(16, 16) +fn shadow_make(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let c = P.f[0]; + let a = sample_bilinear(src, f32(p.x) + 0.5 - P.f[1].y, f32(p.y) + 0.5 - P.f[1].z).w * P.f[1].x * c.w; + textureStore(out, p, vec4(c.xyz * a, a)); +} + +// The layer over its shadow: `aux` (original) over `src` (shadow). +@compute @workgroup_size(16, 16) +fn shadow_combine(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let s = textureLoad(src, p, 0); + let o = textureLoad(aux, p, 0); + textureStore(out, p, o + s * (1.0 - o.w)); +} + +// ---------------------------------------------------------------- per-pixel effects + +fn mix3(a: vec3, b: vec3, t: f32) -> vec3 { + return a + (b - a) * t; +} + +// Curves: data = [n, xs.., ys.., ms.., lut (1025)] per channel at offsets f[c]. +fn curve_exact(base: u32, x: f32) -> f32 { + let n = u32(data[base]); + let xs = base + 1u; + let ys = xs + n; + let ms = ys + n; + if (x <= data[xs]) { + return data[ys] + (x - data[xs]) * data[ms]; + } + if (x >= data[xs + n - 1u]) { + return data[ys + n - 1u] + (x - data[xs + n - 1u]) * data[ms + n - 1u]; + } + // partition_point(|v| v <= x) - 1, clamped to n - 2. + var k = 0u; + for (var i = 0u; i < n; i++) { + if (data[xs + i] <= x) { + k = i + 1u; + } + } + k = min(max(k, 1u) - 1u, n - 2u); + let h = data[xs + k + 1u] - data[xs + k]; + let t = (x - data[xs + k]) / h; + let t2 = t * t; + let t3 = t * t * t; + let h00 = 2.0 * t3 - 3.0 * t2 + 1.0; + let h10 = t3 - 2.0 * t2 + t; + let h01 = -2.0 * t3 + 3.0 * t2; + let h11 = t3 - t2; + return h00 * data[ys + k] + h10 * h * data[ms + k] + h01 * data[ys + k + 1u] + h11 * h * data[ms + k + 1u]; +} + +fn curve_eval(base_f: f32, x: f32) -> f32 { + if (base_f < 0.0) { + return x; + } + let base = u32(base_f); + if (!(x >= 0.0 && x <= 1.0)) { + return curve_exact(base, x); + } + let n = u32(data[base]); + let lut = base + 1u + 3u * n; + let f = x * 1024.0; + let i = min(u32(f), 1023u); + let t = f - f32(i); + return data[lut + i] + (data[lut + i + 1u] - data[lut + i]) * t; +} + +// Fractal Noise lattice (effects::generate::lattice / value_noise). +fn lattice(ix: i32, iy: i32, iz: i32, seed: u32) -> f32 { + var h = (bitcast(ix) * 0x27d4eb2du) ^ (bitcast(iy) * 0x165667b1u) ^ (bitcast(iz) * 0x9e3779b9u) ^ (seed * 0x85ebca6bu); + h = h ^ (h >> 15u); + h = h * 0x2c1b3c6du; + h = h ^ (h >> 12u); + h = h * 0x297a2d39u; + h = h ^ (h >> 15u); + return f32(h & 0xffffu) / 65535.0; +} + +fn fade(t: f32) -> f32 { + return t * t * t * (t * (t * 6.0 - 15.0) + 10.0); +} + +fn value_noise(x: f32, y: f32, z: f32, seed: u32) -> f32 { + let x0 = floor(x); + let y0 = floor(y); + let z0 = floor(z); + let fx = fade(x - x0); + let fy = fade(y - y0); + let fz = fade(z - z0); + let ix = i32(x0); + let iy = i32(y0); + let iz = i32(z0); + let c000 = lattice(ix, iy, iz, seed); + let c100 = lattice(ix + 1, iy, iz, seed); + let c010 = lattice(ix, iy + 1, iz, seed); + let c110 = lattice(ix + 1, iy + 1, iz, seed); + let c001 = lattice(ix, iy, iz + 1, seed); + let c101 = lattice(ix + 1, iy, iz + 1, seed); + let c011 = lattice(ix, iy + 1, iz + 1, seed); + let c111 = lattice(ix + 1, iy + 1, iz + 1, seed); + let l00 = c000 + (c100 - c000) * fx; + let l01 = c010 + (c110 - c010) * fx; + let a = l00 + (l01 - l00) * fy; + let l10 = c001 + (c101 - c001) * fx; + let l11 = c011 + (c111 - c011) * fx; + let b = l10 + (l11 - l10) * fy; + return a + (b - a) * fz; +} + +// generate::put: blend a generated straight colour (alpha ga) over the pixel. +fn put(px: vec4, g: vec3, ga: f32, blend_orig: f32) -> vec4 { + let g2 = vec4(g * ga, ga); + let k = 1.0 - blend_orig; + return px * blend_orig + g2 * k; +} + +// Straight-colour map helper: skips transparent pixels (Image::map_straight). +fn straight(px: vec4) -> vec3 { + return px.xyz / px.w; +} + +// u[0].x = op: +// 1 Tint f[0] = black, f[1] = white, f[2].x = amount +// 2 Bright/Contrast f[0] = (brightness, k) +// 3 Hue/Saturation u[0].y = colorize; f[0] = (hue, sat, light); f[1] = (ch, cs, cl) +// 4 Levels f[0] = (ib, iw, g, ob); f[1] = (ow); u[0].y = clip +// 5 Exposure f[0] = (e, offset, gamma) +// 6 Invert u[0].y = channel; f[0].x = blend +// 7 Fill f[0] = colour; f[1] = (opacity); u[0].y = invert +// 8 Curves f[0] = data offsets (rgb, red, green, blue), f[1].x = alpha (−1 = none) +// 9 Gradient Ramp f[0] = (sx, sy, ex, ey); f[1] = start colour; f[2] = end colour; +// f[3] = (len2, scatter, blend); u[0] = (_, radial, seed) +// 10 Fractal Noise f[0] = (contrast, brightness, scale, frac); f[1] = (ox, oy, sr, cr); +// f[2] = (evo, blend, opacity); u[0] = (_, kind, n_oct, seed); u[1].x = invert +@compute @workgroup_size(16, 16) +fn pointwise(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + var px = textureLoad(src, p, 0); + let a = px.w; + let op = P.u[0].x; + switch op { + case 1u: { + if (a > 0.0) { + let c = straight(px); + let l = clamp(luminance(c), 0.0, 1.0); + let blk = P.f[0].xyz; + let wht = P.f[1].xyz; + let t = blk + (wht - blk) * l; + px = vec4(mix3(c, t, P.f[2].x) * a, a); + } + } + case 2u: { + if (a > 0.0) { + let c = straight(px); + let br = P.f[0].x; + let k = P.f[0].y; + px = vec4(max((c + br - 0.5) * k + 0.5, vec3(0.0)) * a, a); + } + } + case 3u: { + if (a > 0.0) { + let c = straight(px); + let hsl = rgb_to_hsl(c); + var h = hsl.x; + var s = hsl.y; + var l = hsl.z; + if (P.u[0].y != 0u) { + let cl = P.f[1].z; + var k = l; + if (cl > 0.0) { + k = 1.0 - l; + } + h = P.f[1].x; + s = P.f[1].y; + l = clamp(l + cl * k, 0.0, 1.0); + } else { + let hue = P.f[0].x; + let sat = P.f[0].y; + let light = P.f[0].z; + var s2: f32; + if (sat >= 0.0) { + s2 = s + (1.0 - s) * sat * min(s, 1.0); + } else { + s2 = s * (1.0 + sat); + } + var l2: f32; + if (light >= 0.0) { + l2 = l + (1.0 - l) * light; + } else { + l2 = l * (1.0 + light); + } + h = fract_euclid(h + hue); + s = clamp(s2, 0.0, 1.0); + l = l2; + } + px = vec4(hsl_to_rgb(h, s, l) * a, a); + } + } + case 4u: { + if (a > 0.0) { + let c = straight(px); + let ib = P.f[0].x; + let iw = P.f[0].y; + let g = P.f[0].z; + let ob = P.f[0].w; + let ow = P.f[1].x; + var o = vec3(0.0); + for (var i = 0; i < 3; i++) { + var t = (c[i] - ib) / max(iw - ib, 1e-6); + if (P.u[0].y != 0u) { + t = clamp(t, 0.0, 1.0); + } + t = powz(max(t, 0.0), 1.0 / g); + o[i] = ob + (ow - ob) * t; + } + px = vec4(o * a, a); + } + } + case 5u: { + if (a > 0.0) { + let c = straight(px); + var o = vec3(0.0); + for (var i = 0; i < 3; i++) { + let lin = srgb_to_linear(max(c[i], 0.0)); + let v = powz(max(lin * P.f[0].x + P.f[0].y, 0.0), 1.0 / P.f[0].z); + o[i] = linear_to_srgb(v); + } + px = vec4(o * a, a); + } + } + case 6u: { + let ch = P.u[0].y; + let blend = P.f[0].x; + if (ch == 4u) { + let na = 1.0 - a; + var k = 0.0; + if (a > 0.0) { + k = na / a; + } + let inv = vec4(px.xyz * k, na); + px = px + (inv - px) * blend; + } else if (a > 0.0) { + let s = straight(px); + var o = s; + switch ch { + case 1u: { o.x = 1.0 - s.x; } + case 2u: { o.y = 1.0 - s.y; } + case 3u: { o.z = 1.0 - s.z; } + default: { o = 1.0 - s; } + } + px = vec4((s + (o - s) * blend) * a, a); + } + } + case 7u: { + let c = P.f[0]; + let invert = P.u[0].y != 0u; + let opacity = P.f[1].x; + var fa = px.w; + if (invert) { + fa = 1.0 - px.w; + } + let f = c.xyz * fa; + let rgb = px.xyz + (f - px.xyz) * opacity; + var alpha = px.w; + if (invert) { + alpha = px.w + (fa - px.w) * opacity; + } + px = vec4(rgb, alpha); + } + case 8u: { + // effects::color2::map_ca: transparent pixels only change through the alpha curve. + if (a <= 0.0 && px.x == 0.0 && px.y == 0.0 && px.z == 0.0) { + let na = curve_eval(P.f[1].x, 0.0); + if (na > 0.0) { + px = vec4(0.0, 0.0, 0.0, clamp(na, 0.0, 1.0)); + } + } else { + var c = vec3(0.0); + var aa = max(a, 0.0); + if (a > 1e-6) { + c = px.xyz / a; + aa = a; + } + var o = vec3(0.0); + for (var i = 0; i < 3; i++) { + o[i] = curve_eval(P.f[0][i + 1], curve_eval(P.f[0].x, c[i])); + } + let na = clamp(curve_eval(P.f[1].x, aa), 0.0, 1.0); + px = vec4(o * na, na); + } + } + case 9u: { + let s = P.f[0].xy; + let e = P.f[0].zw; + let d = e - s; + let v = vec2(f32(p.x) + 0.5, f32(p.y) + 0.5) - s; + let len2 = P.f[3].x; + var t: f32; + if (P.u[0].y != 0u) { + t = sqrt((v.x * v.x + v.y * v.y) / len2); + } else { + t = (v.x * d.x + v.y * d.y) / len2; + } + let scatter = P.f[3].y; + if (scatter > 0.0) { + t += (hash_noise(gid.x, gid.y, P.u[0].z) - 0.5) * scatter; + } + t = clamp(t, 0.0, 1.0); + let c0 = P.f[1].xyz; + let c1 = P.f[2].xyz; + px = put(px, c0 + (c1 - c0) * t, 1.0, P.f[3].z); + } + case 10u: { + let contrast = P.f[0].x; + let brightness = P.f[0].y; + let scale = P.f[0].z; + let frac = P.f[0].w; + let ox = P.f[1].x; + let oy = P.f[1].y; + let sr = P.f[1].z; + let cr = P.f[1].w; + let evo = P.f[2].x; + let kind = P.u[0].y; + let n_oct = P.u[0].z; + let seed = P.u[0].w; + let dx = f32(p.x) + 0.5 - ox; + let dy = f32(p.y) + 0.5 - oy; + let u = (dx * cr + dy * sr) / scale; + let v = (-dx * sr + dy * cr) / scale; + var sum = 0.0; + var amp = 1.0; + var norm = 0.0; + var f = 1.0; + for (var o = 0u; o < n_oct; o++) { + var w = 1.0; + if (o + 1u == n_oct && frac > 0.0) { + w = frac; + } + var n = value_noise(u * f, v * f, evo + f32(o) * 7.31, seed + o); + if (kind == 1u) { + n = 1.0 - abs(n * 2.0 - 1.0); + } + sum += n * amp * w; + norm += amp * w; + amp *= 0.5; + f *= 2.0; + } + var val = sum / max(norm, 1e-6); + val = clamp((val - 0.5) * contrast + 0.5 + brightness, 0.0, 1.0); + if (P.u[1].x != 0u) { + val = 1.0 - val; + } + px = put(px, vec3(val), P.f[2].z, P.f[2].y); + } + default: {} + } + textureStore(out, p, px); +} diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs new file mode 100644 index 000000000..ca2fb1b0c --- /dev/null +++ b/crates/gpu/src/tests.rs @@ -0,0 +1,517 @@ +//! GPU vs CPU: the CPU renderer is the oracle. Every scene renders on both and must agree +//! within ≤ 1/255 at 8 bpc and ≤ 1e-3 at 16/32 bpc (per channel, premultiplied; relative for +//! over-range 32 bpc values above 1). +//! +//! Two documented exceptions, both from float rounding (the GPU computes in f32 with fast math, +//! the CPU's transforms in f64): +//! * 8/16 bpc quantise after every layer and effect, so a value within ~1e-6 of a rounding +//! boundary can land one step apart and later steps (Overlay's ×2 slope, colour-space curves) +//! can widen it: up to 0.3 % of pixels may differ by at most 4/255. +//! * Discontinuous modes (Hard Mix, Dissolve, Darker/Lighter Color) may flip pixels sitting +//! exactly on a threshold: up to 0.5 % of the frame. +//! +//! The tests skip (pass with a note) when no GPU adapter is available, as on headless CI. + +use std::sync::{Arc, OnceLock}; + +use effectcraft_color::{BlendMode, ColorSpace, Label}; +use effectcraft_keyframe::{Keyframe, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{ + AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, MatteKind, Project, Quality, Sampling, Solid, TrackMatte, +}; +use effectcraft_raster::Image; +use effectcraft_render::{Backend, FootageSource, RenderOpts, Renderer}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::Gpu; + +fn gpu() -> Option<&'static Gpu> { + static G: OnceLock> = OnceLock::new(); + G.get_or_init(|| { + let g = Gpu::headless(); + if g.is_none() { + eprintln!("effectcraft-gpu tests: no GPU adapter, skipping GPU comparisons"); + } + g + }) + .as_ref() +} + +/// Procedural footage: smooth colour gradients, fine noise and an alpha with soft, opaque and +/// fully transparent regions (different per item). +struct Pattern; + +fn pattern(seed: u32, w: u32, h: u32) -> Image { + let mut img = Image::new(w, h); + let k = seed as f32 * 0.37; + for y in 0..h { + for x in 0..w { + let (u, v) = (x as f32 / w as f32, y as f32 / h as f32); + let n = effectcraft_raster::hash_noise(x, y, seed) * 0.25; + let r = (0.5 + 0.5 * (u * 7.0 + k).sin()) * 0.75 + n; + let g = (0.5 + 0.5 * (v * 5.0 - k).cos()) * 0.75 + n * 0.5; + let b = ((u + v) * 0.5 + k).fract() * 0.8 + 0.1; + let d = ((u - 0.5).powi(2) + (v - 0.5).powi(2)).sqrt(); + let a = ((0.48 - d) * 6.0).clamp(0.0, 1.0) * if (x / 9 + y / 7 + seed).is_multiple_of(5) { 0.6 } else { 1.0 }; + img.set(x, y, [r.min(1.0) * a, g.min(1.0) * a, b * a, a]); + } + } + img +} + +impl FootageSource for Pattern { + fn frame(&self, item: ItemId, f: &Footage, _t: Tick) -> Option> { + Some(Arc::new(pattern(item.0 as u32, f.width, f.height))) + } +} + +struct Scene { + p: Project, + cid: ItemId, + comp: Comp, +} + +impl Scene { + fn new(depth: BitDepth) -> Scene { + let mut p = Project::default(); + p.settings.bit_depth = depth; + let comp = Comp::new(97, 61, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + Scene { p, cid, comp } + } + + fn footage(&mut self, w: u32, h: u32) -> Layer { + let f = Footage { + path: "pattern.png".into(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha: AlphaMode::Premultiplied, + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + color_profile: None, + }; + let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); + build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) + } + + fn solid(&mut self, color: [f32; 3], w: u32, h: u32) -> Layer { + let sid = self.p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: 1.0 })); + build::layer(&mut self.p, &self.comp, "Solid", LayerSource::Solid { item: sid }, (w, h), None) + } + + /// Add on top of the stack. + fn push(&mut self, l: Layer) -> effectcraft_project::LayerId { + let id = l.id; + self.p.comp_mut(self.cid).unwrap().layers.insert(0, l); + id + } + + fn effect(&mut self, l: &mut Layer, id: &str, vals: &[(&str, Value)]) { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = self.p.next_id; + let size = effectcraft_render::source_size(&self.p, l); + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [size.0 as f64, size.1 as f64]); + self.p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap_or_else(|| panic!("{id}: no {k}")).value = v.clone(); + } + l.props.sub_mut("effects").unwrap().children.push(g.into()); + } +} + +fn set(l: &mut Layer, path: &str, v: Value) { + l.props.prop_mut(path).unwrap_or_else(|| panic!("no {path}")).value = v; +} + +fn v3(x: f64, y: f64) -> Value { + Value::Vec3([x, y, 0.0]) +} + +/// Differences of a GPU render against the CPU render. +#[derive(Debug)] +struct Diff { + max: f32, + /// Pixels over the tolerance. + over: usize, + total: usize, + worst: (u32, u32, [f32; 4], [f32; 4]), + /// 8/16 bpc render. + quantized: bool, +} + +fn diff(cpu: &Image, gpu: &Image, tol: f32) -> Diff { + assert_eq!((cpu.width, cpu.height), (gpu.width, gpu.height), "size"); + let mut d = Diff { max: 0.0, over: 0, total: cpu.data.len(), worst: (0, 0, [0.0; 4], [0.0; 4]), quantized: false }; + for (i, (a, b)) in cpu.data.iter().zip(&gpu.data).enumerate() { + // Absolute below 1, relative above (32 bpc over-range values, e.g. Divide by ~0). + let m = (0..4).map(|c| (a[c] - b[c]).abs() / a[c].abs().max(1.0)).fold(0.0f32, f32::max); + let m = if m.is_nan() { f32::INFINITY } else { m }; + if m > tol { + d.over += 1; + } + if m > d.max { + d.max = m; + d.worst = (i as u32 % cpu.width, i as u32 / cpu.width, *a, *b); + } + } + d +} + +fn tolerance(depth: BitDepth) -> f32 { + match depth { + BitDepth::Bpc8 => 1.0 / 255.0 + 1e-6, + _ => 1e-3, + } +} + +/// Render on the CPU and the GPU; `Some(diff)` (None without an adapter). +fn compare_at(s: &Scene, opts: RenderOpts, t: Tick) -> Option { + let g = gpu()?; + let cpu = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Cpu, ..opts }).comp_frame_cpu(s.cid, t); + let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..opts }); + r.accel = Some(g); + let img = g.render(&r, s.cid, t).expect("the GPU renders the frame"); + let mut d = diff(&cpu, &img, tolerance(s.p.settings.bit_depth)); + d.quantized = s.p.settings.bit_depth != BitDepth::Bpc32; + Some(d) +} + +fn opts() -> RenderOpts { + RenderOpts { motion_blur: true, ..Default::default() } +} + +/// Assert agreement; `allow` = fraction of pixels allowed over the tolerance. +fn check(label: &str, d: Option, allow: f64) { + let Some(d) = d else { return }; + let frac = d.over as f64 / d.total as f64; + // Quantised depths: rounding-boundary flips (see the module docs). + let (allow, cap) = if d.quantized && allow == 0.0 { (0.003, 4.0 / 255.0 + 1e-6) } else { (allow, f32::INFINITY) }; + assert!(frac <= allow && d.max <= cap, "{label}: {} of {} pixels over tolerance (max {:.6}); worst {:?}", d.over, d.total, d.max, d.worst); + if d.over > 0 { + eprintln!("{label}: {} of {} pixels over tolerance (max {:.6})", d.over, d.total, d.max); + } +} + +fn discontinuous(m: BlendMode) -> bool { + matches!(m, BlendMode::HardMix | BlendMode::Dissolve | BlendMode::DancingDissolve | BlendMode::DarkerColor | BlendMode::LighterColor) +} + +/// Background, an Add layer (over-range values in 32 bpc) and a rotated, scaled top layer in +/// `mode` at 70 % opacity. +fn blend_scene(depth: BitDepth, mode: BlendMode) -> Scene { + let mut s = Scene::new(depth); + let bg = s.footage(97, 61); + s.push(bg); + let mut add = s.footage(80, 50); + add.blend_mode = BlendMode::Add; + set(&mut add, "transform/opacity", Value::Scalar(50.0)); + s.push(add); + let mut top = s.footage(64, 48); + top.blend_mode = mode; + set(&mut top, "transform/opacity", Value::Scalar(70.0)); + set(&mut top, "transform/rotation", Value::Scalar(17.0)); + set(&mut top, "transform/scale", Value::Vec3([115.0, 90.0, 100.0])); + set(&mut top, "transform/position", v3(52.3, 29.6)); + s.push(top); + s +} + +#[test] +fn all_blend_modes_at_every_bit_depth() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { + for mode in BlendMode::ALL { + let s = blend_scene(depth, mode); + let allow = if discontinuous(mode) { 0.005 } else { 0.0 }; + check(&format!("{mode:?} {depth:?}"), compare_at(&s, opts(), Tick::ZERO), allow); + } + } +} + +#[test] +fn transforms_sampling_and_resolution() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + for (i, (sampling, quality, scale, rot, sc)) in [ + (Sampling::Bilinear, Quality::Best, 1.0, 0.0, 100.0), + (Sampling::Bilinear, Quality::Best, 1.0, 33.0, 140.0), + (Sampling::Bicubic, Quality::Best, 1.0, -21.0, 160.0), + (Sampling::Bilinear, Quality::Best, 1.0, 12.0, 30.0), + (Sampling::Bicubic, Quality::Best, 0.5, 45.0, 80.0), + (Sampling::Bilinear, Quality::Draft, 1.0, 0.0, 200.0), + (Sampling::Bilinear, Quality::Best, 0.25, 5.0, 100.0), + ] + .into_iter() + .enumerate() + { + let mut s = Scene::new(depth); + let bg = s.solid([0.1, 0.2, 0.3], 97, 61); + s.push(bg); + let mut l = s.footage(90, 70); + l.switches.sampling = sampling; + l.switches.quality = quality; + set(&mut l, "transform/rotation", Value::Scalar(rot)); + set(&mut l, "transform/scale", Value::Vec3([sc, sc * 0.9, 100.0])); + set(&mut l, "transform/position", v3(47.25, 31.5)); + s.push(l); + check(&format!("transform #{i} {depth:?}"), compare_at(&s, RenderOpts { scale, ..opts() }, Tick::ZERO), 0.0); + } + } +} + +#[test] +fn masks_mattes_and_preserve_transparency() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + for kind in [MatteKind::Alpha, MatteKind::AlphaInverted, MatteKind::Luma, MatteKind::LumaInverted] { + let mut s = Scene::new(depth); + let bg = s.footage(97, 61); + s.push(bg); + let mut matte = s.footage(60, 40); + matte.switches.video = false; + set(&mut matte, "transform/rotation", Value::Scalar(-12.0)); + let mut next = s.p.next_id; + let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::ellipse([30.0, 20.0], 50.0, 34.0), MaskMode::Add, [255, 255, 0]); + s.p.next_id = next; + matte.props.sub_mut("masks").unwrap().children.push(m.into()); + let mid = s.push(matte); + let mut fill = s.footage(97, 61); + fill.track_matte = Some(TrackMatte { layer: mid, kind }); + fill.blend_mode = BlendMode::Screen; + s.push(fill); + let mut pt = s.solid([0.9, 0.4, 0.1], 50, 30); + pt.preserve_transparency = true; + set(&mut pt, "transform/opacity", Value::Scalar(60.0)); + s.push(pt); + check(&format!("matte {kind:?} {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } + } +} + +#[test] +fn motion_blur_accumulates_like_the_cpu() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = Scene::new(depth); + s.p.comp_mut(s.cid).unwrap().enable_motion_blur = true; + s.comp.enable_motion_blur = true; + let bg = s.solid([0.05, 0.05, 0.1], 97, 61); + s.push(bg); + let mut l = s.footage(30, 30); + l.switches.motion_blur = true; + l.blend_mode = BlendMode::Add; + let pos = l.props.prop_mut("transform/position").unwrap(); + pos.keys = vec![Keyframe::new(Tick::ZERO, v3(10.0, 20.0)), Keyframe::new(Tick::from_seconds_f64(0.5), v3(90.0, 45.0))]; + let rot = l.props.prop_mut("transform/rotation").unwrap(); + rot.keys = vec![Keyframe::new(Tick::ZERO, Value::Scalar(0.0)), Keyframe::new(Tick::from_seconds_f64(0.5), Value::Scalar(180.0))]; + s.push(l); + check(&format!("motion blur {depth:?}"), compare_at(&s, opts(), Tick::from_seconds_f64(0.2)), 0.0); + } +} + +#[test] +fn colour_management_and_linear_blending() { + for (ws, linearize, blend_linear) in + [(None, false, true), (Some(ColorSpace::Rec2020), false, false), (Some(ColorSpace::Srgb), true, false), (Some(ColorSpace::Rec709), false, true)] + { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = blend_scene(depth, BlendMode::Overlay); + s.p.settings.working_space = ws; + s.p.settings.linearize = linearize; + s.p.settings.blend_linear = blend_linear; + check(&format!("colour {ws:?} {linearize} {blend_linear} {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } + } +} + +#[test] +fn layer_styles_collapsed_precomps_and_cpu_fallbacks() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + // Layer styles (drop shadow pass + stroke body). + let mut s = Scene::new(depth); + let bg = s.footage(97, 61); + s.push(bg); + let mut l = s.solid([0.8, 0.3, 0.2], 40, 30); + let mut next = s.p.next_id; + effectcraft_project::styles::add_style(&mut l, &mut Ids(&mut next), "dropShadow", &s.comp.global_light, true).unwrap(); + effectcraft_project::styles::add_style(&mut l, &mut Ids(&mut next), "stroke", &s.comp.global_light, true).unwrap(); + s.p.next_id = next; + l.blend_mode = BlendMode::Multiply; + set(&mut l, "transform/rotation", Value::Scalar(10.0)); + s.push(l); + check(&format!("styles {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + + // A collapsed precomp (nested blend modes act on the parent) and an adjustment layer + // (CPU fallback between GPU steps). + let mut s = Scene::new(depth); + let inner = Comp::new(60, 40, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let inner_id = s.p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into())); + let (a, b) = { + let mut a = s.footage(60, 40); + a.blend_mode = BlendMode::Normal; + let mut b = s.solid([0.2, 0.7, 0.4], 30, 20); + b.blend_mode = BlendMode::Difference; + (a, b) + }; + s.p.comp_mut(inner_id).unwrap().layers = vec![b, a]; + let bg = s.footage(97, 61); + s.push(bg); + let mut pre = build::layer(&mut s.p, &s.comp, "Inner", LayerSource::Comp { item: inner_id }, (60, 40), None); + pre.switches.collapse = true; + set(&mut pre, "transform/rotation", Value::Scalar(25.0)); + set(&mut pre, "transform/opacity", Value::Scalar(80.0)); + s.push(pre); + let mut adj = s.solid([1.0, 1.0, 1.0], 50, 61); + adj.switches.adjustment = true; + s.effect(&mut adj, "ec.channel.invert", &[]); + s.push(adj); + check(&format!("collapse + adjustment {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } +} + +/// The effect alone on a buffer: GPU kernels vs the CPU effect (32-bit float), including the +/// buffer geometry (padding, offset). +fn effect_direct(id: &str, vals: &[(&str, Value)], adjustment: bool) { + let Some(g) = gpu() else { return }; + let spec = effectcraft_effects::find(id).unwrap(); + let size = [70.0, 44.0]; + let mut params = + effectcraft_effects::Params { values: spec.params.iter().map(|ps| (ps.id.to_string(), effectcraft_effects::default_value(ps, size))).collect() }; + for (k, v) in vals { + params.values.insert(k.to_string(), v.clone()); + } + for scale in [1.0, 0.5] { + let img = pattern(7, (size[0] * scale) as u32, (size[1] * scale) as u32); + let buf = effectcraft_effects::Buf { img, offset: [0.0, 0.0], scale }; + let ctx = || effectcraft_effects::EffectCtx { params: ¶ms, time: 0.25, layer_size: size, seed: 11, adjustment, env: Default::default() }; + let cpu = (spec.render)(&ctx(), buf.clone()); + let out = effectcraft_render::Accelerator::effects(g, &[effectcraft_render::FxStep { spec, ctx: ctx() }], &buf, None).expect("the GPU runs the effect"); + assert_eq!((out.offset, out.scale), (cpu.offset, cpu.scale), "{id}: geometry"); + let d = diff(&cpu.img, &out.img, 1e-3); + assert!(d.over == 0, "{id} {vals:?} adj {adjustment} scale {scale}: {} of {} pixels over 1e-3 (max {}); worst {:?}", d.over, d.total, d.max, d.worst); + } +} + +/// One footage layer with `fx` applied, composited over a background, at 8 and 32 bpc. +fn effect_case(id: &str, vals: &[(&str, Value)]) { + effect_direct(id, vals, false); + effect_direct(id, vals, true); + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = Scene::new(depth); + let bg = s.solid([0.15, 0.1, 0.2], 97, 61); + s.push(bg); + let mut l = s.footage(70, 44); + s.effect(&mut l, id, vals); + set(&mut l, "transform/rotation", Value::Scalar(8.0)); + s.push(l); + check(&format!("{id} {vals:?} {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + check(&format!("{id} {vals:?} {depth:?} half"), compare_at(&s, RenderOpts { scale: 0.5, ..opts() }, Tick::ZERO), 0.0); + } +} + +fn n(v: f64) -> Value { + Value::Scalar(v) +} + +fn c(r: f64, g: f64, b: f64) -> Value { + Value::Color([r, g, b, 1.0]) +} + +#[test] +fn gpu_effects_match_cpu() { + effect_case("ec.blur.gaussian", &[("blurriness", n(9.0))]); + effect_case("ec.blur.gaussian", &[("blurriness", n(4.0)), ("dimensions", Value::Enum(1)), ("repeatEdge", Value::Bool(true))]); + effect_case("ec.blur.fastbox", &[("radius", n(5.0)), ("iterations", n(2.0))]); + effect_case("ec.blur.fastbox", &[("radius", n(3.0)), ("dimensions", Value::Enum(2)), ("repeatEdge", Value::Bool(true))]); + effect_case("ec.blur.directional", &[("direction", n(30.0)), ("length", n(12.0))]); + effect_case("ec.stylize.glow", &[("threshold", n(40.0)), ("radius", n(14.0)), ("intensity", n(1.5))]); + effect_case("ec.stylize.glow", &[("colors", Value::Enum(1)), ("operation", Value::Enum(1)), ("colorA", c(1.0, 0.8, 0.1))]); + effect_case("ec.color.levels", &[("inBlack", n(0.1)), ("inWhite", n(0.8)), ("gamma", n(1.6)), ("outBlack", n(0.05))]); + effect_case("ec.color.levels", &[("inWhite", n(0.5)), ("noClip", Value::Bool(true))]); + effect_case( + "ec.color.curves", + &[("rgb", Value::Str("0,0 0.3,0.45 1,1".into())), ("red", Value::Str("0,0.1 1,0.9".into())), ("alpha", Value::Str("0,0 0.5,0.7 1,1".into()))], + ); + effect_case("ec.color.huesaturation", &[("hue", n(70.0)), ("saturation", n(40.0)), ("lightness", n(-20.0))]); + effect_case("ec.color.huesaturation", &[("colorize", Value::Bool(true)), ("colorizeHue", n(200.0)), ("colorizeLightness", n(15.0))]); + effect_case("ec.color.tint", &[("black", c(0.1, 0.0, 0.3)), ("white", c(1.0, 0.9, 0.5)), ("amount", n(80.0))]); + effect_case("ec.generate.fill", &[("color", c(0.2, 0.6, 1.0)), ("opacity", n(70.0))]); + effect_case("ec.generate.fill", &[("invert", Value::Bool(true))]); + effect_case("ec.generate.gradientramp", &[("start", Value::Vec2([5.0, 3.0])), ("end", Value::Vec2([60.0, 40.0])), ("blend", n(25.0))]); + effect_case("ec.generate.gradientramp", &[("shape", Value::Enum(1)), ("scatter", n(40.0))]); + effect_case("ec.noise.fractal", &[]); + effect_case( + "ec.noise.fractal", + &[("fractalType", Value::Enum(1)), ("complexity", n(4.5)), ("rotation", n(30.0)), ("evolution", n(90.0)), ("scale", n(40.0)), ("blend", n(30.0))], + ); + effect_case("ec.perspective.dropshadow", &[("distance", n(6.0)), ("softness", n(8.0)), ("opacity", n(80.0))]); + effect_case("ec.perspective.dropshadow", &[("shadowOnly", Value::Bool(true)), ("color", c(0.0, 0.2, 0.6))]); + effect_case("ec.color.brightnesscontrast", &[("brightness", n(20.0)), ("contrast", n(35.0))]); + effect_case("ec.color.brightnesscontrast", &[("brightness", n(-10.0)), ("contrast", n(-40.0))]); + effect_case("ec.color.exposure", &[("exposure", n(1.3)), ("offset", n(0.02)), ("gamma", n(1.2))]); + effect_case("ec.channel.invert", &[]); + effect_case("ec.channel.invert", &[("channel", Value::Enum(4)), ("blend", n(30.0))]); + effect_case("ec.channel.invert", &[("channel", Value::Enum(2))]); + effect_case("ec.distort.transform", &[("rotation", n(20.0)), ("scaleHeight", n(80.0)), ("skew", n(10.0)), ("opacity", n(70.0))]); +} + +/// A chain of GPU effects (one upload, quantised between steps) and a non-GPU effect in the +/// middle (split into two GPU chains around the CPU effect). +#[test] +fn effect_chains_and_mixed_stacks() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = Scene::new(depth); + let mut l = s.footage(80, 50); + s.effect(&mut l, "ec.color.exposure", &[("exposure", n(0.7))]); + s.effect(&mut l, "ec.blur.gaussian", &[("blurriness", n(5.0))]); + s.effect(&mut l, "ec.stylize.posterize", &[]); + s.effect(&mut l, "ec.stylize.glow", &[]); + s.effect(&mut l, "ec.channel.invert", &[("blend", n(50.0))]); + s.push(l); + check(&format!("chain {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } +} + +#[test] +fn registry_badges_match_the_gpu_implementation() { + for s in effectcraft_effects::registry() { + let implemented = crate::effects::supports(s.id); + assert_eq!(s.gpu, implemented, "{}: registry gpu = {}, GPU implementation = {implemented}", s.id, s.gpu); + } +} + +#[test] +fn backend_selection_and_display_frames() { + let Some(g) = gpu() else { return }; + let s = blend_scene(BitDepth::Bpc8, BlendMode::Screen); + // Auto follows the project's Mercury GPU / Software Only setting; Cpu never uses the GPU. + let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Auto, ..opts() }); + r.accel = Some(g); + assert!(r.active_accel().is_some()); + let mut p2 = s.p.clone(); + p2.settings.gpu_acceleration = false; + let mut r2 = Renderer::new(&p2, &Pattern, RenderOpts { backend: Backend::Auto, ..opts() }); + r2.accel = Some(g); + assert!(r2.active_accel().is_none()); + let mut r3 = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Cpu, ..opts() }); + r3.accel = Some(g); + assert!(r3.active_accel().is_none()); + // The display texture holds what the CPU viewer path shows (8-bit premultiplied). + let f = g.render_display(&r, s.cid, Tick::ZERO).expect("display frame"); + let bytes = g.read_display(&f).expect("readback"); + let cpu = Renderer::new(&s.p, &Pattern, opts()).comp_frame_cpu(s.cid, Tick::ZERO); + for (p, q) in cpu.data.iter().zip(bytes.chunks_exact(4)) { + let a = p[3].clamp(0.0, 1.0); + let want = [p[0].clamp(0.0, a), p[1].clamp(0.0, a), p[2].clamp(0.0, a), a].map(|v| (v * 255.0 + 0.5) as u8); + for k in 0..4 { + assert!((want[k] as i32 - q[k] as i32).abs() <= 1, "{want:?} vs {q:?}"); + } + } +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs new file mode 100644 index 000000000..d43c7b1de --- /dev/null +++ b/crates/gpu/src/walk.rs @@ -0,0 +1,188 @@ +//! The per-frame compositing walk on the GPU: the same steps as `Renderer::draw_comp` (layer +//! order, 3D runs, collapsed precomps, track mattes, Preserve Transparency, layer styles, motion +//! blur, bit-depth quantisation, colour conversions), with layer pixels taken from the CPU +//! renderer and its layer cache. Steps without a GPU implementation (3D runs, adjustment and +//! wireframe layers) read the canvas back, run on the CPU and upload the result. + +use std::sync::Arc; + +use effectcraft_color::BlendMode; +use effectcraft_effects::Buf; +use effectcraft_project::{ItemId, Layer, Quality}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, Renderer, styles}; +use effectcraft_time::Tick; + +use crate::context::{Enc, GpuImage}; +use crate::ops; + +/// Render a comp frame into a GPU image (the canvas in the output space, as +/// `Renderer::comp_frame_cpu` returns it). `None` when something cannot run here (the caller +/// renders on the CPU instead). +pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick) -> Option { + let ctx = r.eval_ctx(comp_id, t)?; + let s = r.opts.scale; + let w = ((ctx.comp.width as f64 * s).round() as u32).max(1); + let h = ((ctx.comp.height as f64 * s).round() as u32).max(1); + if r.depth() > 16 || !e.g.fits(w, h) { + return None; + } + let mut canvas = e.image(w, h); + draw_comp(e, r, &ctx, &mut canvas)?; + let pipe = r.pipe(); + if let Some(c) = pipe.from_blend() { + canvas = ops::convert(e, &canvas, &c); + } + if let Some(l) = pipe.levels { + canvas = ops::quantize(e, &canvas, l); + } + if r.depth() == 0 + && let Some(c) = pipe.output() + { + canvas = ops::convert(e, &canvas, &c); + if let Some(l) = pipe.levels { + canvas = ops::quantize(e, &canvas, l); + } + } + Some(canvas) +} + +/// Run a CPU drawing step on the canvas (read back, draw, upload). +fn on_cpu(e: &mut Enc, canvas: &mut GpuImage, f: impl FnOnce(&mut Image)) -> Option<()> { + let mut img = e.download(canvas)?; + f(&mut img); + *canvas = e.g.upload_image(&img)?; + Some(()) +} + +fn draw_comp<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, canvas: &mut GpuImage) -> Option<()> { + let visible = r.visible_layers(ctx); + let levels = r.pipe().levels; + let mut i = 0; + while i < visible.len() { + if visible[i].is_3d() { + let mut j = i; + while j < visible.len() && visible[j].is_3d() { + j += 1; + } + on_cpu(e, canvas, |img| r.draw_3d_run(ctx, &visible[i..j], img))?; + i = j; + } else { + match r.collapsed(ctx, visible[i]) { + Some(item) => draw_collapsed(e, r, ctx, visible[i], item, canvas)?, + None => draw_layer(e, r, ctx, visible[i], canvas)?, + } + i += 1; + } + // 8/16 bpc: the comp is an integer buffer after every layer. + if let Some(l) = levels { + *canvas = ops::quantize(e, canvas, l); + } + } + Some(()) +} + +fn draw_collapsed<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, layer: &Layer, item: ItemId, canvas: &mut GpuImage) -> Option<()> { + let opacity = r.layer_opacity(ctx, layer); + if opacity <= 0.0 { + return Some(()); + } + let matte = r.track_matte(ctx, layer).is_some(); + if layer.blend_mode == BlendMode::Normal && !matte && !layer.preserve_transparency { + if let Some((sub, nctx)) = r.collapse_into(ctx, layer, item, opacity) { + draw_comp(e, &sub, &nctx, canvas)?; + } + return Some(()); + } + let Some((sub, nctx)) = r.collapse_into(ctx, layer, item, 1.0) else { return Some(()) }; + let mut iso = e.image(canvas.width, canvas.height); + draw_comp(e, &sub, &nctx, &mut iso)?; + composite_iso(e, r, ctx, layer, iso, canvas, opacity) +} + +fn draw_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut GpuImage) -> Option<()> { + if layer.switches.adjustment || layer.switches.quality == Quality::Wireframe { + // Adjustment layers run their effects on the comp below (CPU, with GPU effects where + // available); wireframes draw outlines. + return on_cpu(e, canvas, |img| r.draw_layer_cpu(ctx, layer, img)); + } + let opacity = r.layer_opacity(ctx, layer); + if opacity <= 0.0 { + return Some(()); + } + let Some(st) = r.styled_layer(ctx, layer) else { return Some(()) }; + if st.plain { + return composite_layer(e, r, ctx, layer, &st.body, canvas, opacity); + } + // Layer styles: exterior passes with their own modes, then the body; the layer's opacity + // fades the whole stack; knockout and channel switches as on the CPU. + let bl = styles::blending(ctx, layer); + let (w, h) = (canvas.width, canvas.height); + if r.track_matte(ctx, layer).is_some() || layer.preserve_transparency { + let mut iso = e.image(w, h); + for (b, m) in &st.passes { + iso = place(e, r, ctx, layer, b, &iso, *m, 1.0)?; + } + iso = place(e, r, ctx, layer, &st.body, &iso, BlendMode::Normal, 1.0)?; + return composite_iso(e, r, ctx, layer, iso, canvas, opacity); + } + let mut tmp = canvas.clone(); + if bl.knockout > 0 { + let k = place(e, r, ctx, layer, &st.content, &e.image(w, h), BlendMode::Normal, 1.0)?; + tmp = ops::knockout(e, &tmp, &k); + } + for (b, m) in &st.passes { + tmp = place(e, r, ctx, layer, b, &tmp, *m, 1.0)?; + } + tmp = place(e, r, ctx, layer, &st.body, &tmp, layer.blend_mode, 1.0)?; + *canvas = ops::channel_mix(e, canvas, &tmp, bl.channels, opacity); + Some(()) +} + +fn composite_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, buf: &Arc, canvas: &mut GpuImage, opacity: f32) -> Option<()> { + let matte = r.track_matte(ctx, layer).is_some(); + if !matte && !layer.preserve_transparency { + *canvas = place(e, r, ctx, layer, buf, canvas, layer.blend_mode, opacity)?; + return Some(()); + } + // Render in isolation, then matte / preserve transparency, then blend. + let iso = place(e, r, ctx, layer, buf, &e.image(canvas.width, canvas.height), BlendMode::Normal, 1.0)?; + composite_iso(e, r, ctx, layer, iso, canvas, opacity) +} + +/// Track matte / Preserve Transparency on an isolated layer render, then blend. +fn composite_iso(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, mut iso: GpuImage, canvas: &mut GpuImage, opacity: f32) -> Option<()> { + if let Some((m, kind)) = r.track_matte(ctx, layer) { + let mut mimg = e.image(canvas.width, canvas.height); + if m.is_active_at(ctx.time) + && let Some(mb) = r.layer_buf(ctx, m) + { + let mo = ctx.opacity(m) as f32; + mimg = place(e, r, ctx, m, &mb, &mimg, BlendMode::Normal, mo)?; + } + iso = ops::matte(e, &iso, &mimg, kind); + } + if layer.preserve_transparency { + iso = ops::preserve(e, &iso, canvas); + } + *canvas = ops::blend_full(e, canvas, &iso, layer.blend_mode, opacity, layer.id.0 as u32); + Some(()) +} + +/// Draw a processed layer buffer onto `target` (transform, motion blur sub-samples). +fn place(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, buf: &Arc, target: &GpuImage, mode: BlendMode, opacity: f32) -> Option { + if buf.img.is_empty() || opacity <= 0.0 { + return Some(target.clone()); + } + let src = e.g.upload_buf(buf)?; + let pl = r.placement(ctx, layer, buf); + if let [m] = pl.matrices.as_slice() { + return Some(ops::warp(e, target, &src, m, pl.sampling, mode, opacity, pl.seed, None)); + } + let mut acc = e.image(target.width, target.height); + let k = 1.0 / pl.matrices.len() as f32; + for m in &pl.matrices { + acc = ops::warp(e, &acc, &src, m, pl.sampling, BlendMode::Normal, 1.0, 0, Some(k)); + } + Some(ops::blend_full(e, target, &acc, mode, opacity, pl.seed)) +} diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index df563fc71..16199dfbe 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -32,6 +32,7 @@ impl Exporter for FileExporter { project: job.project, footage: job.footage, expr: job.expr, + accel: job.accel, comp: job.item.comp, settings: &job.item.settings, output: &job.item.output, diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 88fc6d02a..d6bbf5e48 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -113,6 +113,10 @@ pub struct ProjectSettings { /// Frame numbering starts at 0 (or 1). pub frame_start: i64, pub audio_sample_rate: u32, + /// Video Rendering and Effects ▸ Use: Mercury GPU Acceleration (`true`, the default; used + /// when a GPU adapter exists) or Mercury Software Only (`false`, the CPU compositor). + #[serde(default = "yes")] + pub gpu_acceleration: bool, } impl Default for ProjectSettings { @@ -125,6 +129,7 @@ impl Default for ProjectSettings { time_display: TimeDisplayStyle::Timecode, frame_start: 0, audio_sample_rate: 48_000, + gpu_acceleration: true, } } } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 05c05efa5..31df59663 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -128,6 +128,53 @@ impl FootageSource for NoFootage { } } +/// Which compositor renders a frame (Project Settings ▸ Video Rendering and Effects). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Backend { + /// The CPU reference compositor (Mercury Software Only). + #[default] + Cpu, + /// The GPU compositor whenever an [`Accelerator`] is attached, whatever the project says. + Gpu, + /// The GPU compositor when an [`Accelerator`] is attached and the project's renderer is + /// Mercury GPU Acceleration (the viewer's choice); the CPU otherwise. + Auto, +} + +impl Backend { + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().as_str() { + "cpu" | "software" | "mercury software only" => Some(Backend::Cpu), + "gpu" | "mercury gpu acceleration" => Some(Backend::Gpu), + "auto" => Some(Backend::Auto), + _ => None, + } + } +} + +/// One effect of a GPU effect chain (see [`Accelerator::effects`]). +pub struct FxStep<'x> { + pub spec: &'static effectcraft_effects::EffectSpec, + pub ctx: EffectCtx<'x>, +} + +/// A GPU (or other hardware) backend for the compositor. The CPU [`Renderer`] stays the +/// reference: an accelerator renders what it supports and returns `None` for the rest, which +/// then runs on the CPU. Implemented by `effectcraft-gpu`. +pub trait Accelerator: Send + Sync { + /// Adapter / backend name for status readouts ("Apple M2 (Metal)"). + fn name(&self) -> String; + /// Render a top-level comp frame (what [`Renderer::comp_frame`] returns). `None` = not + /// handled (the CPU renders it). + fn comp_frame(&self, r: &Renderer, comp: ItemId, t: Tick) -> Option; + /// The accelerator implements effect `id` with the CPU effect's semantics. + fn supports_effect(&self, id: &str) -> bool; + /// Run a chain of supported effects on `buf`, clamping and quantising to `levels` after + /// every effect when set (8/16 bpc). `None` = not handled. + fn effects(&self, chain: &[FxStep], buf: &Buf, levels: Option) -> Option; +} + #[derive(Clone, Copy, Debug)] pub struct RenderOpts { /// Output scale relative to comp pixels (1 = Full, 0.5 = Half…). @@ -139,11 +186,13 @@ pub struct RenderOpts { pub draft: bool, /// 3D view camera override (viewer Front/Top/Custom views); None = the comp's active camera. pub view: Option, + /// CPU or GPU compositor (the GPU needs [`Renderer::accel`]). + pub backend: Backend, } impl Default for RenderOpts { fn default() -> Self { - RenderOpts { scale: 1.0, motion_blur: true, guides: false, draft: false, view: None } + RenderOpts { scale: 1.0, motion_blur: true, guides: false, draft: false, view: None, backend: Backend::Cpu } } } @@ -156,6 +205,8 @@ pub struct Renderer<'a> { pub cache: Option<&'a LayerCache>, /// Optional per-layer timing sink (`frame --bench` / perf readouts). pub profile: Option<&'a std::sync::Mutex>>, + /// GPU backend used when [`RenderOpts::backend`] asks for it. + pub accel: Option<&'a dyn Accelerator>, /// Current nesting depth (precomp recursion guard). depth: usize, /// The project's colour pipeline (bit depth, working space, linear blending). @@ -204,6 +255,7 @@ impl<'a> Renderer<'a> { opts, cache: None, profile: None, + accel: None, depth: 0, pipe: color::Pipe::of(&project.settings), outer: None, @@ -220,7 +272,7 @@ impl<'a> Renderer<'a> { /// The nested comp of a precomp layer whose transformations collapse into this comp: the /// Collapse Transformations switch is on and the layer has no masks, effects or layer /// styles (those force a flattened render of the precomp, as in After Effects). - pub(crate) fn collapsed(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + pub fn collapsed(&self, ctx: &EvalCtx, layer: &Layer) -> Option { if !layer.switches.collapse || layer.switches.adjustment || self.depth > 12 { return None; } @@ -244,7 +296,7 @@ impl<'a> Renderer<'a> { /// their transforms are concatenated with the precomp layer's (one resample, no /// rasterisation at the precomp's bounds), their opacity multiplied by `opacity`, and their /// 3D layers use this comp's camera and lights. - pub(crate) fn collapse_into(&self, ctx: &EvalCtx<'a>, layer: &Layer, item: ItemId, opacity: f32) -> Option<(Renderer<'a>, EvalCtx<'a>)> { + pub fn collapse_into(&self, ctx: &EvalCtx<'a>, layer: &Layer, item: ItemId, opacity: f32) -> Option<(Renderer<'a>, EvalCtx<'a>)> { let nc = self.project.comp(item)?; let nctx = EvalCtx { comp_id: item, comp: nc, time: ctx.source_time(layer), ..*ctx }; let (l2c, _) = ctx.layer_to_comp(layer); @@ -336,8 +388,30 @@ impl<'a> Renderer<'a> { EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr } } + /// The accelerator to use for this render, if any (see [`Backend`]). + pub fn active_accel(&self) -> Option<&'a dyn Accelerator> { + let a = self.accel?; + match self.opts.backend { + Backend::Cpu => None, + Backend::Gpu => Some(a), + Backend::Auto => self.project.settings.gpu_acceleration.then_some(a), + } + } + /// Render a composition at comp time `t` (transparent background, comp size × scale). + /// Top-level frames go to the GPU when [`Self::active_accel`] is set and handles them. pub fn comp_frame(&self, comp_id: ItemId, t: Tick) -> Image { + if self.depth == 0 + && let Some(a) = self.active_accel() + && let Some(img) = a.comp_frame(self, comp_id, t) + { + return img; + } + self.comp_frame_cpu(comp_id, t) + } + + /// [`Self::comp_frame`] on the CPU compositor only. + pub fn comp_frame_cpu(&self, comp_id: ItemId, t: Tick) -> Image { let Some(comp) = self.project.comp(comp_id) else { return Image::new(1, 1) }; let s = self.opts.scale; let w = ((comp.width as f64 * s).round() as u32).max(1); @@ -364,15 +438,8 @@ impl<'a> Renderer<'a> { /// Draw a comp's layers (bottom to top) into `canvas` (blending space). fn draw_comp(&self, ctx: &EvalCtx<'a>, canvas: &mut Image) { - let (comp, t) = (ctx.comp, ctx.time); - let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.source.is_av() && l.is_active_at(t)); // Bottom-to-top, with runs of consecutive 3D layers depth-sorted (farthest first). - let visible: Vec<&Layer> = comp - .layers - .iter() - .rev() - .filter(|l| l.is_active_at(t) && l.has_video() && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) - .collect(); + let visible = self.visible_layers(ctx); let mut i = 0; while i < visible.len() { if visible[i].is_3d() { @@ -425,18 +492,49 @@ impl<'a> Renderer<'a> { let host = FxHost { r: self, ctx, layer, index: Default::default() }; let env = EffectEnv { masks: &mask_shapes, host: Some(&host), comp_time: ctx.time.seconds(), frame_rate: ctx.comp.frame_rate.as_f64(), effect_index: 0 }; - for (i, g) in fx.groups().enumerate() { - if i >= limit { - break; - } - if !g.enabled { - continue; - } - let GroupKind::Effect { effect } = &g.kind else { continue }; - let Some(spec) = effectcraft_effects::find(effect) else { continue }; - if effectcraft_effects::audio_fx::is_audio_effect(spec.id) { - continue; + // Video effects in stack order (index, group, spec); disabled and audio effects skipped. + let stack: Vec<(usize, &effectcraft_project::PropGroup, &'static effectcraft_effects::EffectSpec)> = fx + .groups() + .enumerate() + .take_while(|(i, _)| *i < limit) + .filter(|(_, g)| g.enabled) + .filter_map(|(i, g)| match &g.kind { + GroupKind::Effect { effect } => effectcraft_effects::find(effect).map(|s| (i, g, s)), + _ => None, + }) + .filter(|(_, _, s)| !effectcraft_effects::audio_fx::is_audio_effect(s.id)) + .collect(); + let accel = self.active_accel(); + let mut k = 0; + while k < stack.len() { + let (i, g, spec) = stack[k]; + // A run of GPU effects goes to the accelerator in one upload / readback. + if let Some(a) = accel + && spec.gpu + && a.supports_effect(spec.id) + { + let run: Vec<_> = stack[k..].iter().take_while(|(_, _, s)| s.gpu && a.supports_effect(s.id)).collect(); + let params: Vec = run.iter().map(|(_, g, _)| self.effect_params(ctx, layer, g)).collect(); + let steps: Vec = run + .iter() + .zip(¶ms) + .map(|((i, g, s), params)| FxStep { + spec: s, + ctx: EffectCtx { params, time: lt.seconds(), layer_size, seed: g.uid as u32, adjustment, env: EffectEnv { effect_index: *i, ..env } }, + }) + .collect(); + let t0 = web_time::Instant::now(); + if let Some(out) = a.effects(&steps, &buf, self.pipe.levels) { + buf = out; + if let Some(v) = timing.as_deref_mut() { + let ms = t0.elapsed().as_secs_f64() * 1e3 / run.len() as f64; + v.extend(run.iter().map(|(_, _, s)| (format!("{} (gpu)", s.id), ms))); + } + k += run.len(); + continue; + } } + k += 1; let params = self.effect_params(ctx, layer, g); host.index.store(i, std::sync::atomic::Ordering::Relaxed); let env = EffectEnv { effect_index: i, ..env }; @@ -574,7 +672,7 @@ impl<'a> Renderer<'a> { } /// [`Self::layer_buf`] in the blending space (linear with Blend Colors Using 1.0 Gamma). - pub(crate) fn blend_layer_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option> { + pub fn blend_layer_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option> { let (b, k) = self.layer_buf_keyed(ctx, layer)?; Some(self.to_blend(b, k)) } @@ -733,32 +831,45 @@ impl<'a> Renderer<'a> { Mat3::scale(vec2(s, s)) * l2c * Mat3::scale(vec2(1.0 / buf.scale, 1.0 / buf.scale)) * Mat3::translate(vec2(-buf.offset[0], -buf.offset[1])) } - /// Draw a processed layer into `target` (transform, motion blur) with `mode`/`opacity`. - fn place(&self, ctx: &EvalCtx, layer: &Layer, buf: &Buf, target: &mut Image, mode: BlendMode, opacity: f32) { + /// How a processed layer buffer lands in the output: one buffer → output matrix, or one per + /// motion-blur sub-sample (averaged), with the layer's sampling filter and dissolve seed. + pub fn placement(&self, ctx: &EvalCtx, layer: &Layer, buf: &Buf) -> Placement { let samples = if self.opts.motion_blur && ctx.comp.enable_motion_blur && layer.switches.motion_blur { if self.opts.draft { 4 } else { ctx.comp.motion_blur_samples.clamp(2, 64) as usize } } else { 1 }; - let opts = WarpOpts { sampling: self.sampling(layer), opacity, mode, seed: layer.id.0 as u32, clip: None }; + let (sampling, seed) = (self.sampling(layer), layer.id.0 as u32); if samples <= 1 { - let m = self.buf_matrix(ctx, layer, buf); - composite_warp(target, &buf.img, &m, &opts); - return; + return Placement { matrices: vec![self.buf_matrix(ctx, layer, buf)], sampling, seed }; } let fd = ctx.comp.frame_duration().seconds(); let angle = ctx.comp.shutter_angle / 360.0; let phase = ctx.comp.shutter_phase / 360.0; + let matrices = (0..samples) + .map(|i| { + let f = phase + angle * i as f64 / (samples - 1) as f64; + self.buf_matrix(&ctx.at(ctx.time + Tick::from_seconds_f64(f * fd)), layer, buf) + }) + .collect(); + Placement { matrices, sampling, seed } + } + + /// Draw a processed layer into `target` (transform, motion blur) with `mode`/`opacity`. + fn place(&self, ctx: &EvalCtx, layer: &Layer, buf: &Buf, target: &mut Image, mode: BlendMode, opacity: f32) { + let pl = self.placement(ctx, layer, buf); + let opts = WarpOpts { sampling: pl.sampling, opacity, mode, seed: pl.seed, clip: None }; + if let [m] = pl.matrices.as_slice() { + composite_warp(target, &buf.img, m, &opts); + return; + } // Sub-samples sum straight into one buffer (each warp is row-parallel). let mut acc = Image::new(target.width, target.height); - let k = 1.0 / samples as f32; - for i in 0..samples { - let f = phase + angle * i as f64 / (samples - 1) as f64; - let sub = ctx.at(ctx.time + Tick::from_seconds_f64(f * fd)); - let m = self.buf_matrix(&sub, layer, buf); - effectcraft_raster::accumulate_warp(&mut acc, &buf.img, &m, opts.sampling, k); + let k = 1.0 / pl.matrices.len() as f32; + for m in &pl.matrices { + effectcraft_raster::accumulate_warp(&mut acc, &buf.img, m, opts.sampling, k); } - target.blend_from(&acc, mode, opacity, layer.id.0 as u32); + target.blend_from(&acc, mode, opacity, pl.seed); } fn draw_layer(&self, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image) { @@ -906,18 +1017,99 @@ impl<'a> Renderer<'a> { } } -/// A layer ready to composite (see [`Renderer::styled`]). -struct StyledLayer { - passes: Vec<(Arc, BlendMode)>, - body: Arc, +/// A layer ready to composite (see [`Renderer::styled_layer`]). +pub struct StyledLayer { + /// Exterior style passes (Drop Shadow, Outer Glow) with their blend modes. + pub passes: Vec<(Arc, BlendMode)>, + pub body: Arc, /// Pre-style pixels (knockout shape). - content: Arc, + pub content: Arc, /// Styled cache key (the content key when plain). key: Option, /// Content cache key. ckey: Option, /// No layer styles: composite `body` the usual way. - plain: bool, + pub plain: bool, +} + +/// Where a layer buffer lands (see [`Renderer::placement`]). +#[derive(Clone, Debug)] +pub struct Placement { + /// Buffer pixel → output pixel; several = motion-blur sub-samples (equal weights). + pub matrices: Vec, + pub sampling: effectcraft_raster::Sampling, + /// Dissolve noise seed. + pub seed: u32, +} + +/// Hooks for accelerated compositors (`effectcraft-gpu`): the pieces of the CPU walk they reuse +/// or fall back to. The CPU compositor itself is [`Renderer::comp_frame_cpu`]. +impl<'a> Renderer<'a> { + /// The project's colour pipeline. + pub fn pipe(&self) -> color::Pipe { + self.pipe + } + + /// Nesting depth (0 = the top-level comp). + pub fn depth(&self) -> usize { + self.depth + } + + /// Evaluation context of comp `comp_id` at comp time `t`. + pub fn eval_ctx(&self, comp_id: ItemId, t: Tick) -> Option> { + Some(self.ctx(comp_id, self.project.comp(comp_id)?, t)) + } + + /// The layers drawn at the context time, bottom to top (solo, guides and visibility + /// applied). + pub fn visible_layers(&self, ctx: &EvalCtx<'a>) -> Vec<&'a Layer> { + let (comp, t) = (ctx.comp, ctx.time); + let any_solo = comp.layers.iter().any(|l| l.switches.solo && l.source.is_av() && l.is_active_at(t)); + comp.layers + .iter() + .rev() + .filter(|l| l.is_active_at(t) && l.has_video() && (!any_solo || l.switches.solo) && (self.opts.guides || !l.switches.guide)) + .collect() + } + + /// Draw a run of consecutive 3D layers into `canvas` (CPU). + pub fn draw_3d_run(&self, ctx: &EvalCtx<'a>, run: &[&Layer], canvas: &mut Image) { + three_d::compose::draw_run(self, ctx, run, canvas); + } + + /// Draw one 2D layer into `canvas` on the CPU (any kind: adjustment, wireframe, styled…). + pub fn draw_layer_cpu(&self, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image) { + self.draw_layer(ctx, layer, canvas); + } + + /// Draw a collapsed precomp layer into `canvas` on the CPU. + pub fn draw_collapsed_cpu(&self, ctx: &EvalCtx<'a>, layer: &Layer, item: ItemId, canvas: &mut Image) { + self.draw_collapsed(ctx, layer, item, canvas); + } + + /// The layer's finished buffers (styles included) in the blending space. + pub fn styled_layer(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + let mut timing = self.profile.map(|_| LayerTiming { depth: self.depth, layer: layer.name.clone(), ..Default::default() }); + let t0 = web_time::Instant::now(); + let st = self.styled(ctx, layer, timing.as_mut()).map(|st| self.styled_to_blend(st)); + if let (Some(prof), Some(mut timing)) = (self.profile, timing) { + timing.process_ms = t0.elapsed().as_secs_f64() * 1e3; + if let Ok(mut v) = prof.lock() { + v.push(timing); + } + } + st + } + + /// Opacity the layer composites with (its Opacity × collapsed precomps' opacity). + pub fn layer_opacity(&self, ctx: &EvalCtx, layer: &Layer) -> f32 { + ctx.opacity(layer) as f32 * self.opacity_mul + } + + /// The layer's track matte layer and kind, if it has a valid one. + pub fn track_matte<'c>(&self, ctx: &EvalCtx<'c>, layer: &Layer) -> Option<(&'c Layer, MatteKind)> { + layer.track_matte.and_then(|tm| ctx.comp.layer(tm.layer).filter(|m| m.id != layer.id).map(|m| (m, tm.kind))) + } } /// The layer's frame blending mode, if the comp's Enable Frame Blending switch is on. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 7627c0963..9a0324429 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -319,7 +319,14 @@ impl EffectcraftApp { let Some(c) = self.session.project.comp(comp) else { return }; let key = self.frame_key(comp, frame, scale); let t = c.frame_rate.tick_of(frame); - let opts = RenderOpts { scale, motion_blur: true, guides: true, draft: self.ui.viewer.fast_preview, view: self.session.view_camera(comp) }; + let opts = RenderOpts { + scale, + motion_blur: true, + guides: true, + draft: self.ui.viewer.fast_preview, + view: self.session.view_camera(comp), + backend: effectcraft_engine::render::Backend::Auto, + }; if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { diff --git a/docs/architecture.md b/docs/architecture.md index dc95f35b5..c3d478e3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -27,6 +27,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | | L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | | L4 | `host` | A fully wired `Session` (media, expressions, exporter) for the frontends | @@ -35,7 +36,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path`, -`media / expr / export → render`, `export → media`, `lottie → format`, `host → engine`. +`media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine`. ## 2. Time @@ -103,6 +104,27 @@ groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `pup Half, Third and Quarter resolution render proportionally fewer pixels end to end. +**GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU +Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the +reference and keeps rendering layer content (sources, masks, CPU effects, layer styles) into the +layer cache. The render crate defines an `Accelerator` trait; `effectcraft_gpu::Gpu` implements it +on wgpu compute shaders. `RenderOpts::backend` picks `Cpu`, `Gpu` or `Auto` (GPU when an +accelerator is attached and the project's renderer is the GPU). A GPU frame walks the comp like +`draw_comp`: cached layer buffers are uploaded once per buffer, then transformed with the CPU's +sampling (nearest, bilinear, Catmull-Rom bicubic, the same minification pre-filter), motion-blur +sub-samples are accumulated, and track mattes, Preserve Transparency, layer style passes, +knockout, all 38 blend modes (a WGSL port of `color::blend`), the 8/16 bpc clamp-and-quantise steps +and colour-space conversions run on the GPU. 3D runs, adjustment layers and wireframes run on the +CPU between GPU steps (read back, draw, upload). GPU effects (`effects::GPU_EFFECTS`: Gaussian, +Fast Box and Directional Blur, Glow, Levels, Curves, Hue/Saturation, Tint, Fill, Gradient Ramp, +Fractal Noise, Drop Shadow, Brightness & Contrast, Exposure, Invert, Transform) repeat the CPU +effect's steps (padding, box radii, parameters) as kernels; consecutive GPU effects run as one +chain with one upload and one readback. Tests render scenes on both paths and compare them +(≤ 1/255 at 8 bpc, ≤ 1e-3 at 32 bpc); they skip without an adapter. The desktop viewer builds the +`Gpu` on egui-wgpu's device and shows frames from GPU textures without reading them back +(`ui-egui::frames`); headless renders, the CLI (unless `--gpu`) and CI use the CPU. On the web +(WebGPU) the GPU composites viewer frames; steps that need a readback fall back to the CPU. + **Colour and bit depth** (`crates/render/src/color.rs`, `crates/color/src/space.rs`). Pixels are `f32`, but 8 and 16 bpc projects clamp and quantise each layer after its source and masks and after every effect, and the comp after every layer (16 bpc uses 0..32768), so over-range values diff --git a/docs/parity.md b/docs/parity.md index df6b963a0..31e73c875 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -69,11 +69,11 @@ rendering, on-canvas text editing) are where the conservative figure is most lik | Text | 63% | 5.8 | on-canvas editing with a cursor, per-character styles, paragraph spacing and indents | | Web | 60% | 2.5 | threads, browser storage, audio | | 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | -| Effects | 56% | 14.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU effects, 41 missing effects | +| Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, viewer rulers/snapping/channels/snapshots, preferences, native macOS menus | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 44% | 13.0 | mask tracking, Roto Brush, free transform and RotoBezier | -| Preview | 43% | 8.1 | GPU compositor, disk cache, region of interest, snapshots and exposure | +| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache, region of interest, snapshots and exposure | | Tracking | 0% | 13.0 | point tracker, Warp Stabilizer, mask and face tracking | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -90,7 +90,8 @@ Mocha and Cineware are third-party and not counted. 1. On-canvas text editing and per-character styles. 2. Effect Controls widgets: angle dial, point crosshair, eyedropper, curves and levels editors. 3. Viewer basics: snapping, rulers, channel view, snapshots, exposure, a drawable region of interest. -4. A GPU (wgpu) compositor, then GPU effects. +4. ~~A GPU (wgpu) compositor, then GPU effects~~ (done: 2D compositing and 16 GPU effects; 3D runs + and adjustment layers still composite on the CPU). 5. Pen tool for shape paths, shape vertex editing, the Layer viewer, free transform and RotoBezier. 6. Motion-path handles in the viewer; graph editor transform box and snapping. 7. Point tracking and stabilization (in progress). From af651a6f5697a43b9e6babee26a4991b8737e75c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 11:06:05 -0400 Subject: [PATCH 2/3] M12.2: viewer on gpu The desktop viewer builds the GPU compositor on egui-wgpu's device and, with Mercury GPU Acceleration, renders frames into RGBA8 textures that are registered with egui-wgpu and drawn directly (no readback); the RAM preview cache holds them under a video-memory budget. Pixels are read back only on demand (Info panel, eyedroppers, histograms). Project Settings gets the Video Rendering and Effects renderer choice. ui_gpu test: GPU viewer frame matches the CPU one (skips without an adapter). Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/Cargo.toml | 1 + crates/ui-egui/src/frames.rs | 101 +++++++++++++++---- crates/ui-egui/src/lib.rs | 66 +++++++++++- crates/ui-egui/src/panels/effect_controls.rs | 8 +- crates/ui-egui/src/panels/forms.rs | 11 ++ crates/ui-egui/src/panels/info.rs | 4 + crates/ui-egui/src/panels/misc.rs | 2 +- crates/ui-egui/src/panels/project.rs | 4 +- crates/ui-egui/src/panels/timeline.rs | 2 +- crates/ui-egui/src/panels/viewer.rs | 51 ++++++++-- crates/ui-egui/tests/ui_gpu.rs | 56 ++++++++++ 11 files changed, 269 insertions(+), 37 deletions(-) create mode 100644 crates/ui-egui/tests/ui_gpu.rs diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index d7c5ea7e3..31cd782c2 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-render = { workspace = true } effectcraft-engine = { workspace = true } +effectcraft-gpu = { workspace = true } egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit"] } serde = { workspace = true } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index a703620ad..3c3e5df14 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -6,6 +6,11 @@ //! //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. +//! +//! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set +//! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer +//! registers with egui-wgpu and draws directly; pixels are read back only when something needs +//! them (Info panel, eyedroppers, histograms). use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; @@ -13,8 +18,34 @@ use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; +use effectcraft_gpu::{DisplayFrame, Gpu}; use rayon::prelude::*; +/// GPU frames kept for RAM preview (bytes of video memory). +const GPU_BUDGET: usize = 1 << 30; + +/// A rendered viewer frame: CPU pixels or a GPU texture. +#[derive(Clone)] +pub enum FrameImage { + Cpu(Arc), + Gpu(Arc), +} + +impl FrameImage { + pub fn size(&self) -> [usize; 2] { + match self { + FrameImage::Cpu(c) => c.size, + FrameImage::Gpu(f) => [f.width as usize, f.height as usize], + } + } + fn bytes(&self) -> usize { + self.size()[0] * self.size()[1] * 4 + } + fn is_gpu(&self) -> bool { + matches!(self, FrameImage::Gpu(_)) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct FrameKey { pub revision: u64, @@ -27,26 +58,55 @@ pub struct FrameKey { } struct Cache { - map: HashMap>, + map: HashMap, order: VecDeque, bytes: usize, + /// Bytes of GPU frames (also counted in `bytes`). + gpu_bytes: usize, budget: usize, } impl Cache { - fn insert(&mut self, k: FrameKey, img: Arc) { - let sz = img.pixels.len() * 4; - if self.map.insert(k, img).is_none() { + fn insert(&mut self, k: FrameKey, img: FrameImage) { + let (sz, gpu) = (img.bytes(), img.is_gpu()); + if let Some(old) = self.map.insert(k, img) { + self.forget(&old); + } else { self.order.push_back(k); - self.bytes += sz; + } + self.bytes += sz; + if gpu { + self.gpu_bytes += sz; } while self.bytes > self.budget { let Some(old) = self.order.pop_front() else { break }; - if let Some(i) = self.map.remove(&old) { - self.bytes -= i.pixels.len() * 4; + self.remove(&old); + } + // Video memory: drop the oldest GPU frames past their own budget. + while self.gpu_bytes > GPU_BUDGET { + let Some(i) = self.order.iter().position(|k| self.map.get(k).is_some_and(FrameImage::is_gpu)) else { break }; + if let Some(old) = self.order.remove(i) { + self.remove(&old); } } } + fn forget(&mut self, img: &FrameImage) { + self.bytes -= img.bytes(); + if img.is_gpu() { + self.gpu_bytes -= img.bytes(); + } + } + fn remove(&mut self, k: &FrameKey) { + if let Some(i) = self.map.remove(k) { + self.forget(&i); + } + } + fn evict_to_budget(&mut self) { + while self.bytes > self.budget { + let Some(old) = self.order.pop_front() else { break }; + self.remove(&old); + } + } } /// What a render job needs (all cheap clones). @@ -56,6 +116,8 @@ pub struct RenderSource { pub footage: Arc, pub expr: Option>, pub layer_cache: Arc, + /// GPU compositor on the viewer's device (Mercury GPU Acceleration), if available. + pub gpu: Option, } pub struct Frames { @@ -76,7 +138,7 @@ impl Default for Frames { #[cfg(not(target_arch = "wasm32"))] let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 16); Frames { - cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, budget: 3 << 30 })), + cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, gpu_bytes: 0, budget: 3 << 30 })), inflight: Arc::new(Mutex::new(HashSet::new())), queue: Arc::new(Mutex::new(Queue::default())), #[cfg(not(target_arch = "wasm32"))] @@ -109,7 +171,7 @@ impl Frames { } } - pub fn get(&self, k: &FrameKey) -> Option> { + pub fn get(&self, k: &FrameKey) -> Option { self.cache.lock().ok()?.map.get(k).cloned() } @@ -134,12 +196,7 @@ impl Frames { pub fn set_budget(&self, bytes: usize) { if let Ok(mut c) = self.cache.lock() { c.budget = bytes; - while c.bytes > c.budget { - let Some(old) = c.order.pop_front() else { break }; - if let Some(i) = c.map.remove(&old) { - c.bytes -= i.pixels.len() * 4; - } - } + c.evict_to_budget(); } } @@ -152,6 +209,7 @@ impl Frames { c.map.clear(); c.order.clear(); c.bytes = 0; + c.gpu_bytes = 0; } } @@ -266,8 +324,17 @@ impl Worker { let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); r.expr = job.src.expr.as_deref(); r.cache = Some(&job.src.layer_cache); - let img = r.comp_frame(job.comp, job.t); - let ci = Arc::new(to_color_image(&img)); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) => g.render_display(&r, job.comp, job.t), + _ => None, + }; + let ci = match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + }; if job.urgent && let Ok(mut l) = last.lock() { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 9a0324429..5e0b67c65 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -118,10 +118,20 @@ pub struct EffectcraftApp { next_token: u64, pub(crate) last_ui_time: f64, pub(crate) toast: Option<(String, f64)>, - /// Texture of the frame shown in the viewer and the key it came from. + /// Texture of the last CPU frame shown in the viewer and the key it came from. pub(crate) viewer_tex: Option<(egui::TextureHandle, FrameKey)>, - /// Last displayed image (for the Info panel colour picker). + /// The last GPU frame shown: its egui texture id (registered with egui-wgpu), key and texture. + pub(crate) viewer_native: Option<(egui::TextureId, FrameKey, Arc)>, + /// The texture the viewer draws and its frame key (CPU or GPU frame). + pub(crate) viewer_shown: Option<(egui::TextureId, FrameKey)>, + /// Last displayed image (Info panel colour, eyedroppers, histograms). GPU frames fill it on + /// demand ([`EffectcraftApp::viewer_pixels`]). pub(crate) viewer_image: Option>, + /// egui-wgpu's device, once the first frame arrives (desktop and WebGPU). + pub(crate) wgpu: Option, + /// The GPU compositor on that device (None: no usable adapter → CPU only). + pub(crate) gpu: Option, + gpu_checked: bool, /// Commands from the native menu bar. pub command_inbox: Option>, /// Pointer position in comp pixels (Info panel). @@ -167,7 +177,12 @@ impl EffectcraftApp { last_ui_time: 0.0, toast: None, viewer_tex: None, + viewer_native: None, + viewer_shown: None, viewer_image: None, + wgpu: None, + gpu: None, + gpu_checked: false, command_inbox: None, pointer_comp: None, dialog_state: Default::default(), @@ -272,9 +287,53 @@ impl EffectcraftApp { footage: self.session.footage.clone(), expr: self.session.expr.clone(), layer_cache: self.session.layer_cache.clone(), + gpu: self.gpu.clone(), } } + /// Set up the GPU compositor on egui-wgpu's device (once). Without an adapter that runs + /// compute shaders (WebGL2, software GL) everything stays on the CPU. + pub(crate) fn init_gpu(&mut self, frame: &eframe::Frame) { + if self.gpu_checked { + return; + } + self.gpu_checked = true; + let Some(rs) = frame.wgpu_render_state() else { return }; + match effectcraft_gpu::Gpu::new(&rs.adapter, rs.device.clone(), rs.queue.clone()) { + Ok(g) => { + log::info!("GPU compositor: {}", effectcraft_engine::render::Accelerator::name(&g)); + self.session.accel = Some(Arc::new(g.clone())); + self.gpu = Some(g); + self.wgpu = Some(rs.clone()); + } + Err(e) => log::info!("GPU compositor unavailable: {e}"), + } + } + + /// The viewer shows a frame composited on the GPU (drawn from its wgpu texture). + pub fn viewer_on_gpu(&self) -> bool { + matches!((&self.viewer_shown, &self.viewer_native), (Some(s), Some(n)) if s.1 == n.1) + } + + /// The GPU compositor's adapter, when the viewer has one. + pub fn gpu_adapter(&self) -> Option { + self.gpu.as_ref().map(effectcraft_engine::render::Accelerator::name) + } + + /// The viewer's current pixels as 8-bit premultiplied RGBA, reading a GPU frame back the + /// first time something asks for it. + pub fn viewer_pixels(&mut self) -> Option> { + if self.viewer_image.is_none() + && let (Some((_, _, f)), Some(g)) = (&self.viewer_native, &self.gpu) + && self.viewer_shown.as_ref().map(|s| s.1) == self.viewer_native.as_ref().map(|n| n.1) + && let Some(bytes) = g.read_display(f) + { + let px = bytes.chunks_exact(4).map(|c| egui::Color32::from_rgba_premultiplied(c[0], c[1], c[2], c[3])).collect(); + self.viewer_image = Some(Arc::new(egui::ColorImage::new([f.width as usize, f.height as usize], px))); + } + self.viewer_image.clone() + } + /// The render scale used by the viewer right now. pub fn viewer_scale(&self, zoom: f32, ppp: f32) -> f64 { if self.playback.playing && self.ui.viewer.res == state::Resolution::Auto { @@ -811,7 +870,8 @@ impl eframe::App for EffectcraftApp { self.collect_screenshots(ctx); } - fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { + self.init_gpu(frame); if !self.fonts_ready { ui.ctx().request_repaint(); return; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index eda2ed763..78facbcc9 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -528,8 +528,8 @@ fn curves_size(width: f32) -> f32 { } /// Cached histogram of the displayed frame (recomputed when the frame changes). -fn frame_histogram(app: &EffectcraftApp, ctx: &egui::Context) -> Option> { - let img = app.viewer_image.clone()?; +fn frame_histogram(app: &mut EffectcraftApp, ctx: &egui::Context) -> Option> { + let img = app.viewer_pixels()?; let key = std::sync::Arc::as_ptr(&img) as usize; let id = egui::Id::new("ec-histogram"); if let Some((k, h)) = ctx.data(|d| d.get_temp::<(usize, std::sync::Arc<[[u32; 256]; 5]>)>(id)) @@ -969,6 +969,10 @@ pub fn viewer_hook( app.ui.fx_pick = None; return; }; + if pick.kind == "color" { + // GPU frames: read the shown frame back for sampling. + app.viewer_pixels(); + } let resp = ui.interact(map.area, egui::Id::new("viewer-fx-pick"), Sense::click()); app.auto.add("viewer.fxPick", map.area, &pick.name); let hint = if pick.kind == "point" { format!("Click to set {}", pick.name) } else { format!("Click to sample {}", pick.name) }; diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index e0f1a5f21..0f1edda6a 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -253,6 +253,10 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { } "file.projectSettings" if p.as_object().is_none_or(|m| m.is_empty()) => { let st = &s.project.settings; + let gpu_label = match &s.accel { + Some(a) => format!("Mercury GPU Acceleration ({})", a.name()), + None => "Mercury GPU Acceleration (no GPU: software)".to_string(), + }; let depth = match st.bit_depth.label() { l if l.starts_with("16") => 1, l if l.starts_with("32") => 2, @@ -287,6 +291,13 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ), Field::bool("linearize", "Linearize working space", st.linearize), Field::bool("blendLinear", "Blend colors using 1.0 gamma", st.blend_linear), + // Video Rendering and Effects ▸ Use. + Field::choice( + "renderer", + "Video rendering and effects", + &[(gpu_label.as_str(), json!("gpu")), ("Mercury Software Only", json!("software"))], + usize::from(!st.gpu_acceleration), + ), ], ) } diff --git a/crates/ui-egui/src/panels/info.rs b/crates/ui-egui/src/panels/info.rs index dd8354e6d..aeaeb3559 100644 --- a/crates/ui-egui/src/panels/info.rs +++ b/crates/ui-egui/src/panels/info.rs @@ -10,6 +10,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let p = ui.painter().with_clip_rect(rect); let x0 = rect.min.x + 12.0; let mut y = rect.min.y + 14.0; + if app.pointer_comp.is_some() { + // GPU frames: read the shown frame back for sampling. + app.viewer_pixels(); + } let (rgba, xy) = match (app.pointer_comp, &app.viewer_image) { (Some([cx, cy]), Some(img)) => { let comp = app.session.active_comp(); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 056e6acda..d54d9fdd2 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -69,7 +69,7 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { yy += 26.0; if let Some(c) = comp { let cid = app.session.active_comp_id().map(|i| i.0).unwrap_or(0); - let scale = app.viewer_tex.as_ref().map(|(_, k)| k.scale).unwrap_or(1000); + let scale = app.viewer_shown.as_ref().map(|(_, k)| k.scale).unwrap_or(1000); let cached = app.frames.cached_frames(app.session.revision, cid, scale).len(); let total = c.frame_rate.frame_at(c.work_area.1) - c.frame_rate.frame_at(c.work_area.0); let bar = Rect::from_min_size(pos2(rect.min.x + 12.0, yy), vec2(rect.width() - 24.0, 6.0)); diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index 31d7d4de1..d7d1cd9aa 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -82,10 +82,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { p.rect_filled(thumb.shrink(4.0), 2.0, Color32::from_rgb((s.color[0] * 255.0) as u8, (s.color[1] * 255.0) as u8, (s.color[2] * 255.0) as u8)); } ItemKind::Comp(_) => { - if let Some((tex, k)) = &app.viewer_tex + if let Some((tex, k)) = &app.viewer_shown && k.comp == it.id.0 { - p.image(tex.id(), thumb, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + p.image(*tex, thumb, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } else { icons::paint(&p, Rect::from_center_size(thumb.center(), vec2(24.0, 24.0)), Icon::Comp, t.text_dim); } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index be891090f..0d0b70230 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -579,7 +579,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } // Cache bar (green: cached frames at the viewer's resolution). - let scale_key = app.viewer_tex.as_ref().map(|(_, k)| k.scale).unwrap_or(1000); + let scale_key = app.viewer_shown.as_ref().map(|(_, k)| k.scale).unwrap_or(1000); let cached = app.frames.cached_frames(app.session.revision, cid.0, scale_key); let fd = comp.frame_duration().seconds(); let cy0 = ruler.min.y + 11.0; diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index a02c9b3a3..00f9a20cf 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -381,16 +381,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let key = app.frame_key(cid, frame, scale); app.request_frame_urgent(cid, frame, scale); if let Some(img) = app.frames.get(&key) { - let stale = app.viewer_tex.as_ref().is_none_or(|(_, k)| *k != key); + let stale = app.viewer_shown.as_ref().is_none_or(|(_, k)| *k != key); if stale { - match &mut app.viewer_tex { - Some((tex, k)) if tex.size() == img.size => { - tex.set((*img).clone(), egui::TextureOptions::LINEAR); - *k = key; - } - _ => app.viewer_tex = Some((ctx.load_texture("viewer-frame", (*img).clone(), egui::TextureOptions::LINEAR), key)), - } - app.viewer_image = Some(img); + show_frame(app, &ctx, key, img); } } if app.ui.viewer.transparency_grid { @@ -399,10 +392,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let bg = comp.background; painter.rect_filled(comp_rect, 0.0, Color32::from_rgb((bg[0] * 255.0) as u8, (bg[1] * 255.0) as u8, (bg[2] * 255.0) as u8)); } - if let Some((tex, k)) = &app.viewer_tex + if let Some((tex, k)) = &app.viewer_shown && k.comp == cid.0 { - painter.image(tex.id(), comp_rect, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + painter.image(*tex, comp_rect, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); app.auto.add("viewer.comp", comp_rect, &comp_name); @@ -1490,3 +1483,39 @@ pub(crate) fn hex_rgb(s: &str) -> Option<[u8; 3]> { let p = |i: usize| u8::from_str_radix(&h[i..i + 2], 16).ok(); Some([p(0)?, p(2)?, p(4)?]) } + +/// Put a rendered frame on screen: CPU pixels go into an egui texture; GPU frames are drawn +/// straight from their wgpu texture (registered with egui-wgpu, no readback). +fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames::FrameKey, img: crate::frames::FrameImage) { + use crate::frames::FrameImage; + match img { + FrameImage::Cpu(img) => { + match &mut app.viewer_tex { + Some((tex, k)) if tex.size() == img.size => { + tex.set((*img).clone(), egui::TextureOptions::LINEAR); + *k = key; + } + _ => app.viewer_tex = Some((ctx.load_texture("viewer-frame", (*img).clone(), egui::TextureOptions::LINEAR), key)), + } + app.viewer_shown = app.viewer_tex.as_ref().map(|(t, k)| (t.id(), *k)); + app.viewer_image = Some(img); + } + FrameImage::Gpu(f) => { + let Some(rs) = &app.wgpu else { return }; + let view = f.texture.create_view(&Default::default()); + let mut ren = rs.renderer.write(); + let id = match &app.viewer_native { + Some((id, _, _)) => { + ren.update_egui_texture_from_wgpu_texture(&rs.device, &view, eframe::wgpu::FilterMode::Linear, *id); + *id + } + None => ren.register_native_texture(&rs.device, &view, eframe::wgpu::FilterMode::Linear), + }; + drop(ren); + app.viewer_native = Some((id, key, f)); + app.viewer_shown = Some((id, key)); + // Pixels are read back on demand (viewer_pixels). + app.viewer_image = None; + } + } +} diff --git a/crates/ui-egui/tests/ui_gpu.rs b/crates/ui-egui/tests/ui_gpu.rs new file mode 100644 index 000000000..399535d4d --- /dev/null +++ b/crates/ui-egui/tests/ui_gpu.rs @@ -0,0 +1,56 @@ +//! The viewer on the GPU compositor (egui_kittest with wgpu): frames are drawn from GPU textures +//! and match the CPU viewer. Skips without a GPU adapter (CI). Set `EC_SNAPSHOT_DIR` to keep PNGs. + +use effectcraft_engine::Session; +use effectcraft_ui_egui::EffectcraftApp; +use egui_kittest::Harness; +use serde_json::json; + +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +fn viewer_draws_gpu_frames_that_match_the_cpu() { + if effectcraft_gpu::Gpu::headless().is_none() { + eprintln!("no GPU adapter: skipping"); + return; + } + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + settle(&mut h); + assert!(h.state().gpu_adapter().is_some(), "GPU compositor on egui-wgpu's device"); + assert!(h.state().viewer_on_gpu(), "viewer frame comes from the GPU"); + let gpu_px = h.state_mut().viewer_pixels().expect("read back the GPU frame"); + let gpu_shot = h.render().expect("render"); + // Mercury Software Only: the same frame from the CPU. + h.state_mut().session.execute("render.backend", json!({"backend": "cpu"})).unwrap(); + settle(&mut h); + assert!(!h.state().viewer_on_gpu()); + let cpu_px = h.state_mut().viewer_pixels().expect("CPU frame"); + let cpu_shot = h.render().expect("render"); + assert_eq!(gpu_px.size, cpu_px.size); + let mut worst = 0i32; + let mut over = 0usize; + for (a, b) in gpu_px.pixels.iter().zip(&cpu_px.pixels) { + let d = a.to_array().iter().zip(b.to_array()).map(|(x, y)| (*x as i32 - y as i32).abs()).max().unwrap_or(0); + worst = worst.max(d); + over += usize::from(d > 1); + } + assert!(over * 1000 <= gpu_px.pixels.len() * 3 && worst <= 4, "viewer GPU vs CPU: {over} pixels > 1/255, worst {worst}/255"); + if let Ok(d) = std::env::var("EC_SNAPSHOT_DIR") { + gpu_shot.save(format!("{d}/ui_gpu_viewer.png")).unwrap(); + cpu_shot.save(format!("{d}/ui_cpu_viewer.png")).unwrap(); + } +} From 65f03366f1b275879ffb9103213ca6e0df977850 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 11:10:49 -0400 Subject: [PATCH 3/3] M6.6: mask tracking + mask interpolation core (KLT, RANSAC fits, stabilization math) Library groundwork for mask tracking (MSK-3) and the Warp Stabilizer (TRK-2), clean-room from published methods: - track::klt: Shi-Tomasi corners, pyramidal Lucas-Kanade with forward-backward check - track::fit: RANSAC + least-squares translation/scale/similarity/affine/homography - track::mask: MaskTracker following the pixels inside a mask outline per frame - track::stabilize: incremental clip analysis (serde WarpAnalysis), Gaussian path smoothing (Matsushita 2006), No Motion, preserve scale, crop / auto-scale with max-scale relaxation - path::interp: Mask Interpolation correspondence (arc-length vertex insertion, shape-context matching) and rigid/linear in-betweens Engine commands, UI and the effect itself are not wired yet (stubs unchanged). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/path/src/interp.rs | 586 ++++++++++++++++++++++++++++++++++ crates/path/src/lib.rs | 1 + crates/track/Cargo.toml | 1 + crates/track/src/fit.rs | 308 ++++++++++++++++++ crates/track/src/klt.rs | 428 +++++++++++++++++++++++++ crates/track/src/lib.rs | 4 + crates/track/src/mask.rs | 222 +++++++++++++ crates/track/src/solve.rs | 85 ++++- crates/track/src/stabilize.rs | 576 +++++++++++++++++++++++++++++++++ 10 files changed, 2211 insertions(+), 1 deletion(-) create mode 100644 crates/path/src/interp.rs create mode 100644 crates/track/src/fit.rs create mode 100644 crates/track/src/klt.rs create mode 100644 crates/track/src/mask.rs create mode 100644 crates/track/src/stabilize.rs diff --git a/Cargo.lock b/Cargo.lock index 6ce4ac9fe..bf29ec8a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,6 +1398,7 @@ dependencies = [ "effectcraft-raster", "rayon", "serde", + "serde_json", ] [[package]] diff --git a/crates/path/src/interp.rs b/crates/path/src/interp.rs new file mode 100644 index 000000000..a3fe075a6 --- /dev/null +++ b/crates/path/src/interp.rs @@ -0,0 +1,586 @@ +//! Smart mask interpolation (the Mask Interpolation panel): give two mask shapes the same vertex +//! count with a sensible vertex correspondence, and compute in-between shapes. +//! +//! - **Vertex insertion** splits Bezier segments with de Casteljau subdivision, so both shapes +//! keep their exact outline while gaining vertices; vertices are distributed by **arc length** +//! so the i-th vertex of each shape sits at a comparable fraction of the outline. +//! - **Correspondence** picks the cyclic offset (and, for closed shapes, the winding) of the +//! second shape that minimises a matching cost: normalised position differences plus, when +//! Quality is above 0, the χ² distance of log-polar **shape context** histograms (Belongie, +//! Malik & Puzicha, "Shape Matching and Object Recognition Using Shape Contexts", PAMI 2002). +//! *First Vertices Match* pins the offset to 0; *Use 1:1 Vertex Matches* keeps the original +//! vertices when the counts agree. +//! - **In-betweens** move vertices along straight lines (*Use Linear Vertex Paths*) or, by +//! default, with the shapes' best-fit rotation and scale (Procrustes) interpolated so turning +//! shapes turn instead of collapsing; *Bending Resistance* weights that rigid component +//! against the straight-line one. + +use effectcraft_keyframe::ShapePath; + +/// Matching Method. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Matching { + #[default] + Auto, + Curve, + Polyline, +} + +/// Add Mask Shape Vertices. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum AddVertices { + /// Pixels between vertices. + Pixels(f64), + /// Total vertices. + Total(usize), + /// One vertex every this percentage of the outline. + Percent(f64), +} + +/// Mask Interpolation options. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct InterpOpts { + pub linear: bool, + /// Bending Resistance, 0–100. + pub bending_resistance: f64, + /// Quality, 0–100. + pub quality: f64, + pub add_vertices: Option, + pub matching: Matching, + pub one_to_one: bool, + pub first_vertices_match: bool, +} + +impl Default for InterpOpts { + fn default() -> Self { + InterpOpts { + linear: false, + bending_resistance: 50.0, + quality: 50.0, + add_vertices: Some(AddVertices::Pixels(15.0)), + matching: Matching::Auto, + one_to_one: false, + first_vertices_match: true, + } + } +} + +type P = [f64; 2]; + +fn add(a: P, b: P) -> P { + [a[0] + b[0], a[1] + b[1]] +} +fn sub(a: P, b: P) -> P { + [a[0] - b[0], a[1] - b[1]] +} +fn mul(a: P, k: f64) -> P { + [a[0] * k, a[1] * k] +} +fn lerp(a: P, b: P, t: f64) -> P { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t] +} + +/// Segment `i` (vertex i → i+1, wrapping for closed paths) as cubic control points. +fn segment(p: &ShapePath, i: usize) -> [P; 4] { + let n = p.len(); + let j = (i + 1) % n; + let (a, b) = (p.vertices[i], p.vertices[j]); + let o = p.out_tangents.get(i).copied().unwrap_or([0.0; 2]); + let it = p.in_tangents.get(j).copied().unwrap_or([0.0; 2]); + [a, add(a, o), add(b, it), b] +} + +fn seg_count(p: &ShapePath) -> usize { + let n = p.len(); + if n < 2 { + 0 + } else if p.closed { + n + } else { + n - 1 + } +} + +fn cubic(c: &[P; 4], t: f64) -> P { + let u = 1.0 - t; + let (a, b, cc, d) = (u * u * u, 3.0 * u * u * t, 3.0 * u * t * t, t * t * t); + [a * c[0][0] + b * c[1][0] + cc * c[2][0] + d * c[3][0], a * c[0][1] + b * c[1][1] + cc * c[2][1] + d * c[3][1]] +} + +fn seg_len(c: &[P; 4]) -> f64 { + let mut l = 0.0; + let mut prev = c[0]; + for k in 1..=24 { + let q = cubic(c, k as f64 / 24.0); + l += (q[0] - prev[0]).hypot(q[1] - prev[1]); + prev = q; + } + l +} + +/// Outline length. +pub fn length(p: &ShapePath) -> f64 { + (0..seg_count(p)).map(|i| seg_len(&segment(p, i))).sum() +} + +/// Split cubic `c` at parameter values `ts` (increasing, in (0, 1)) into `ts.len() + 1` cubics. +fn split_many(c: &[P; 4], ts: &[f64]) -> Vec<[P; 4]> { + let mut out = vec![]; + let mut rest = *c; + let mut t0 = 0.0; + for &t in ts { + let u = ((t - t0) / (1.0 - t0)).clamp(0.0, 1.0); + let (l, r) = split(&rest, u); + out.push(l); + rest = r; + t0 = t; + } + out.push(rest); + out +} + +fn split(c: &[P; 4], t: f64) -> ([P; 4], [P; 4]) { + let p01 = lerp(c[0], c[1], t); + let p12 = lerp(c[1], c[2], t); + let p23 = lerp(c[2], c[3], t); + let a = lerp(p01, p12, t); + let b = lerp(p12, p23, t); + let m = lerp(a, b, t); + ([c[0], p01, a, m], [m, b, p23, c[3]]) +} + +/// Parameter values splitting `c` into `k` pieces of (approximately) equal arc length. +fn arc_params(c: &[P; 4], k: usize) -> Vec { + if k <= 1 { + return vec![]; + } + const S: usize = 64; + let mut cum = vec![0.0; S + 1]; + let mut prev = c[0]; + for i in 1..=S { + let q = cubic(c, i as f64 / S as f64); + cum[i] = cum[i - 1] + (q[0] - prev[0]).hypot(q[1] - prev[1]); + prev = q; + } + let total = cum[S]; + if total < 1e-12 { + return (1..k).map(|j| j as f64 / k as f64).collect(); + } + (1..k) + .map(|j| { + let target = total * j as f64 / k as f64; + let i = cum.partition_point(|v| *v < target).clamp(1, S); + let f = (target - cum[i - 1]) / (cum[i] - cum[i - 1]).max(1e-12); + ((i - 1) as f64 + f) / S as f64 + }) + .collect() +} + +/// Rebuild a ShapePath from consecutive cubics (closed: the last ends at the first's start). +fn from_cubics(cs: &[[P; 4]], closed: bool) -> ShapePath { + let n = if closed { cs.len() } else { cs.len() + 1 }; + let mut v = vec![[0.0; 2]; n]; + let mut ins = vec![[0.0; 2]; n]; + let mut outs = vec![[0.0; 2]; n]; + for (i, c) in cs.iter().enumerate() { + let j = if closed { (i + 1) % n } else { i + 1 }; + v[i] = c[0]; + outs[i] = sub(c[1], c[0]); + ins[j] = sub(c[2], c[3]); + if !closed && j == n - 1 { + v[j] = c[3]; + } + } + ShapePath { vertices: v, in_tangents: ins, out_tangents: outs, closed } +} + +/// Exactly the same outline with `total` vertices (≥ the current count), the extra vertices +/// distributed over segments in proportion to their length. +pub fn with_vertex_count(p: &ShapePath, total: usize) -> ShapePath { + let m = seg_count(p); + let base = if p.closed { m } else { m + 1 }; + if m == 0 || total <= base { + return p.clone(); + } + let segs: Vec<[P; 4]> = (0..m).map(|i| segment(p, i)).collect(); + let lens: Vec = segs.iter().map(seg_len).collect(); + let sum: f64 = lens.iter().sum::().max(1e-12); + let pieces = total - base + m; // total segments wanted + // Largest remainder allocation, at least one piece per segment. + let extra = pieces - m; + let want: Vec = lens.iter().map(|l| l / sum * extra as f64).collect(); + let mut k: Vec = want.iter().map(|w| 1 + w.floor() as usize).collect(); + let mut left = pieces - k.iter().sum::(); + let mut order: Vec = (0..m).collect(); + order.sort_by(|a, b| (want[*b] - want[*b].floor()).total_cmp(&(want[*a] - want[*a].floor())).then(a.cmp(b))); + for i in order.iter().cycle() { + if left == 0 { + break; + } + k[*i] += 1; + left -= 1; + } + let mut cubics = vec![]; + for (c, ki) in segs.iter().zip(&k) { + cubics.extend(split_many(c, &arc_params(c, *ki))); + } + from_cubics(&cubics, p.closed) +} + +/// The same outline as a polyline with `total` vertices at equal arc-length spacing, starting at +/// vertex 0. +pub fn polyline(p: &ShapePath, total: usize) -> ShapePath { + let m = seg_count(p); + if m == 0 || total < 2 { + return ShapePath::polygon(&p.vertices, p.closed); + } + // Dense samples with cumulative length. + let mut pts = vec![p.vertices[0]]; + for i in 0..m { + let c = segment(p, i); + for k in 1..=32 { + pts.push(cubic(&c, k as f64 / 32.0)); + } + } + let mut cum = vec![0.0]; + for w in pts.windows(2) { + let l = cum.last().copied().unwrap_or(0.0) + (w[1][0] - w[0][0]).hypot(w[1][1] - w[0][1]); + cum.push(l); + } + let total_len = *cum.last().unwrap_or(&0.0); + let n = total; + let div = if p.closed { n as f64 } else { (n - 1) as f64 }; + let out: Vec

= (0..n) + .map(|j| { + let target = total_len * j as f64 / div; + let i = cum.partition_point(|v| *v < target).clamp(1, cum.len() - 1); + let f = (target - cum[i - 1]) / (cum[i] - cum[i - 1]).max(1e-12); + lerp(pts[i - 1], pts[i], f) + }) + .collect(); + ShapePath::polygon(&out, p.closed) +} + +fn has_curves(p: &ShapePath) -> bool { + p.in_tangents.iter().chain(&p.out_tangents).any(|t| t[0].abs() > 1e-9 || t[1].abs() > 1e-9) +} + +/// Rotate a closed path's vertex order to start at `k`. +fn rotate(p: &ShapePath, k: usize) -> ShapePath { + let n = p.len(); + if n == 0 || k.is_multiple_of(n) { + return p.clone(); + } + let r = |v: &Vec

| (0..n).map(|i| v[(i + k) % n]).collect(); + ShapePath { vertices: r(&p.vertices), in_tangents: r(&p.in_tangents), out_tangents: r(&p.out_tangents), closed: p.closed } +} + +/// The same path traversed the other way (closed paths keep vertex 0 first). +pub fn reversed(p: &ShapePath) -> ShapePath { + let n = p.len(); + if n == 0 { + return p.clone(); + } + let idx: Vec = if p.closed { (0..n).map(|i| (n - i) % n).collect() } else { (0..n).rev().collect() }; + ShapePath { + vertices: idx.iter().map(|i| p.vertices[*i]).collect(), + in_tangents: idx.iter().map(|i| p.out_tangents[*i]).collect(), + out_tangents: idx.iter().map(|i| p.in_tangents[*i]).collect(), + closed: p.closed, + } +} + +fn signed_area(v: &[P]) -> f64 { + let n = v.len(); + (0..n).map(|i| v[i][0] * v[(i + 1) % n][1] - v[(i + 1) % n][0] * v[i][1]).sum::() * 0.5 +} + +/// Centroid and RMS radius of points. +fn frame(v: &[P]) -> (P, f64) { + let n = v.len().max(1) as f64; + let c = [v.iter().map(|p| p[0]).sum::() / n, v.iter().map(|p| p[1]).sum::() / n]; + let r = (v.iter().map(|p| (p[0] - c[0]).powi(2) + (p[1] - c[1]).powi(2)).sum::() / n).sqrt().max(1e-9); + (c, r) +} + +const RBINS: usize = 5; +const ABINS: usize = 12; + +/// Log-polar shape context histograms of every point (normalised by the RMS radius). +fn shape_contexts(v: &[P]) -> Vec<[f64; RBINS * ABINS]> { + let (_, r) = frame(v); + let n = v.len(); + (0..n) + .map(|i| { + let mut h = [0.0; RBINS * ABINS]; + for j in 0..n { + if i == j { + continue; + } + let d = sub(v[j], v[i]); + let dist = d[0].hypot(d[1]) / r; + // log radius in [1/8, 2] → 5 bins. + let lr = (dist.max(1e-6).log2() + 3.0) / 4.0 * RBINS as f64; + if !(0.0..RBINS as f64).contains(&lr) { + continue; + } + let a = (d[1].atan2(d[0]) + std::f64::consts::PI) / std::f64::consts::TAU * ABINS as f64; + let ai = (a as usize).min(ABINS - 1); + h[lr as usize * ABINS + ai] += 1.0; + } + let s: f64 = h.iter().sum::().max(1.0); + h.iter_mut().for_each(|x| *x /= s); + h + }) + .collect() +} + +fn chi2(a: &[f64], b: &[f64]) -> f64 { + a.iter().zip(b).map(|(x, y)| if x + y > 1e-12 { (x - y) * (x - y) / (x + y) } else { 0.0 }).sum::() * 0.5 +} + +/// Matching cost of `a[i] ↔ b[(i + k) % n]`. +fn cost(a: &[P], b: &[P], sa: Option<&[[f64; RBINS * ABINS]]>, sb: Option<&[[f64; RBINS * ABINS]]>, k: usize) -> f64 { + let n = a.len(); + let (ca, ra) = frame(a); + let (cb, rb) = frame(b); + let mut c = 0.0; + for i in 0..n { + let j = (i + k) % n; + let pa = mul(sub(a[i], ca), 1.0 / ra); + let pb = mul(sub(b[j], cb), 1.0 / rb); + c += (pa[0] - pb[0]).powi(2) + (pa[1] - pb[1]).powi(2); + if let (Some(sa), Some(sb)) = (sa, sb) { + c += chi2(&sa[i], &sb[j]); + } + } + c / n as f64 +} + +/// Give `a` and `b` the same vertex count with matching vertices (the shapes that the first +/// and last keys get, and the input of [`interpolate`]). +pub fn correspond(a: &ShapePath, b: &ShapePath, o: &InterpOpts) -> (ShapePath, ShapePath) { + if a.is_empty() || b.is_empty() { + return (a.clone(), b.clone()); + } + let closed = a.closed && b.closed; + let curves = match o.matching { + Matching::Curve => true, + Matching::Polyline => false, + Matching::Auto => has_curves(a) || has_curves(b), + }; + let (mut ra, mut rb); + if o.one_to_one && a.len() == b.len() { + (ra, rb) = (a.clone(), b.clone()); + } else { + let (la, lb) = (length(a), length(b)); + let mut n = a.len().max(b.len()); + if let Some(av) = o.add_vertices { + let want = match av { + AddVertices::Pixels(px) => (la.max(lb) / px.max(1.0)).ceil() as usize, + AddVertices::Total(t) => t, + AddVertices::Percent(pc) => (100.0 / pc.clamp(0.1, 100.0)).ceil() as usize, + }; + n = n.max(want.min(2000)); + } + if curves { + ra = with_vertex_count(a, n); + rb = with_vertex_count(b, n); + } else { + ra = polyline(a, n); + rb = polyline(b, n); + } + // Proportional splitting can leave counts unequal when one shape already had more + // vertices than its share: fall back to arc-length polylines then. + if ra.len() != rb.len() { + ra = polyline(a, n); + rb = polyline(b, n); + } + } + if closed && ra.len() == rb.len() && ra.len() >= 3 { + // Same winding. + if signed_area(&ra.vertices).signum() != signed_area(&rb.vertices).signum() { + rb = reversed(&rb); + } + if !o.first_vertices_match { + let n = ra.len(); + let use_sc = o.quality > 0.0; + let (sa, sb) = if use_sc { (Some(shape_contexts(&ra.vertices)), Some(shape_contexts(&rb.vertices))) } else { (None, None) }; + // Quality limits the offsets examined exhaustively (coarse step), then refines. + let step = if o.quality >= 50.0 { 1 } else { ((n as f64 / 24.0).ceil() as usize).max(1) }; + let mut best = (f64::MAX, 0usize); + let mut k = 0; + while k < n { + let c = cost(&ra.vertices, &rb.vertices, sa.as_deref(), sb.as_deref(), k); + if c < best.0 { + best = (c, k); + } + k += step; + } + if step > 1 { + let k0 = best.1; + for d in 1..step { + for k in [(k0 + d) % n, (k0 + n - d) % n] { + let c = cost(&ra.vertices, &rb.vertices, sa.as_deref(), sb.as_deref(), k); + if c < best.0 { + best = (c, k); + } + } + } + } + rb = rotate(&rb, best.1); + } + } + (ra, rb) +} + +/// The in-between shape at fraction `u` of two corresponded shapes (equal vertex counts). +pub fn interpolate(a: &ShapePath, b: &ShapePath, u: f64, o: &InterpOpts) -> ShapePath { + if a.len() != b.len() || a.is_empty() { + return if u < 1.0 { a.clone() } else { b.clone() }; + } + let n = a.len(); + let lin = |pa: &[P], pb: &[P]| -> Vec

{ pa.iter().zip(pb).map(|(x, y)| lerp(*x, *y, u)).collect() }; + if o.linear { + return ShapePath { + vertices: lin(&a.vertices, &b.vertices), + in_tangents: lin(&a.in_tangents, &b.in_tangents), + out_tangents: lin(&a.out_tangents, &b.out_tangents), + closed: a.closed, + }; + } + // Procrustes: b ≈ s R (a − ca) + cb. + let (ca, _) = frame(&a.vertices); + let (cb, _) = frame(&b.vertices); + let (mut ss, mut dot, mut cross) = (0.0, 0.0, 0.0); + for i in 0..n { + let (p, q) = (sub(a.vertices[i], ca), sub(b.vertices[i], cb)); + ss += p[0] * p[0] + p[1] * p[1]; + dot += p[0] * q[0] + p[1] * q[1]; + cross += p[0] * q[1] - p[1] * q[0]; + } + let theta = cross.atan2(dot); + let s = if ss > 1e-12 { dot.hypot(cross) / ss } else { 1.0 }; + let rot = |v: P, ang: f64, k: f64| { + let (sn, cs) = ang.sin_cos(); + [k * (cs * v[0] - sn * v[1]), k * (sn * v[0] + cs * v[1])] + }; + let (th_u, s_u) = (theta * u, s.max(1e-9).powf(u)); + let c_u = lerp(ca, cb, u); + let w = (o.bending_resistance / 100.0).clamp(0.0, 1.0); + let mut out = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: a.closed }; + for i in 0..n { + // Rigid path: a's vertex carried by the interpolated similarity, plus the residual + // (b − fully transformed a) blended in linearly. + let pa = sub(a.vertices[i], ca); + let full = add(rot(pa, theta, s), cb); + let resid = sub(b.vertices[i], full); + let rigid = add(add(rot(pa, th_u, s_u), c_u), mul(resid, u)); + let linear = lerp(a.vertices[i], b.vertices[i], u); + out.vertices.push(lerp(linear, rigid, w)); + for (src_a, src_b, dst) in [(&a.in_tangents, &b.in_tangents, &mut out.in_tangents), (&a.out_tangents, &b.out_tangents, &mut out.out_tangents)] { + let ta = src_a[i]; + let tr = sub(src_b[i], rot(ta, theta, s)); + let rigid_t = add(rot(ta, th_u, s_u), mul(tr, u)); + dst.push(lerp(lerp(ta, src_b[i], u), rigid_t, w)); + } + } + out +} + +/// Do any two vertex paths (straight segments from `a[i]` to `b[i]`) cross? (A sanity measure of +/// the correspondence: good matches don't make vertices pass through each other.) +pub fn paths_cross(a: &ShapePath, b: &ShapePath) -> usize { + let n = a.len().min(b.len()); + let mut c = 0; + let inter = |p1: P, p2: P, p3: P, p4: P| { + let d = |a: P, b: P, c: P| (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]); + let (d1, d2, d3, d4) = (d(p3, p4, p1), d(p3, p4, p2), d(p1, p2, p3), d(p1, p2, p4)); + d1 * d2 < 0.0 && d3 * d4 < 0.0 + }; + for i in 0..n { + for j in i + 1..n { + if inter(a.vertices[i], b.vertices[i], a.vertices[j], b.vertices[j]) { + c += 1; + } + } + } + c +} + +#[cfg(test)] +mod tests { + use super::*; + + fn star(c: P, r: f64, k: usize, rot: f64) -> ShapePath { + let pts: Vec

= (0..2 * k) + .map(|i| { + let a = rot + i as f64 * std::f64::consts::PI / k as f64; + let rr = if i % 2 == 0 { r } else { r * 0.5 }; + [c[0] + rr * a.cos(), c[1] + rr * a.sin()] + }) + .collect(); + ShapePath::polygon(&pts, true) + } + + #[test] + fn splitting_keeps_the_outline() { + let e = ShapePath::ellipse([50.0, 50.0], 80.0, 40.0); + let m = with_vertex_count(&e, 23); + assert_eq!(m.len(), 23); + assert!((length(&m) - length(&e)).abs() < 0.05, "{} vs {}", length(&m), length(&e)); + // Points on the split path lie on the ellipse. + for i in 0..m.len() { + let v = m.vertices[i]; + let q = ((v[0] - 50.0) / 40.0).powi(2) + ((v[1] - 50.0) / 20.0).powi(2); + assert!((q - 1.0).abs() < 0.01, "{v:?}"); + } + let r = reversed(&ShapePath::rect([0.0, 0.0], 10.0, 10.0)); + assert_eq!(r.vertices[0], [-5.0, -5.0]); + assert_eq!(r.vertices[1], [-5.0, 5.0]); + } + + #[test] + fn correspondence_finds_the_rotation_and_paths_dont_cross() { + // A star rotated by 2 points' worth and with its first vertex elsewhere. + let a = star([100.0, 100.0], 60.0, 5, 0.0); + let b0 = star([140.0, 110.0], 50.0, 5, 0.3); + let b = rotate(&b0, 3); + let o = InterpOpts { first_vertices_match: false, add_vertices: None, ..Default::default() }; + let (ra, rb) = correspond(&a, &b, &o); + assert_eq!(ra.len(), rb.len()); + // The true correspondence is recovered (b0's vertex 0 matches a's vertex 0). + assert!((rb.vertices[0][0] - b0.vertices[0][0]).abs() < 1e-9 && (rb.vertices[0][1] - b0.vertices[0][1]).abs() < 1e-9); + assert!(paths_cross(&ra, &rb) <= 1); + // With First Vertices Match the given (bad) start is kept: paths cross. + let o2 = InterpOpts { first_vertices_match: true, add_vertices: None, ..Default::default() }; + let (xa, xb) = correspond(&a, &b, &o2); + assert!(paths_cross(&xa, &xb) > 5); + // In-betweens keep the vertex count and stay star-like (area between the ends). + let mid = interpolate(&ra, &rb, 0.5, &o); + assert_eq!(mid.len(), ra.len()); + let (aa, ab, am) = (signed_area(&ra.vertices).abs(), signed_area(&rb.vertices).abs(), signed_area(&mid.vertices).abs()); + assert!(am > ab.min(aa) * 0.8 && am < aa.max(ab) * 1.2, "{aa} {ab} {am}"); + // Different vertex counts with added vertices. + let c = ShapePath::ellipse([100.0, 100.0], 120.0, 120.0); + let o3 = InterpOpts { first_vertices_match: false, ..Default::default() }; + let (pa, pb) = correspond(&a, &c, &o3); + assert_eq!(pa.len(), pb.len()); + assert!(pa.len() >= 25); + assert!(paths_cross(&pa, &pb) <= pa.len() / 10, "{}", paths_cross(&pa, &pb)); + } + + #[test] + fn rigid_interpolation_turns_instead_of_collapsing() { + let a = ShapePath::polygon(&[[-50.0, -10.0], [50.0, -10.0], [50.0, 10.0], [-50.0, 10.0]], true); + // The same bar turned 90°. + let b = ShapePath::polygon(&[[10.0, -50.0], [10.0, 50.0], [-10.0, 50.0], [-10.0, -50.0]], true); + let o = InterpOpts { bending_resistance: 100.0, ..Default::default() }; + let m = interpolate(&a, &b, 0.5, &o); + let len = (m.vertices[1][0] - m.vertices[0][0]).hypot(m.vertices[1][1] - m.vertices[0][1]); + assert!((len - 100.0).abs() < 1e-6, "{len}"); + let lin = interpolate(&a, &b, 0.5, &InterpOpts { linear: true, ..Default::default() }); + let l2 = (lin.vertices[1][0] - lin.vertices[0][0]).hypot(lin.vertices[1][1] - lin.vertices[0][1]); + assert!(l2 < 80.0); + } +} diff --git a/crates/path/src/lib.rs b/crates/path/src/lib.rs index 1a1ae2ad1..d57f857ef 100644 --- a/crates/path/src/lib.rs +++ b/crates/path/src/lib.rs @@ -7,6 +7,7 @@ pub mod boolean; mod fit; +pub mod interp; pub mod offset; pub mod ops; diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index f8cd71d1f..ba449c92b 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -13,3 +13,4 @@ workspace = true effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/track/src/fit.rs b/crates/track/src/fit.rs new file mode 100644 index 000000000..75a0d9d1b --- /dev/null +++ b/crates/track/src/fit.rs @@ -0,0 +1,308 @@ +//! Robust motion-model fits between point correspondences: **RANSAC** (Fischler & Bolles, +//! "Random Sample Consensus", 1981) with minimal samples, followed by least-squares refits on the +//! consensus set (Hartley & Zisserman, *Multiple View Geometry*, §4.7–4.8). +//! +//! Models, in increasing freedom: translation, translation + uniform scale, similarity (rotation, +//! uniform scale, translation), affine (adds skew / non-uniform scale) and homography +//! (perspective). Every fit is returned as a [`Homography`] matrix mapping `src` to `dst`. + +use serde::{Deserialize, Serialize}; + +use crate::solve::{Homography, solve_linear}; + +/// A 2D motion model. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Model { + Translation, + TranslationScale, + #[default] + Similarity, + Affine, + Homography, +} + +impl Model { + /// Points in a minimal sample. + pub fn min_points(self) -> usize { + match self { + Model::Translation => 1, + Model::TranslationScale | Model::Similarity => 2, + Model::Affine => 3, + Model::Homography => 4, + } + } +} + +fn centroid(p: &[[f64; 2]]) -> [f64; 2] { + let n = p.len().max(1) as f64; + [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n] +} + +/// Least-squares fit of `model` taking `src` to `dst` (`None` when degenerate). +pub fn least_squares(model: Model, src: &[[f64; 2]], dst: &[[f64; 2]]) -> Option { + let n = src.len().min(dst.len()); + if n < model.min_points() { + return None; + } + let (src, dst) = (&src[..n], &dst[..n]); + let (cs, cd) = (centroid(src), centroid(dst)); + let h = match model { + Model::Translation => Homography::translation([cd[0] - cs[0], cd[1] - cs[1]]), + Model::TranslationScale | Model::Similarity => { + let (mut ss, mut dot, mut cross) = (0.0, 0.0, 0.0); + for (p, q) in src.iter().zip(dst) { + let (a, b) = ([p[0] - cs[0], p[1] - cs[1]], [q[0] - cd[0], q[1] - cd[1]]); + ss += a[0] * a[0] + a[1] * a[1]; + dot += a[0] * b[0] + a[1] * b[1]; + cross += a[0] * b[1] - a[1] * b[0]; + } + if ss < 1e-12 { + return None; + } + let (a, b) = if model == Model::Similarity { (dot / ss, cross / ss) } else { (dot / ss, 0.0) }; + if a.hypot(b) < 1e-9 { + return None; + } + // q = R p + t with R = [[a, -b], [b, a]]. + let t = [cd[0] - (a * cs[0] - b * cs[1]), cd[1] - (b * cs[0] + a * cs[1])]; + Homography([[a, -b, t[0]], [b, a, t[1]], [0.0, 0.0, 1.0]]) + } + Model::Affine => { + let mut m = [[0.0; 3]; 3]; + let (mut rx, mut ry) = ([0.0; 3], [0.0; 3]); + for (p, q) in src.iter().zip(dst) { + let v = [p[0] - cs[0], p[1] - cs[1], 1.0]; + for i in 0..3 { + for j in 0..3 { + m[i][j] += v[i] * v[j]; + } + rx[i] += v[i] * (q[0] - cd[0]); + ry[i] += v[i] * (q[1] - cd[1]); + } + } + let x = solve_linear(m, rx)?; + let y = solve_linear(m, ry)?; + // q - cd = A (p - cs) + t' + let t = [cd[0] + x[2] - x[0] * cs[0] - x[1] * cs[1], cd[1] + y[2] - y[0] * cs[0] - y[1] * cs[1]]; + Homography([[x[0], x[1], t[0]], [y[0], y[1], t[1]], [0.0, 0.0, 1.0]]) + } + Model::Homography => { + // Normalised DLT with h33 = 1 (normal equations). + let norm = |p: &[[f64; 2]], c: [f64; 2]| { + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / p.len() as f64; + if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 } + }; + let (ks, kd) = (norm(src, cs), norm(dst, cd)); + let mut a = [[0.0; 8]; 8]; + let mut b = [0.0; 8]; + for (p, q) in src.iter().zip(dst) { + let (x, y) = ((p[0] - cs[0]) * ks, (p[1] - cs[1]) * ks); + let (u, v) = ((q[0] - cd[0]) * kd, (q[1] - cd[1]) * kd); + let rows = [([x, y, 1.0, 0.0, 0.0, 0.0, -u * x, -u * y], u), ([0.0, 0.0, 0.0, x, y, 1.0, -v * x, -v * y], v)]; + for (r, t) in rows { + for i in 0..8 { + b[i] += r[i] * t; + for j in 0..8 { + a[i][j] += r[i] * r[j]; + } + } + } + } + let h = solve_linear(a, b)?; + let hn = Homography([[h[0], h[1], h[2]], [h[3], h[4], h[5]], [h[6], h[7], 1.0]]); + let ts = Homography([[ks, 0.0, -ks * cs[0]], [0.0, ks, -ks * cs[1]], [0.0, 0.0, 1.0]]); + let td_inv = Homography([[1.0 / kd, 0.0, cd[0]], [0.0, 1.0 / kd, cd[1]], [0.0, 0.0, 1.0]]); + let m = td_inv.then_after(&hn.then_after(&ts)); + if m.0[2][2].abs() < 1e-12 { + return None; + } + m.normalized() + } + }; + h.0.iter().flatten().all(|v| v.is_finite()).then_some(h) +} + +fn residual2(h: &Homography, p: [f64; 2], q: [f64; 2]) -> f64 { + let m = h.apply(p); + (m[0] - q[0]).powi(2) + (m[1] - q[1]).powi(2) +} + +/// Deterministic xorshift generator (reproducible fits). +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +/// A robust fit: the model, which correspondences are inliers, and the RMS inlier error. +#[derive(Clone, Debug)] +pub struct Fit { + pub h: Homography, + pub inliers: Vec, + pub rms: f64, +} + +impl Fit { + pub fn inlier_count(&self) -> usize { + self.inliers.iter().filter(|b| **b).count() + } +} + +/// RANSAC fit of `model` with inlier `threshold` (pixels) and at most `iterations` samples. +pub fn ransac(model: Model, src: &[[f64; 2]], dst: &[[f64; 2]], threshold: f64, iterations: usize, seed: u64) -> Option { + let n = src.len().min(dst.len()); + let k = model.min_points(); + if n < k { + return None; + } + let t2 = threshold * threshold; + let count = |h: &Homography| { + let mut c = 0; + let mut e = 0.0; + for i in 0..n { + let r = residual2(h, src[i], dst[i]); + if r < t2 { + c += 1; + e += r; + } + } + (c, e) + }; + let mut best: Option<(usize, f64, Homography)> = None; + if n == k { + let h = least_squares(model, src, dst)?; + let (c, e) = count(&h); + best = Some((c, e, h)); + } else { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut max_iter = iterations.max(1); + let mut it = 0; + let mut idx = vec![0usize; k]; + let (mut ss, mut sd) = (vec![[0.0; 2]; k], vec![[0.0; 2]; k]); + while it < max_iter { + it += 1; + // Distinct random sample. + let mut j = 0; + let mut guard = 0; + while j < k && guard < 100 { + guard += 1; + let c = rng.below(n); + if idx[..j].contains(&c) { + continue; + } + idx[j] = c; + j += 1; + } + if j < k { + continue; + } + for (m, &i) in idx.iter().enumerate() { + ss[m] = src[i]; + sd[m] = dst[i]; + } + let Some(h) = least_squares(model, &ss, &sd) else { continue }; + let (c, e) = count(&h); + let better = match &best { + None => true, + Some((bc, be, _)) => c > *bc || (c == *bc && e < *be), + }; + if better { + best = Some((c, e, h)); + // Adaptive stopping (99 % confidence of an outlier-free sample). + let w = c as f64 / n as f64; + let p_good = w.powi(k as i32); + if p_good > 1e-9 && p_good < 1.0 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(it + 1)); + } + } else if p_good >= 1.0 { + break; + } + } + } + } + let (_, _, mut h) = best?; + // Refine on the consensus set (twice: the set can grow after the first refit). + let mut inl: Vec = (0..n).map(|i| residual2(&h, src[i], dst[i]) < t2).collect(); + for _ in 0..2 { + let (s2, d2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (src[i], dst[i])).unzip(); + if s2.len() < k { + break; + } + if let Some(r) = least_squares(model, &s2, &d2) { + h = r; + } + inl = (0..n).map(|i| residual2(&h, src[i], dst[i]) < t2).collect(); + } + let c = inl.iter().filter(|b| **b).count(); + let e: f64 = (0..n).filter(|i| inl[*i]).map(|i| residual2(&h, src[i], dst[i])).sum(); + Some(Fit { h, inliers: inl, rms: if c > 0 { (e / c as f64).sqrt() } else { f64::INFINITY } }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pts() -> Vec<[f64; 2]> { + let mut v = vec![]; + let mut s = 12345u64; + for _ in 0..80 { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let x = (s >> 33) as f64 / (1u64 << 31) as f64 * 300.0; + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let y = (s >> 33) as f64 / (1u64 << 31) as f64 * 200.0; + v.push([x, y]); + } + v + } + + #[test] + fn ransac_rejects_outliers_for_every_model() { + let truth = [ + (Model::Translation, Homography::translation([5.0, -3.0])), + (Model::TranslationScale, Homography([[1.1, 0.0, -4.0], [0.0, 1.1, 2.0], [0.0, 0.0, 1.0]])), + (Model::Similarity, Homography([[0.98, -0.17, 12.0], [0.17, 0.98, -6.0], [0.0, 0.0, 1.0]])), + (Model::Affine, Homography([[1.05, 0.12, 3.0], [-0.04, 0.93, 1.0], [0.0, 0.0, 1.0]])), + (Model::Homography, Homography([[1.02, 0.05, 4.0], [-0.03, 0.97, 2.0], [0.0004, -0.0003, 1.0]])), + ]; + for (m, h) in truth { + let src = pts(); + let mut dst: Vec<[f64; 2]> = src.iter().map(|p| h.apply(*p)).collect(); + // 30 % gross outliers. + for (i, d) in dst.iter_mut().enumerate() { + if i % 10 < 3 { + d[0] += 40.0 + i as f64; + d[1] -= 25.0; + } + } + let f = ransac(m, &src, &dst, 1.0, 500, 7).unwrap(); + assert_eq!(f.inlier_count(), 56, "{m:?}"); + for p in [[0.0, 0.0], [150.0, 100.0], [300.0, 200.0]] { + let (a, b) = (f.h.apply(p), h.apply(p)); + assert!((a[0] - b[0]).abs() < 1e-6 && (a[1] - b[1]).abs() < 1e-6, "{m:?} {a:?} {b:?}"); + } + } + } + + #[test] + fn inverse_and_compose() { + let h = Homography([[1.02, 0.05, 4.0], [-0.03, 0.97, 2.0], [0.0004, -0.0003, 1.0]]); + let i = h.inverse().unwrap(); + let p = [123.0, 45.0]; + let q = i.apply(h.apply(p)); + assert!((q[0] - p[0]).abs() < 1e-9 && (q[1] - p[1]).abs() < 1e-9); + let c = Homography::compose(&h, &i); + assert!(c.max_diff(&Homography::IDENTITY) < 1e-9); + } +} diff --git a/crates/track/src/klt.rs b/crates/track/src/klt.rs new file mode 100644 index 000000000..6c569c310 --- /dev/null +++ b/crates/track/src/klt.rs @@ -0,0 +1,428 @@ +//! Sparse feature tracking (KLT), clean-room from the published methods: +//! +//! - **corners**: the Shi–Tomasi criterion (Shi & Tomasi, "Good Features to Track", CVPR 1994): +//! the smaller eigenvalue of the gradient structure tensor summed over a small window, with +//! grid-based non-maximum suppression and a minimum distance between features; +//! - **tracking**: iterative Lucas–Kanade translation estimation (Lucas & Kanade 1981) run coarse +//! to fine over an image pyramid (the pyramidal scheme described in Bouguet's 2000 technical +//! note "Pyramidal Implementation of the Lucas Kanade Feature Tracker"), with a +//! **forward–backward** consistency check (Kalal et al., "Forward-Backward Error", ICPR 2010) to +//! drop features that were lost or occluded. +//! +//! Frames are analysed in luma at a reduced resolution ([`GrayPyramid::from_image`] takes an +//! integer box-downsampling factor). Coordinates follow [`Plane`]: pixel `i` covers `[i, i + 1)`. + +use effectcraft_raster::Image; +use rayon::prelude::*; + +use crate::plane::Plane; + +/// A luma pyramid with gradients per level. +#[derive(Clone, Debug, Default)] +pub struct GrayPyramid { + pub levels: Vec, + pub grads: Vec<(Plane, Plane)>, + /// Image pixels per level-0 pixel. + pub factor: f64, + /// Image pixel offset of the plane origin (layer pixel `p` is image pixel `p + offset`). + pub offset: [f64; 2], +} + +/// Luma of `img` averaged over `factor × factor` blocks (straight colour over premultiplied +/// alpha is ignored: tracking looks at the visible pixels). +pub fn luma_plane(img: &Image, factor: usize) -> Plane { + let f = factor.max(1); + let w = (img.width as usize).div_ceil(f).max(1); + let h = (img.height as usize).div_ceil(f).max(1); + let mut p = Plane::new(w, h, 1); + if img.is_empty() { + return p; + } + p.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let mut s = 0.0f32; + let mut n = 0.0f32; + for yy in y * f..((y + 1) * f).min(img.height as usize) { + for xx in x * f..((x + 1) * f).min(img.width as usize) { + let px = img.data[yy * img.width as usize + xx]; + s += 0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2]; + n += 1.0; + } + } + *o = if n > 0.0 { s / n } else { 0.0 }; + } + }); + p +} + +impl GrayPyramid { + /// A pyramid of `img` downsampled by `factor`, with `levels` extra half-resolution levels. + pub fn from_image(img: &Image, offset: [f64; 2], factor: usize, levels: usize) -> GrayPyramid { + let base = luma_plane(img, factor); + Self::from_plane(base, offset, factor as f64, levels) + } + + pub fn from_plane(base: Plane, offset: [f64; 2], factor: f64, levels: usize) -> GrayPyramid { + let mut lv = vec![base]; + for _ in 0..levels { + let last = lv.last().expect("level"); + if last.w < 16 || last.h < 16 { + break; + } + let d = last.downsample(); + lv.push(d); + } + let grads = lv.par_iter().map(Plane::gradients).collect(); + GrayPyramid { levels: lv, grads, factor, offset } + } + + /// Layer pixels → level-0 plane coordinates. + pub fn to_plane(&self, p: [f64; 2]) -> [f64; 2] { + [(p[0] + self.offset[0]) / self.factor, (p[1] + self.offset[1]) / self.factor] + } + /// Level-0 plane coordinates → layer pixels. + pub fn to_layer(&self, q: [f64; 2]) -> [f64; 2] { + [q[0] * self.factor - self.offset[0], q[1] * self.factor - self.offset[1]] + } + pub fn width(&self) -> usize { + self.levels.first().map(|p| p.w).unwrap_or(0) + } + pub fn height(&self) -> usize { + self.levels.first().map(|p| p.h).unwrap_or(0) + } +} + +/// Feature detection options. +#[derive(Clone, Copy, Debug)] +pub struct CornerOpts { + pub max_features: usize, + /// Minimum distance between features (level-0 plane pixels). + pub min_distance: f64, + /// Keep corners whose score is at least this fraction of the strongest. + pub quality: f64, + /// Structure-tensor window radius. + pub window: usize, + /// Ignore this many pixels at the border. + pub border: usize, +} + +impl Default for CornerOpts { + fn default() -> Self { + CornerOpts { max_features: 300, min_distance: 8.0, quality: 0.01, window: 2, border: 6 } + } +} + +/// Shi–Tomasi corner score (smaller eigenvalue of the structure tensor) per pixel. +pub fn corner_scores(p: &Plane, gx: &Plane, gy: &Plane, r: usize) -> Vec { + let (w, h) = (p.w, p.h); + // Products, then box sums via summed-area tables. + let n = w * h; + let mut sxx = vec![0.0f64; (w + 1) * (h + 1)]; + let mut sxy = vec![0.0f64; (w + 1) * (h + 1)]; + let mut syy = vec![0.0f64; (w + 1) * (h + 1)]; + for y in 0..h { + let (mut axx, mut axy, mut ayy) = (0.0, 0.0, 0.0); + for x in 0..w { + let i = y * w + x; + let (dx, dy) = (gx.data[i] as f64, gy.data[i] as f64); + axx += dx * dx; + axy += dx * dy; + ayy += dy * dy; + let o = (y + 1) * (w + 1) + x + 1; + let up = y * (w + 1) + x + 1; + sxx[o] = sxx[up] + axx; + sxy[o] = sxy[up] + axy; + syy[o] = syy[up] + ayy; + } + } + let mut out = vec![0.0f32; n]; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let y0 = y.saturating_sub(r); + let y1 = (y + r + 1).min(h); + for (x, o) in row.iter_mut().enumerate() { + let x0 = x.saturating_sub(r); + let x1 = (x + r + 1).min(w); + let bs = |s: &[f64]| s[y1 * (w + 1) + x1] - s[y0 * (w + 1) + x1] - s[y1 * (w + 1) + x0] + s[y0 * (w + 1) + x0]; + let (a, b, c) = (bs(&sxx), bs(&sxy), bs(&syy)); + let tr = 0.5 * (a + c); + let d = (0.25 * (a - c) * (a - c) + b * b).sqrt(); + *o = (tr - d).max(0.0) as f32; + } + }); + out +} + +/// Good features to track in `pyr` level 0, optionally restricted by `inside(plane point)`. +/// Returns level-0 plane coordinates (pixel centres), strongest first. +pub fn good_features(pyr: &GrayPyramid, opts: &CornerOpts, inside: Option<&(dyn Fn([f64; 2]) -> bool + Sync)>) -> Vec<[f64; 2]> { + let Some(p) = pyr.levels.first() else { return vec![] }; + let (gx, gy) = &pyr.grads[0]; + let score = corner_scores(p, gx, gy, opts.window.max(1)); + let (w, h) = (p.w, p.h); + let b = opts.border.min(w / 4).min(h / 4); + // Local maxima (3 × 3) above the quality threshold. + let mut cands: Vec<(f32, usize, usize)> = vec![]; + let mut best = 0.0f32; + for y in b..h.saturating_sub(b) { + for x in b..w.saturating_sub(b) { + let v = score[y * w + x]; + if v <= 1e-9 { + continue; + } + let mut is_max = true; + 'n: for dy in -1i64..=1 { + for dx in -1i64..=1 { + if dx == 0 && dy == 0 { + continue; + } + let (xx, yy) = (x as i64 + dx, y as i64 + dy); + if xx < 0 || yy < 0 || xx >= w as i64 || yy >= h as i64 { + continue; + } + let o = score[yy as usize * w + xx as usize]; + if o > v || (o == v && (dy < 0 || (dy == 0 && dx < 0))) { + is_max = false; + break 'n; + } + } + } + if !is_max { + continue; + } + let c = [x as f64 + 0.5, y as f64 + 0.5]; + if let Some(f) = inside + && !f(c) + { + continue; + } + best = best.max(v); + cands.push((v, x, y)); + } + } + let thr = best * opts.quality as f32; + cands.retain(|c| c.0 >= thr); + cands.sort_by(|a, b| b.0.total_cmp(&a.0).then(a.2.cmp(&b.2)).then(a.1.cmp(&b.1))); + // Minimum distance via a coarse occupancy grid. + let md = opts.min_distance.max(1.0); + let cell = md; + let gw = (w as f64 / cell).ceil() as usize + 1; + let gh = (h as f64 / cell).ceil() as usize + 1; + let mut grid: Vec> = vec![vec![]; gw * gh]; + let mut out = vec![]; + for (_, x, y) in cands { + let c = [x as f64 + 0.5, y as f64 + 0.5]; + let (cx, cy) = ((c[0] / cell) as usize, (c[1] / cell) as usize); + let mut ok = true; + 'g: for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + for q in &grid[yy * gw + xx] { + if (q[0] - c[0]).powi(2) + (q[1] - c[1]).powi(2) < md * md { + ok = false; + break 'g; + } + } + } + } + if ok { + grid[cy * gw + cx].push(c); + out.push(c); + if out.len() >= opts.max_features { + break; + } + } + } + out +} + +/// Lucas–Kanade options. +#[derive(Clone, Copy, Debug)] +pub struct LkOpts { + /// Window radius (the window is `2r + 1` square). + pub radius: i32, + pub iterations: usize, + /// Stop when the update is below this (pixels). + pub epsilon: f64, + /// Reject windows whose structure tensor's smaller eigenvalue (per pixel) is below this. + pub min_eigen: f64, + /// Forward–backward error limit (level-0 pixels); 0 = no check. + pub fb_max: f64, +} + +impl Default for LkOpts { + fn default() -> Self { + LkOpts { radius: 5, iterations: 20, epsilon: 0.01, min_eigen: 1e-5, fb_max: 1.0 } + } +} + +fn lk_point(a: &GrayPyramid, b: &GrayPyramid, p: [f64; 2], guess: [f64; 2], o: &LkOpts) -> Option<[f64; 2]> { + let top = a.levels.len().min(b.levels.len()).checked_sub(1)?; + let r = o.radius; + let n = ((2 * r + 1) * (2 * r + 1)) as f64; + let s0 = (1u32 << top) as f64; + let mut g = [guess[0] / s0, guess[1] / s0]; + let mut win = vec![0.0f32; ((2 * r + 1) * (2 * r + 1)) as usize]; + let mut wgx = win.clone(); + let mut wgy = win.clone(); + let mut v1 = [0.0f32]; + for l in (0..=top).rev() { + let s = (1u32 << l) as f64; + let (ia, ib) = (&a.levels[l], &b.levels[l]); + let (gx, gy) = &a.grads[l]; + let c = [p[0] / s, p[1] / s]; + if c[0] < -1.0 || c[1] < -1.0 || c[0] > ia.w as f64 + 1.0 || c[1] > ia.h as f64 + 1.0 { + return None; + } + let (mut gxx, mut gxy, mut gyy) = (0.0f64, 0.0f64, 0.0f64); + let mut k = 0; + for dy in -r..=r { + for dx in -r..=r { + let (x, y) = (c[0] + dx as f64, c[1] + dy as f64); + ia.sample(x, y, &mut v1); + win[k] = v1[0]; + gx.sample(x, y, &mut v1); + wgx[k] = v1[0]; + gy.sample(x, y, &mut v1); + wgy[k] = v1[0]; + let (ix, iy) = (wgx[k] as f64, wgy[k] as f64); + gxx += ix * ix; + gxy += ix * iy; + gyy += iy * iy; + k += 1; + } + } + let det = gxx * gyy - gxy * gxy; + let tr = 0.5 * (gxx + gyy); + let mineig = (tr - (0.25 * (gxx - gyy).powi(2) + gxy * gxy).sqrt()) / n; + if mineig < o.min_eigen || det.abs() < 1e-18 { + return None; + } + let mut v = [0.0f64; 2]; + for _ in 0..o.iterations { + let (mut bx, mut by) = (0.0f64, 0.0f64); + let mut k = 0; + for dy in -r..=r { + for dx in -r..=r { + ib.sample(c[0] + g[0] + v[0] + dx as f64, c[1] + g[1] + v[1] + dy as f64, &mut v1); + let diff = (win[k] - v1[0]) as f64; + bx += diff * wgx[k] as f64; + by += diff * wgy[k] as f64; + k += 1; + } + } + let nx = (gyy * bx - gxy * by) / det; + let ny = (gxx * by - gxy * bx) / det; + v[0] += nx; + v[1] += ny; + if !(v[0].is_finite() && v[1].is_finite()) { + return None; + } + if nx * nx + ny * ny < o.epsilon * o.epsilon { + break; + } + } + if l > 0 { + g = [2.0 * (g[0] + v[0]), 2.0 * (g[1] + v[1])]; + } else { + g = [g[0] + v[0], g[1] + v[1]]; + } + } + let q = [p[0] + g[0], p[1] + g[1]]; + let l0 = &b.levels[0]; + if q[0] < 0.0 || q[1] < 0.0 || q[0] > l0.w as f64 || q[1] > l0.h as f64 { + return None; + } + Some(q) +} + +/// Track level-0 points from pyramid `a` to pyramid `b` (with optional per-point initial +/// displacement guesses). `None` for lost points (untrackable window, left the frame, or failed +/// the forward–backward check). +pub fn track(a: &GrayPyramid, b: &GrayPyramid, pts: &[[f64; 2]], guesses: Option<&[[f64; 2]]>, o: &LkOpts) -> Vec> { + pts.par_iter() + .enumerate() + .map(|(i, p)| { + let g = guesses.and_then(|g| g.get(i).copied()).unwrap_or([0.0; 2]); + let q = lk_point(a, b, *p, g, o)?; + if o.fb_max > 0.0 { + let back = lk_point(b, a, q, [p[0] - q[0], p[1] - q[1]], o)?; + if (back[0] - p[0]).powi(2) + (back[1] - p[1]).powi(2) > o.fb_max * o.fb_max { + return None; + } + } + Some(q) + }) + .collect() +} + +/// Integer downsampling factor so the longer side is at most `max_side` pixels. +pub fn analysis_factor(w: u32, h: u32, max_side: u32) -> usize { + let long = w.max(h).max(1); + long.div_ceil(max_side.max(16)).max(1) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Value noise: smooth random blobs at two scales. + fn tex(x: f64, y: f64) -> f32 { + fn h(i: i64, j: i64) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64, cell: f64) -> f64 { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor(), fy.floor()); + let (tx, ty) = (fx - i, fy - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = h(i, j) + (h(i + 1, j) - h(i, j)) * sx; + let b = h(i, j + 1) + (h(i + 1, j + 1) - h(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.2 + 0.4 * noise(x + 1000.0, y + 1000.0, 9.0) + 0.3 * noise(x + 1000.0, y + 1000.0, 4.0)) as f32 + } + + fn img(dx: f64, dy: f64) -> Image { + let mut im = Image::new(160, 120); + for y in 0..120 { + for x in 0..160 { + let v = tex(x as f64 + 0.5 - dx, y as f64 + 0.5 - dy); + im.set(x, y, [v, v, v, 1.0]); + } + } + im + } + + #[test] + fn klt_recovers_a_shift() { + let a = GrayPyramid::from_image(&img(0.0, 0.0), [0.0; 2], 1, 3); + let b = GrayPyramid::from_image(&img(6.3, -4.2), [0.0; 2], 1, 3); + let pts = good_features(&a, &CornerOpts { max_features: 60, ..Default::default() }, None); + assert!(pts.len() >= 20, "{}", pts.len()); + let q = track(&a, &b, &pts, None, &LkOpts::default()); + let good: Vec<_> = pts.iter().zip(&q).filter_map(|(p, q)| q.map(|q| [q[0] - p[0], q[1] - p[1]])).collect(); + assert!(good.len() * 2 > pts.len(), "{} of {}", good.len(), pts.len()); + let mut errs: Vec = good.iter().map(|d| ((d[0] - 6.3).powi(2) + (d[1] + 4.2).powi(2)).sqrt()).collect(); + errs.sort_by(f64::total_cmp); + assert!(errs[errs.len() / 2] < 0.1, "median error {}", errs[errs.len() / 2]); + } + + #[test] + fn features_respect_the_region_and_spacing() { + let a = GrayPyramid::from_image(&img(0.0, 0.0), [0.0; 2], 1, 2); + let inside = |p: [f64; 2]| p[0] < 80.0; + let pts = good_features(&a, &CornerOpts { min_distance: 10.0, ..Default::default() }, Some(&inside)); + assert!(!pts.is_empty()); + assert!(pts.iter().all(|p| p[0] < 80.0)); + for (i, p) in pts.iter().enumerate() { + for q in &pts[i + 1..] { + assert!((p[0] - q[0]).hypot(p[1] - q[1]) >= 10.0 - 1e-9); + } + } + assert_eq!(analysis_factor(1920, 1080, 640), 3); + } +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d106b2096..d7b239f5f 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,8 +18,12 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod fit; +pub mod klt; +pub mod mask; pub mod plane; pub mod solve; +pub mod stabilize; use effectcraft_raster::Image; use rayon::prelude::*; diff --git a/crates/track/src/mask.rs b/crates/track/src/mask.rs new file mode 100644 index 000000000..e1e133409 --- /dev/null +++ b/crates/track/src/mask.rs @@ -0,0 +1,222 @@ +//! Mask tracking: follow the pixels inside a mask from frame to frame and report the motion as a +//! 2D transform (translation, + scale, + rotation, + skew, or perspective), which the caller +//! applies to the mask's vertices and tangents. +//! +//! Each step detects Shi–Tomasi features inside the current mask outline ([`crate::klt`]), tracks +//! them into the next frame with pyramidal Lucas–Kanade and a forward–backward check, and fits the +//! chosen model robustly with RANSAC ([`crate::fit`]), falling back to simpler models when too few +//! features survive. Only a crop around the mask (plus a margin for the motion) is analysed. + +use effectcraft_raster::Image; + +use crate::Frame; +use crate::fit::{Model, ransac}; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, luma_plane, track}; +use crate::solve::Homography; + +/// Point-in-polygon (even–odd rule). +pub fn inside_polygon(poly: &[[f64; 2]], p: [f64; 2]) -> bool { + let n = poly.len(); + if n < 3 { + return false; + } + let mut c = false; + let mut j = n - 1; + for i in 0..n { + let (a, b) = (poly[i], poly[j]); + if (a[1] > p[1]) != (b[1] > p[1]) && p[0] < (b[0] - a[0]) * (p[1] - a[1]) / (b[1] - a[1]) + a[0] { + c = !c; + } + j = i; + } + c +} + +/// Bounding box `[x0, y0, x1, y1]` of points. +pub fn bounds(p: &[[f64; 2]]) -> [f64; 4] { + let mut b = [f64::MAX, f64::MAX, f64::MIN, f64::MIN]; + for q in p { + b[0] = b[0].min(q[0]); + b[1] = b[1].min(q[1]); + b[2] = b[2].max(q[0]); + b[3] = b[3].max(q[1]); + } + b +} + +/// Mask tracker state. +#[derive(Clone, Debug)] +pub struct MaskTracker { + pub model: Model, + /// The mask outline (layer pixels) on the last tracked frame. + pub outline: Vec<[f64; 2]>, + /// The last frame-to-frame motion (a constant-velocity guess for the next frame). + velocity: Homography, + prev: Option, + prev_offset: [f64; 2], + /// Features used on the last step (layer pixels), for display and tests. + pub last_features: usize, +} + +/// One mask tracking step's outcome. +#[derive(Clone, Debug)] +pub struct MaskStep { + /// Motion from the previous frame to this one (layer pixels). + pub motion: Homography, + /// Correspondences that agreed with the motion. + pub inliers: usize, + /// The model actually fitted (may be simpler than requested when features are scarce). + pub model: Model, +} + +impl MaskTracker { + /// Start on `frame` with the mask `outline` (a closed polyline in layer pixels). + pub fn new(model: Model, outline: Vec<[f64; 2]>, frame: &Frame) -> MaskTracker { + MaskTracker { model, outline, velocity: Homography::IDENTITY, prev: Some(frame.img.clone()), prev_offset: frame.offset, last_features: 0 } + } + + /// Track into `frame` (the next frame in either direction). `None` when the mask's pixels + /// can't be followed (no texture, or the mask left the frame). + pub fn step(&mut self, frame: &Frame) -> Option { + let prev = self.prev.take()?; + let res = self.step_inner(&prev, self.prev_offset, frame); + self.prev = Some(frame.img.clone()); + self.prev_offset = frame.offset; + let r = res?; + self.outline = self.outline.iter().map(|p| r.motion.apply(*p)).collect(); + self.velocity = r.motion; + Some(r) + } + + fn step_inner(&mut self, prev: &Image, prev_off: [f64; 2], frame: &Frame) -> Option { + let b = bounds(&self.outline); + if !(b[0].is_finite() && b[2] > b[0] && b[3] > b[1]) { + return None; + } + let (bw, bh) = (b[2] - b[0], b[3] - b[1]); + // Crop region (layer pixels): the mask plus room for its motion. + let margin = (0.35 * bw.max(bh)).max(32.0); + let (lx0, ly0, lx1, ly1) = (b[0] - margin, b[1] - margin, b[2] + margin, b[3] + margin); + // Both frames share one plane coordinate system: crop in the frames' image pixels. + let crop = |img: &Image, off: [f64; 2]| { + let x0 = (lx0 + off[0]).floor().max(0.0) as i64; + let y0 = (ly0 + off[1]).floor().max(0.0) as i64; + let x1 = ((lx1 + off[0]).ceil() as i64).min(img.width as i64); + let y1 = ((ly1 + off[1]).ceil() as i64).min(img.height as i64); + if x1 <= x0 + 8 || y1 <= y0 + 8 { + return None; + } + Some((img.crop(x0, y0, (x1 - x0) as u32, (y1 - y0) as u32), [off[0] - x0 as f64, off[1] - y0 as f64])) + }; + let (ca, oa) = crop(prev, prev_off)?; + let (cb, ob) = crop(frame.img, frame.offset)?; + let factor = analysis_factor(ca.width.max(cb.width), ca.height.max(cb.height), 720); + // Pyramids must share coordinates: use a common origin (layer pixels → plane). + let levels = 3; + let pa = GrayPyramid::from_plane(luma_plane(&ca, factor), oa, factor as f64, levels); + let pb_raw = luma_plane(&cb, factor); + // Shift frame b's crop into frame a's plane coordinates when the offsets differ. + let pb = if (oa[0] - ob[0]).abs() < 1e-9 && (oa[1] - ob[1]).abs() < 1e-9 && ca.width == cb.width && ca.height == cb.height { + GrayPyramid::from_plane(pb_raw, oa, factor as f64, levels) + } else { + let mut q = crate::plane::Plane::new(pa.levels[0].w, pa.levels[0].h, 1); + let mut v = [0.0f32]; + for y in 0..q.h { + for x in 0..q.w { + // plane a (x, y) → layer → plane b. + let l = pa.to_layer([x as f64 + 0.5, y as f64 + 0.5]); + let pbp = [(l[0] + ob[0]) / factor as f64, (l[1] + ob[1]) / factor as f64]; + pb_raw.sample(pbp[0], pbp[1], &mut v); + q.data[y * q.w + x] = v[0]; + } + } + GrayPyramid::from_plane(q, oa, factor as f64, levels) + }; + let outline: Vec<[f64; 2]> = self.outline.iter().map(|p| pa.to_plane(*p)).collect(); + let area = bw * bh / (factor * factor) as f64; + let md = (area / 500.0).sqrt().clamp(3.0, 12.0); + let inside = |p: [f64; 2]| inside_polygon(&outline, p); + let feats = good_features(&pa, &CornerOpts { max_features: 500, min_distance: md, quality: 0.005, window: 2, border: 3 }, Some(&inside)); + self.last_features = feats.len(); + if feats.len() < 3 { + return None; + } + // Constant-velocity guess. + let guesses: Vec<[f64; 2]> = feats + .iter() + .map(|p| { + let l = pa.to_layer(*p); + let m = self.velocity.apply(l); + [(m[0] - l[0]) / factor as f64, (m[1] - l[1]) / factor as f64] + }) + .collect(); + let tracked = track(&pa, &pb, &feats, Some(&guesses), &LkOpts { radius: 6, fb_max: 0.75, ..Default::default() }); + let (src, dst): (Vec<[f64; 2]>, Vec<[f64; 2]>) = feats.iter().zip(&tracked).filter_map(|(p, q)| q.map(|q| (pa.to_layer(*p), pa.to_layer(q)))).unzip(); + let thr = 1.5 * factor as f64; + let chain: &[Model] = match self.model { + Model::Homography => &[Model::Homography, Model::Affine, Model::Similarity, Model::Translation], + Model::Affine => &[Model::Affine, Model::Similarity, Model::Translation], + Model::Similarity => &[Model::Similarity, Model::Translation], + Model::TranslationScale => &[Model::TranslationScale, Model::Translation], + Model::Translation => &[Model::Translation], + }; + for m in chain { + // Enough support for the model's freedom. + let need = (m.min_points() * 3).max(3); + if src.len() < need { + continue; + } + if let Some(f) = ransac(*m, &src, &dst, thr, 400, 0x5eed + src.len() as u64) { + let c = f.inlier_count(); + if c >= need && c * 3 >= src.len() { + return Some(MaskStep { motion: f.h, inliers: c, model: *m }); + } + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tex(x: f64, y: f64) -> f32 { + let v = (x * 0.37).sin() * (y * 0.29).cos() + 0.6 * ((x + 1.7 * y) * 0.21).sin() + 0.4 * ((x * 0.83 - y * 0.61).sin()); + (0.5 + 0.18 * v) as f32 + } + + /// A textured disk of radius 50 posed by `h` (object → layer) over flat grey. + fn frame(h: &Homography) -> Image { + let inv = h.inverse().unwrap(); + let mut im = Image::new(320, 240); + for y in 0..240 { + for x in 0..320 { + let o = inv.apply([x as f64 + 0.5, y as f64 + 0.5]); + let v = if o[0].hypot(o[1]) <= 50.0 { tex(o[0], o[1]) } else { 0.3 }; + im.set(x, y, [v, v, v, 1.0]); + } + } + im + } + + #[test] + fn follows_a_rotating_scaling_disk() { + let pose = |f: usize| { + let a = (f as f64 * 4.0).to_radians(); + let s = 1.0 + 0.03 * f as f64; + Homography([[s * a.cos(), -s * a.sin(), 140.0 + 4.0 * f as f64], [s * a.sin(), s * a.cos(), 110.0 + 2.0 * f as f64], [0.0, 0.0, 1.0]]) + }; + let ring: Vec<[f64; 2]> = (0..24).map(|i| (i as f64 * 15.0).to_radians()).map(|a| [40.0 * a.cos(), 40.0 * a.sin()]).collect(); + let f0 = frame(&pose(0)); + let mut mt = MaskTracker::new(Model::Similarity, ring.iter().map(|p| pose(0).apply(*p)).collect(), &Frame::new(&f0)); + for f in 1..8 { + let img = frame(&pose(f)); + let st = mt.step(&Frame::new(&img)).expect("tracked"); + assert!(st.inliers > 10); + let want: Vec<[f64; 2]> = ring.iter().map(|p| pose(f).apply(*p)).collect(); + let err = mt.outline.iter().zip(&want).map(|(a, b)| (a[0] - b[0]).hypot(a[1] - b[1])).fold(0.0, f64::max); + assert!(err < 1.5, "frame {f}: max vertex error {err}"); + } + } +} diff --git a/crates/track/src/solve.rs b/crates/track/src/solve.rs index bf3f2eb54..6c2677210 100644 --- a/crates/track/src/solve.rs +++ b/crates/track/src/solve.rs @@ -47,6 +47,89 @@ impl Homography { [(m[0][0] * p[0] + m[0][1] * p[1] + m[0][2]) / w, (m[1][0] * p[0] + m[1][1] * p[1] + m[1][2]) / w] } + /// `self ∘ other` (apply `other` first). + pub fn then_after(&self, other: &Homography) -> Homography { + Homography(mul(self.0, other.0)).normalized() + } + + /// Composition `b ∘ a`: apply `a`, then `b`. + pub fn compose(a: &Homography, b: &Homography) -> Homography { + b.then_after(a) + } + + /// Scale so that `h33 = 1` (when it isn't ≈ 0). + pub fn normalized(self) -> Homography { + let s = self.0[2][2]; + if s.abs() < 1e-12 || (s - 1.0).abs() < 1e-15 { + return self; + } + Homography(self.0.map(|r| r.map(|v| v / s))) + } + + /// The inverse map (`None` when singular). + pub fn inverse(&self) -> Option { + let m = &self.0; + let c00 = m[1][1] * m[2][2] - m[1][2] * m[2][1]; + let c01 = m[1][2] * m[2][0] - m[1][0] * m[2][2]; + let c02 = m[1][0] * m[2][1] - m[1][1] * m[2][0]; + let det = m[0][0] * c00 + m[0][1] * c01 + m[0][2] * c02; + if det.abs() < 1e-18 || !det.is_finite() { + return None; + } + let inv = [ + [c00, m[0][2] * m[2][1] - m[0][1] * m[2][2], m[0][1] * m[1][2] - m[0][2] * m[1][1]], + [c01, m[0][0] * m[2][2] - m[0][2] * m[2][0], m[0][2] * m[1][0] - m[0][0] * m[1][2]], + [c02, m[0][1] * m[2][0] - m[0][0] * m[2][1], m[0][0] * m[1][1] - m[0][1] * m[1][0]], + ]; + Some(Homography(inv.map(|r| r.map(|v| v / det))).normalized()) + } + + /// Translation by `t`. + pub fn translation(t: [f64; 2]) -> Homography { + Homography([[1.0, 0.0, t[0]], [0.0, 1.0, t[1]], [0.0, 0.0, 1.0]]) + } + + /// Uniform scale `k` about `c`. + pub fn scale_about(k: f64, c: [f64; 2]) -> Homography { + Homography([[k, 0.0, c[0] * (1.0 - k)], [0.0, k, c[1] * (1.0 - k)], [0.0, 0.0, 1.0]]) + } + + /// `(1 - t) · self + t · other` element-wise (both normalised to `h33 = 1`): blends between + /// nearby transforms. + pub fn lerp(&self, other: &Homography, t: f64) -> Homography { + let (a, b) = (self.normalized().0, other.normalized().0); + let mut o = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + o[i][j] = a[i][j] + (b[i][j] - a[i][j]) * t; + } + } + Homography(o) + } + + /// Map a Bezier tangent (relative to vertex `v`) through the transform: the tangent's end + /// point is mapped and made relative to the mapped vertex. + pub fn apply_tangent(&self, v: [f64; 2], tangent: [f64; 2]) -> [f64; 2] { + if tangent == [0.0, 0.0] { + return tangent; + } + let a = self.apply(v); + let b = self.apply([v[0] + tangent[0], v[1] + tangent[1]]); + [b[0] - a[0], b[1] - a[1]] + } + + /// Largest absolute difference of the matrices (normalised). + pub fn max_diff(&self, other: &Homography) -> f64 { + let (a, b) = (self.normalized().0, other.normalized().0); + let mut d: f64 = 0.0; + for i in 0..3 { + for j in 0..3 { + d = d.max((a[i][j] - b[i][j]).abs()); + } + } + d + } + /// The homography taking `src[i]` to `dst[i]` (four points, no three collinear). pub fn from_points(src: [[f64; 2]; 4], dst: [[f64; 2]; 4]) -> Option { // Normalise both sets (centroid at 0, mean distance √2) for conditioning. @@ -114,7 +197,7 @@ fn sub(a: [f64; 2], b: [f64; 2]) -> [f64; 2] { [a[0] - b[0], a[1] - b[1]] } -fn mul(a: [[f64; 3]; 3], b: [[f64; 3]; 3]) -> [[f64; 3]; 3] { +pub(crate) fn mul(a: [[f64; 3]; 3], b: [[f64; 3]; 3]) -> [[f64; 3]; 3] { let mut o = [[0.0; 3]; 3]; for i in 0..3 { for j in 0..3 { diff --git a/crates/track/src/stabilize.rs b/crates/track/src/stabilize.rs new file mode 100644 index 000000000..1abe79f9a --- /dev/null +++ b/crates/track/src/stabilize.rs @@ -0,0 +1,576 @@ +//! Video stabilization (the Warp Stabilizer's analysis and stabilization steps), clean-room from +//! the published approach of 2D stabilizers: +//! +//! 1. **Analysis** ([`Analyzer`]): Shi–Tomasi features are tracked between consecutive frames +//! with pyramidal Lucas–Kanade ([`crate::klt`]), and the dominant (background) motion between +//! each pair of frames is fitted robustly with RANSAC ([`crate::fit`]) for three models at +//! once: translation, similarity and homography. The result ([`WarpAnalysis`]) is small and +//! serialisable; it is stored in the effect instance. +//! 2. **Stabilization** ([`plan`]): the camera path is smoothed and per-frame corrective +//! transforms are derived. For *Smooth Motion* each frame is warped to the Gaussian-weighted +//! average of its neighbours' viewpoints — the motion-smoothing scheme of Matsushita et al., +//! "Full-Frame Video Stabilization with Motion Inpainting" (PAMI 2006) — so the low-frequency +//! camera move stays and the jitter goes; *No Motion* maps every frame onto one reference +//! frame. Framing then crops to the region valid on every frame (a centred rectangle inside +//! every warped frame outline) and optionally scales it up to fill the frame; when that would +//! exceed Maximum Scale, the correction is relaxed towards identity on the offending frames — +//! trading smoothness for less crop, the constraint idea of Grundmann et al., "Auto-Directed +//! Video Stabilization with Robust L1 Optimal Camera Paths" (CVPR 2011), applied here +//! per frame rather than through a linear program. +//! +//! Coordinates are layer pixels; transforms map *source* frame pixels to *stabilized* pixels. + +use serde::{Deserialize, Serialize}; + +use crate::Frame; +use crate::fit::{Model, ransac}; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; +use crate::solve::Homography; + +/// Inter-frame motion of one frame (from the previous frame to this one) for each model. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct FrameMotion { + /// Translation `[tx, ty]`. + pub t: [f64; 2], + /// Similarity `[a, b, tx, ty]` (`x' = a x − b y + tx`, `y' = b x + a y + ty`). + pub s: [f64; 4], + /// Homography, row-major without `h33 = 1`. + pub h: [f64; 8], + /// Correspondences that agreed with the homography / tracked in total. + pub inliers: u32, + pub features: u32, + /// A subset of this frame's tracked features (layer pixels), for Show Track Points. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub points: Vec<[f32; 2]>, +} + +impl Default for FrameMotion { + fn default() -> Self { + FrameMotion { t: [0.0; 2], s: [1.0, 0.0, 0.0, 0.0], h: [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], inliers: 0, features: 0, points: vec![] } + } +} + +impl FrameMotion { + /// The motion under a stabilization method. + pub fn motion(&self, method: Method) -> Homography { + match method { + Method::Position => Homography::translation(self.t), + Method::Similarity => { + let s = self.s; + Homography([[s[0], -s[1], s[2]], [s[1], s[0], s[3]], [0.0, 0.0, 1.0]]) + } + Method::Perspective | Method::SubspaceWarp => { + let h = self.h; + Homography([[h[0], h[1], h[2]], [h[3], h[4], h[5]], [h[6], h[7], 1.0]]) + } + } + } +} + +/// The stored analysis of a clip. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct WarpAnalysis { + pub version: u32, + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + /// Layer size (pixels). + pub size: [f64; 2], + /// Detailed Analysis was on. + pub detailed: bool, + /// Per frame; frame 0's motion is the identity. + pub frames: Vec, +} + +impl WarpAnalysis { + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } +} + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental analysis: push the clip's frames in order. +pub struct Analyzer { + opts: AnalyzeOpts, + prev: Option, + frames: Vec, + size: [f64; 2], +} + +fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { + let step = (p.len() / n.max(1)).max(1); + p.iter().step_by(step).take(n).map(|q| [((q[0] * 10.0).round() / 10.0) as f32, ((q[1] * 10.0).round() / 10.0) as f32]).collect() +} + +impl Analyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> Analyzer { + Analyzer { opts, prev: None, frames: vec![], size } + } + + pub fn len(&self) -> usize { + self.frames.len() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, max_feat) = if self.opts.detailed { (960, 700) } else { (480, 350) }; + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 3); + let mut fm = FrameMotion::default(); + if let Some(prev) = &self.prev + && prev.width() == pyr.width() + && prev.height() == pyr.height() + && prev.offset == pyr.offset + { + let md = (prev.width().max(prev.height()) as f64 / 40.0).clamp(4.0, 16.0); + let feats = good_features(prev, &CornerOpts { max_features: max_feat, min_distance: md, quality: 0.01, window: 2, border: 4 }, None); + let q = track(prev, &pyr, &feats, None, &LkOpts { radius: 5, fb_max: 0.75, ..Default::default() }); + let (src, dst): (Vec<[f64; 2]>, Vec<[f64; 2]>) = feats.iter().zip(&q).filter_map(|(p, q)| q.map(|q| (prev.to_layer(*p), pyr.to_layer(q)))).unzip(); + fm.features = src.len() as u32; + let thr = 1.0 * factor as f64; + let seed = self.frames.len() as u64 + 1; + if let Some(f) = ransac(Model::Translation, &src, &dst, thr * 1.5, 200, seed) { + let m = f.h.0; + fm.t = [m[0][2], m[1][2]]; + } + if let Some(f) = ransac(Model::Similarity, &src, &dst, thr * 1.5, 300, seed) { + let m = f.h.0; + fm.s = [m[0][0], m[1][0], m[0][2], m[1][2]]; + } + if let Some(f) = ransac(Model::Homography, &src, &dst, thr, 500, seed) { + let m = f.h.normalized().0; + // Reject wild projective fits (few features): keep the similarity then. + let sane = m[2][0].abs() < 0.01 && m[2][1].abs() < 0.01; + fm.h = if sane { + [m[0][0], m[0][1], m[0][2], m[1][0], m[1][1], m[1][2], m[2][0], m[2][1]] + } else { + [fm.s[0], -fm.s[1], fm.s[2], fm.s[1], fm.s[0], fm.s[3], 0.0, 0.0] + }; + fm.inliers = f.inlier_count() as u32; + let inl: Vec<[f64; 2]> = dst.iter().zip(&f.inliers).filter(|(_, b)| **b).map(|(p, _)| *p).collect(); + fm.points = round_pts(&inl, 48); + } + } + self.frames.push(fm); + self.prev = Some(pyr); + } + + pub fn finish(self, start: f64, frame_duration: f64) -> WarpAnalysis { + WarpAnalysis { version: 1, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames } + } +} + +// ---------------------------------------------------------------- stabilization + +/// Stabilization ▸ Result. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum StabResult { + #[default] + SmoothMotion, + NoMotion, +} + +/// Stabilization ▸ Method. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Method { + Position, + /// Position, Scale, Rotation. + Similarity, + Perspective, + /// Approximated by Perspective (see the module docs of the effect). + #[default] + SubspaceWarp, +} + +/// Borders ▸ Framing. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Framing { + StabilizeOnly, + StabilizeCrop, + #[default] + StabilizeCropAutoScale, + SynthesizeEdges, +} + +/// Effect parameters that shape the stabilization. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StabSettings { + pub result: StabResult, + /// Smoothness, percent (50 = default). + pub smoothness: f64, + pub method: Method, + pub preserve_scale: bool, + pub framing: Framing, + /// Maximum Scale, percent. + pub max_scale: f64, + /// Action Safe Margin, percent of the frame on each side. + pub action_safe: f64, + /// Additional Scale, percent. + pub additional_scale: f64, + /// Crop Less ↔ Smooth More (−100…100). + pub crop_less_smooth_more: f64, + /// Frames per second (for Smoothness). + pub fps: f64, +} + +impl Default for StabSettings { + fn default() -> Self { + StabSettings { + result: StabResult::SmoothMotion, + smoothness: 50.0, + method: Method::SubspaceWarp, + preserve_scale: false, + framing: Framing::StabilizeCropAutoScale, + max_scale: 150.0, + action_safe: 0.0, + additional_scale: 100.0, + crop_less_smooth_more: 0.0, + fps: 30.0, + } + } +} + +impl StabSettings { + /// Gaussian smoothing sigma in frames. + pub fn sigma(&self) -> f64 { + let base = self.smoothness.max(0.0) / 100.0 * self.fps.max(1.0) * 0.6; + base * 2f64.powf(self.crop_less_smooth_more.clamp(-100.0, 100.0) / 100.0) + } +} + +/// The per-frame stabilization of a clip. +#[derive(Clone, Debug, Default)] +pub struct Plan { + /// Final source → output transform per frame (layer pixels), framing scale included. + pub warps: Vec, + /// The visible region in output pixels `[x0, y0, x1, y1]` (Stabilize, Crop / Auto-scale); + /// `None` = everything the warp produces. + pub crop: Option<[f64; 4]>, + /// Auto-scale factor (1 = none). + pub auto_scale: f64, + /// Fraction of the frame (per side length) valid on every frame before scaling. + pub valid_fraction: f64, +} + +fn convex_contains(q: &[[f64; 2]; 4], p: [f64; 2]) -> bool { + let mut sign = 0.0; + for i in 0..4 { + let (a, b) = (q[i], q[(i + 1) % 4]); + let c = (b[0] - a[0]) * (p[1] - a[1]) - (b[1] - a[1]) * (p[0] - a[0]); + if c.abs() < 1e-12 { + continue; + } + if sign == 0.0 { + sign = c.signum(); + } else if c.signum() != sign { + return false; + } + } + true +} + +/// Largest `s ∈ [0, 1]` such that the centred `s·W × s·H` rectangle lies inside the frame +/// outline mapped by `b`. +pub fn valid_fraction(b: &Homography, size: [f64; 2]) -> f64 { + let [w, h] = size; + let quad = [b.apply([0.0, 0.0]), b.apply([w, 0.0]), b.apply([w, h]), b.apply([0.0, h])]; + // A fold or a point at infinity: nothing is valid. + let m = b.normalized().0; + for p in [[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]] { + if m[2][0] * p[0] + m[2][1] * p[1] + m[2][2] <= 1e-9 { + return 0.0; + } + } + let c = [w / 2.0, h / 2.0]; + let fits = |s: f64| { + let (hw, hh) = (s * w / 2.0, s * h / 2.0); + [[c[0] - hw, c[1] - hh], [c[0] + hw, c[1] - hh], [c[0] + hw, c[1] + hh], [c[0] - hw, c[1] + hh]].iter().all(|p| convex_contains(&quad, *p)) + }; + if fits(1.0) { + return 1.0; + } + if !fits(0.0) { + return 0.0; + } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..30 { + let mid = 0.5 * (lo + hi); + if fits(mid) { + lo = mid; + } else { + hi = mid; + } + } + lo +} + +fn remove_scale(b: &Homography, c: [f64; 2]) -> Homography { + // Local scale at the frame centre from the Jacobian of the map. + let e = 1.0; + let p0 = b.apply(c); + let px = b.apply([c[0] + e, c[1]]); + let py = b.apply([c[0], c[1] + e]); + let det = ((px[0] - p0[0]) * (py[1] - p0[1]) - (px[1] - p0[1]) * (py[0] - p0[0])).abs(); + let s = det.sqrt(); + if s < 1e-9 { + return *b; + } + Homography::scale_about(1.0 / s, p0).then_after(b) +} + +/// Corrective transforms (source → stabilized, before framing) for every frame. +pub fn corrections(a: &WarpAnalysis, s: &StabSettings) -> Vec { + let n = a.frames.len(); + if n == 0 { + return vec![]; + } + let method = s.method; + let inter: Vec = a.frames.iter().map(|f| f.motion(method)).collect(); + let inv: Vec = inter.iter().map(|h| h.inverse().unwrap_or(Homography::IDENTITY)).collect(); + let c = [a.size[0] / 2.0, a.size[1] / 2.0]; + let mut out = Vec::with_capacity(n); + match s.result { + StabResult::NoMotion => { + // Cumulative path C_t (frame 0 → frame t); every frame maps onto the middle frame. + let mut cum = Vec::with_capacity(n); + let mut cur = Homography::IDENTITY; + for (k, h) in inter.iter().enumerate() { + if k > 0 { + cur = h.then_after(&cur); + } + cum.push(cur); + } + let r = n / 2; + for c_t in &cum { + let ci = c_t.inverse().unwrap_or(Homography::IDENTITY); + out.push(cum[r].then_after(&ci)); + } + } + StabResult::SmoothMotion => { + let sigma = s.sigma(); + let rad = ((3.0 * sigma).ceil() as usize).min(n.saturating_sub(1)); + for t in 0..n { + if sigma < 0.3 || rad == 0 { + out.push(Homography::IDENTITY); + continue; + } + let mut acc = [[0.0; 3]; 3]; + let mut wsum = 0.0; + let mut add = |h: &Homography, d: f64| { + let w = (-(d * d) / (2.0 * sigma * sigma)).exp(); + let m = h.normalized().0; + for i in 0..3 { + for j in 0..3 { + acc[i][j] += w * m[i][j]; + } + } + wsum += w; + }; + add(&Homography::IDENTITY, 0.0); + // Forward: T_t^{i+1} = inter[i+1] ∘ T_t^i. + let mut tf = Homography::IDENTITY; + for i in t + 1..=(t + rad).min(n - 1) { + tf = inter[i].then_after(&tf); + add(&tf, (i - t) as f64); + } + // Backward: T_t^{i-1} = inter[i]⁻¹ ∘ T_t^i. + let mut tb = Homography::IDENTITY; + for i in (t.saturating_sub(rad)..t).rev() { + tb = inv[i + 1].then_after(&tb); + add(&tb, (t - i) as f64); + } + let m = acc.map(|r| r.map(|v| v / wsum)); + out.push(Homography(m).normalized()); + } + } + } + if s.preserve_scale && method != Method::Position { + out = out.iter().map(|b| remove_scale(b, c)).collect(); + } + out +} + +/// Stabilization plan: corrections plus framing. +pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { + let mut b = corrections(a, s); + let size = a.size; + let c = [size[0] / 2.0, size[1] / 2.0]; + let add = (s.additional_scale / 100.0).max(0.01); + let z_add = Homography::scale_about(add, c); + let fracs = |b: &[Homography]| b.iter().map(|h| valid_fraction(h, size)).collect::>(); + let mut fr = fracs(&b); + let mut vmin = fr.iter().copied().fold(1.0, f64::min); + let margin = (s.action_safe / 100.0).clamp(0.0, 0.45); + let need_cover = 1.0 - 2.0 * margin; + let max_scale = (s.max_scale / 100.0).max(1.0); + if s.framing == Framing::StabilizeCropAutoScale && vmin * max_scale < need_cover && !b.is_empty() { + // Relax the correction on frames that need more than Maximum Scale. + let target = need_cover / max_scale; + let n = b.len(); + let mut lam = vec![1.0f64; n]; + for t in 0..n { + if fr[t] >= target { + continue; + } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if valid_fraction(&Homography::IDENTITY.lerp(&b[t], mid), size) >= target { + lo = mid; + } else { + hi = mid; + } + } + lam[t] = lo; + } + // Spread the relaxation smoothly (min filter, then a Gaussian of the same radius) so + // the camera doesn't jump. + let r = ((s.sigma() * 0.5).ceil() as usize).max(1); + let minf: Vec = (0..n).map(|t| lam[t.saturating_sub(r)..=(t + r).min(n - 1)].iter().copied().fold(1.0, f64::min)).collect(); + let sig = r as f64 / 2.0; + let smooth: Vec = (0..n) + .map(|t| { + let (mut acc, mut ws) = (0.0, 0.0); + for i in t.saturating_sub(r)..=(t + r).min(n - 1) { + let d = i as f64 - t as f64; + let w = (-(d * d) / (2.0 * sig * sig)).exp(); + acc += w * minf[i]; + ws += w; + } + (acc / ws).min(lam[t]) + }) + .collect(); + b = b.iter().zip(&smooth).map(|(h, l)| if *l >= 1.0 { *h } else { Homography::IDENTITY.lerp(h, *l) }).collect(); + fr = fracs(&b); + vmin = fr.iter().copied().fold(1.0, f64::min); + } + let rect = |k: f64| [c[0] - k * size[0] / 2.0, c[1] - k * size[1] / 2.0, c[0] + k * size[0] / 2.0, c[1] + k * size[1] / 2.0]; + let (auto, crop) = match s.framing { + Framing::StabilizeOnly | Framing::SynthesizeEdges => (1.0, None), + Framing::StabilizeCrop => (1.0, Some(rect(vmin * add))), + Framing::StabilizeCropAutoScale => { + let k = if vmin > 1e-6 { (need_cover / vmin).clamp(1.0, max_scale) } else { max_scale }; + let vis = vmin * k * add; + (k, if vis >= 1.0 - 1e-9 { None } else { Some(rect(vis)) }) + } + }; + let z = Homography::scale_about(auto, c); + let warps = b.iter().map(|h| z_add.then_after(&z.then_after(h))).collect(); + Plan { warps, crop, auto_scale: auto, valid_fraction: vmin } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A synthetic analysis: a slow pan plus per-frame jitter. + fn shaky(n: usize) -> (WarpAnalysis, Vec<[f64; 2]>) { + let pos = |k: usize| { + let j = [((k * 7919) % 13) as f64 - 6.0, ((k * 104729) % 11) as f64 - 5.0]; + [2.0 * k as f64 + j[0], 0.5 * k as f64 + j[1]] + }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![] }; + let path: Vec<[f64; 2]> = (0..n).map(pos).collect(); + for k in 0..n { + let mut f = FrameMotion::default(); + if k > 0 { + // Scene points move opposite to the camera. + let d = [path[k - 1][0] - path[k][0], path[k - 1][1] - path[k][1]]; + f.t = d; + f.s = [1.0, 0.0, d[0], d[1]]; + f.h = [1.0, 0.0, d[0], 0.0, 1.0, d[1], 0.0, 0.0]; + } + a.frames.push(f); + } + (a, path) + } + + fn jitter(p: &[[f64; 2]]) -> f64 { + // Mean magnitude of the second difference (acceleration). + let mut s = 0.0; + for k in 1..p.len() - 1 { + s += (p[k + 1][0] - 2.0 * p[k][0] + p[k - 1][0]).hypot(p[k + 1][1] - 2.0 * p[k][1] + p[k - 1][1]); + } + s / (p.len() - 2) as f64 + } + + #[test] + fn smoothing_removes_jitter_and_no_motion_locks() { + let (a, path) = shaky(90); + // A scene point at (320, 180) on frame 0 appears at (320, 180) - path[k] on frame k. + let scene = |k: usize| [320.0 - path[k][0] + path[0][0], 180.0 - path[k][1] + path[0][1]]; + let before: Vec<[f64; 2]> = (0..90).map(scene).collect(); + for method in [Method::Position, Method::Similarity, Method::Perspective] { + let s = StabSettings { method, framing: Framing::StabilizeOnly, ..Default::default() }; + let p = plan(&a, &s); + let after: Vec<[f64; 2]> = (0..90).map(|k| p.warps[k].apply(scene(k))).collect(); + let (j0, j1) = (jitter(&before), jitter(&after)); + assert!(j1 < 0.2 * j0, "{method:?}: jitter {j0} → {j1}"); + } + let s = StabSettings { result: StabResult::NoMotion, framing: Framing::StabilizeOnly, ..Default::default() }; + let p = plan(&a, &s); + let fixed: Vec<[f64; 2]> = (0..90).map(|k| p.warps[k].apply(scene(k))).collect(); + for q in &fixed { + assert!((q[0] - fixed[45][0]).abs() < 1e-6 && (q[1] - fixed[45][1]).abs() < 1e-6); + } + } + + #[test] + fn framing_crops_and_scales() { + let (a, _) = shaky(60); + let p = plan(&a, &StabSettings { framing: Framing::StabilizeCrop, ..Default::default() }); + let crop = p.crop.unwrap(); + assert!(p.auto_scale == 1.0 && crop[0] > 0.0 && crop[2] < 640.0); + // Every frame covers the crop rectangle. + for w in &p.warps { + let inv = w.inverse().unwrap(); + for q in [[crop[0], crop[1]], [crop[2], crop[1]], [crop[2], crop[3]], [crop[0], crop[3]]] { + let s = inv.apply(q); + assert!(s[0] >= -1e-6 && s[1] >= -1e-6 && s[0] <= 640.0 + 1e-6 && s[1] <= 360.0 + 1e-6, "{s:?}"); + } + } + let p = plan(&a, &StabSettings::default()); + assert!(p.auto_scale > 1.0 && p.crop.is_none(), "{} {:?}", p.auto_scale, p.crop); + for w in &p.warps { + let inv = w.inverse().unwrap(); + for q in [[0.0, 0.0], [640.0, 0.0], [640.0, 360.0], [0.0, 360.0]] { + let s = inv.apply(q); + assert!(s[0] >= -1e-3 && s[1] >= -1e-3 && s[0] <= 640.001 && s[1] <= 360.001, "{s:?}"); + } + } + // A tight Maximum Scale relaxes the correction instead of showing borders. + let p = plan(&a, &StabSettings { max_scale: 101.0, ..Default::default() }); + assert!(p.auto_scale <= 1.01 + 1e-9); + assert!(p.valid_fraction * p.auto_scale >= 0.999 || p.crop.is_some()); + let j = WarpAnalysis::from_json(&a.to_json()).unwrap(); + assert_eq!(j, a); + } +}