From 8696d1e9ba0a670949cb5c9b1eb95400bdb6dab9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 00:27:47 -0400 Subject: [PATCH 01/36] M7.7: camera/light rigs and Advanced 3D motion blur, blend modes, track mattes Layer > Camera > Create Stereo 3D Rig (left/right eye comps with expression-linked eye cameras, Stereo 3D Controls effect, stereo comp through 3D Glasses), Create Orbit Null, Create Cameras from 3D Model; Layer > Light > Create Lights from 3D Model (KHR_lights_punctual), Control Light with Camera, Create Environment Light Background Layer (equirectangular sky drawn through the camera). glTF cameras and punctual lights are parsed by effectcraft-model. Advanced 3D: per-sample motion blur over the comp shutter, and blend modes / track mattes / Preserve Transparency composited through the 2D path in depth order. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/controls.rs | 19 + crates/effects/src/lib.rs | 2 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/rig3d.rs | 614 +++++++++++++++++++++++++ crates/engine/src/commands/stubs.rs | 6 - crates/engine/src/lib.rs | 2 + crates/engine/src/tests_menu_cmds.rs | 2 +- crates/engine/src/tests_rig3d.rs | 338 ++++++++++++++ crates/expr/src/prelude.js | 2 + crates/host/src/lib.rs | 42 ++ crates/model/src/gltf.rs | 56 ++- crates/model/src/lib.rs | 85 ++++ crates/model/src/tests.rs | 44 ++ crates/project/src/build.rs | 1 + crates/project/src/lib.rs | 4 + crates/render/src/lib.rs | 30 +- crates/render/src/three_d/adv/mod.rs | 134 +++++- crates/render/src/three_d/adv/scene.rs | 15 + crates/render/src/three_d/adv/tests.rs | 113 +++++ crates/render/src/three_d/compose.rs | 55 +++ docs/architecture.md | 12 +- 21 files changed, 1543 insertions(+), 35 deletions(-) create mode 100644 crates/engine/src/commands/rig3d.rs create mode 100644 crates/engine/src/tests_rig3d.rs diff --git a/crates/effects/src/controls.rs b/crates/effects/src/controls.rs index 98a5b140..9f220bfc 100644 --- a/crates/effects/src/controls.rs +++ b/crates/effects/src/controls.rs @@ -6,6 +6,13 @@ use effectcraft_project::ParamUi; use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; +/// Id of the Stereo 3D Controls effect (Layer ▸ Camera ▸ Create Stereo 3D Rig). +pub const STEREO_CONTROLS: &str = "ec.control.stereo3d"; +/// Stereo 3D Controls ▸ Configuration: which eyes are offset from the master camera. +pub const STEREO_CONFIGURATIONS: [&str; 3] = ["Stereo Pair (Left & Right)", "Center & Right", "Center & Left"]; +/// Stereo 3D Controls ▸ Convergence Of: the zero-parallax distance. +pub const STEREO_CONVERGENCE: [&str; 2] = ["Camera Point of Interest", "Camera Zoom Plane"]; + fn passthrough(_: &EffectCtx, b: Buf) -> Buf { b } @@ -24,5 +31,17 @@ pub fn specs() -> Vec { spec("ec.control.point3d", "3D Point Control", vec![p("point", "3D Point", Value::Vec3([0.0, 0.0, 0.0]), ParamUi::Point3)]), spec("ec.control.layer", "Layer Control", vec![p("layer", "Layer", Value::Layer(None), ParamUi::Layer)]), spec("ec.control.dropdown", "Dropdown Menu Control", vec![p("menu", "Menu", Value::Enum(0), popup(&["Item 1", "Item 2", "Item 3"]))]), + // Layer ▸ Camera ▸ Create Stereo 3D Rig: the rig's eye cameras read these (expressions). + spec( + STEREO_CONTROLS, + "Stereo 3D Controls", + vec![ + p("configuration", "Configuration", Value::Enum(0), popup(&STEREO_CONFIGURATIONS)), + p("sceneDepth", "Stereo Scene Depth", num(3.0), slider(0.0, 100.0, 0.0, 10.0, 2)), + p("convergence", "Enable Convergence", Value::Bool(false), ParamUi::Checkbox), + p("convergenceOf", "Convergence Of", Value::Enum(0), popup(&STEREO_CONVERGENCE)), + p("zOffset", "Convergence Z Offset", num(0.0), slider(-100_000.0, 100_000.0, -2000.0, 2000.0, 1)), + ], + ), ] } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6f47b01e..39912324 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -16,7 +16,7 @@ mod channel3d; mod color2; mod color3; mod color_fx; -mod controls; +pub mod controls; mod distort; mod distort2; mod distort3; diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 56a585dd..4d2adea4 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -36,6 +36,7 @@ mod proxy; pub mod puppet; mod query; mod render_queue; +pub(crate) mod rig3d; pub mod roto_cmds; mod settings; mod stubs; @@ -111,6 +112,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(styles::specs()); v.extend(three_d::specs()); v.extend(model3d::specs()); + v.extend(rig3d::specs()); v.extend(text_anim::specs()); v.extend(text_edit::specs()); v.extend(layer_time::specs()); diff --git a/crates/engine/src/commands/rig3d.rs b/crates/engine/src/commands/rig3d.rs new file mode 100644 index 00000000..f034fa64 --- /dev/null +++ b/crates/engine/src/commands/rig3d.rs @@ -0,0 +1,614 @@ +//! Camera and light rigs: Layer ▸ Camera ▸ Create Stereo 3D Rig / Create Orbit Null / Create +//! Cameras from 3D Model, Layer ▸ Light ▸ Create Lights from 3D Model / Control Light with +//! Camera / Create Environment Light Background Layer. + +use effectcraft_color::Label; +use effectcraft_effects::controls::{STEREO_CONFIGURATIONS, STEREO_CONTROLS, STEREO_CONVERGENCE}; +use effectcraft_geom::{Mat4, Vec3, vec3}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{AutoOrient, Comp, Expression, ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project}; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::adv::scene::model_to_layer; +use effectcraft_render::three_d::camera::{self, layer_frame}; +use serde_json::{Value, json}; + +use super::layer::insert_layer; +use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, resolve_layer}; +use crate::{EngineError, Result, Session, cmd}; + +fn arr(v: Vec3) -> [f64; 3] { + [v.x, v.y, v.z] +} + +/// Set a property's static value (clears its keyframes and expression). +fn set(l: &mut Layer, path: &str, v: KV) { + if let Some(pr) = l.props.prop_mut(path) { + pr.value = v; + pr.keys.clear(); + pr.expr = None; + } +} + +fn set_expr(l: &mut Layer, path: &str, text: String) { + if let Some(pr) = l.props.prop_mut(path) { + pr.expr = Some(Expression { text, enabled: true }); + } +} + +/// What evaluation needs from the session (cloned so edits can evaluate their own project). +struct Env { + time: effectcraft_time::Tick, + expr: Option>, + footage: std::sync::Arc, +} + +impl Env { + fn of(s: &Session, cid: ItemId) -> Env { + Env { time: s.time_of(cid), expr: s.expr.clone(), footage: s.footage.clone() } + } + fn ctx<'a>(&'a self, proj: &'a Project, cid: ItemId, comp: &'a Comp) -> EvalCtx<'a> { + EvalCtx { project: proj, comp_id: cid, comp, time: self.time, expr: self.expr.as_deref(), footage: Some(self.footage.as_ref()) } + } +} + +/// The camera a rig command acts on: `layer`, else a selected camera, else the active camera. +fn camera_p(s: &Session, p: &Value, cid: ItemId, cmd: &str) -> Result { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let id = match p.get("layer").or_else(|| p.get("camera")) { + Some(v) => resolve_layer(comp, v).ok_or_else(|| bad(cmd, "no such layer"))?, + None => s + .state + .selected_layers + .iter() + .copied() + .find(|id| comp.layer(*id).is_some_and(Layer::is_camera)) + .or_else(|| comp.active_camera(s.time_of(cid)).map(|l| l.id)) + .ok_or_else(|| bad(cmd, "the composition has no camera (Layer ▸ New ▸ Camera)"))?, + }; + if !comp.layer(id).is_some_and(Layer::is_camera) { + return Err(bad(cmd, "the layer is not a camera")); + } + Ok(id) +} + +/// Distance from a camera to what it looks at: its point of interest (two-node), else its zoom. +fn focus_distance(ctx: &EvalCtx, cam: &Layer) -> f64 { + let st = camera::layer_camera(ctx, cam); + if camera::is_two_node(cam) + && let Some(tr) = cam.transform() + { + let poi = Vec3::from(ctx.v3(cam, tr, "poi", [0.0; 3])); + let poi = camera::parent_world(ctx, cam).apply(poi); + return (poi - st.eye).length().max(1.0); + } + st.zoom +} + +// ---------------------------------------------------------------- stereo 3D rig + +/// Stereo rig settings (Stereo 3D Controls). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StereoParams { + /// 0 Stereo Pair (Left & Right), 1 Center & Right, 2 Center & Left. + pub configuration: u32, + /// Interaxial separation, % of the comp width. + pub scene_depth: f64, + pub convergence: bool, + /// 0 the camera's point of interest, 1 its zoom plane. + pub convergence_of: u32, + pub z_offset: f64, +} + +/// Eye offsets along the camera's x axis, in units of the interaxial separation. +pub fn eye_offsets(configuration: u32) -> [f64; 2] { + match configuration { + 1 => [0.0, 1.0], + 2 => [-1.0, 0.0], + _ => [-0.5, 0.5], + } +} + +/// (position, point of interest) of the left (`eye` 0) or right (1) camera, from the master +/// camera's eye, axes, focus distance and zoom. +pub fn eye_camera(sp: &StereoParams, eye: usize, comp_w: f64, master: (Vec3, Vec3, Vec3), focus: f64, zoom: f64) -> (Vec3, Vec3) { + let (e, fwd, right) = master; + let d = sp.scene_depth / 100.0 * comp_w; + let pos = e + right * (eye_offsets(sp.configuration)[eye.min(1)] * d); + let poi = if sp.convergence { + let dc = if sp.convergence_of == 1 { zoom } else { focus } + sp.z_offset; + e + fwd * dc.max(1.0) + } else { + pos + fwd * focus + }; + (pos, poi) +} + +/// Expression snippets that rebuild [`eye_camera`] from the master camera and the controls. +fn eye_exprs(src_comp: &str, master: &str, controls: &str, two_node: bool, eye: usize) -> (String, String, String) { + let q = |s: &str| serde_json::to_string(s).unwrap_or_else(|_| "\"\"".into()); + let k = eye_offsets(0)[eye.min(1)]; + let k1 = eye_offsets(1)[eye.min(1)]; + let k2 = eye_offsets(2)[eye.min(1)]; + let head = format!( + "var src = comp({c});\nvar m = src.layer({m});\nvar fx = src.layer({x}).effect(\"Stereo 3D Controls\");\n\ + var d = fx(\"Stereo Scene Depth\").value / 100 * src.width;\nvar cfg = fx(\"Configuration\").value;\n\ + var k = cfg == 2 ? {k1} : (cfg == 3 ? {k2} : {k});\n\ + var E = m.toWorld([0, 0, 0]);\nvar F = normalize(m.toWorldVec([0, 0, 1]));\n\ + var eye = add(E, mul(normalize(m.toWorldVec([1, 0, 0])), k * d));\n", + c = q(src_comp), + m = q(master), + x = q(controls), + ); + let dm = if two_node { "Math.max(length(sub(m.transform.pointOfInterest, m.transform.position)), 1)" } else { "m.cameraOption.zoom" }; + let pos = format!("{head}eye;"); + let poi = format!( + "{head}var dm = {dm};\nvar dc = (fx(\"Convergence Of\").value == 2 ? m.cameraOption.zoom : dm) + fx(\"Convergence Z Offset\").value;\n\ + fx(\"Enable Convergence\").value ? add(E, mul(F, Math.max(dc, 1))) : add(eye, mul(F, dm));" + ); + let zoom = format!("comp({}).layer({}).cameraOption.zoom;", q(src_comp), q(master)); + (pos, poi, zoom) +} + +fn stereo_p(p: &Value, cmd: &str) -> Result { + let pick = |k: &str, opts: &[&str], ids: &[&str]| -> Result> { + match p.get(k) { + None => Ok(None), + Some(Value::Number(n)) => Ok(n.as_u64().map(|v| (v as u32).min(opts.len() as u32 - 1))), + Some(Value::String(s)) => { + let n = s.to_ascii_lowercase().replace([' ', '-', '_', '&', '(', ')'], ""); + let i = ids + .iter() + .position(|v| v.to_ascii_lowercase() == n) + .or_else(|| opts.iter().position(|o| o.to_ascii_lowercase().replace([' ', '-', '_', '&', '(', ')'], "") == n)); + i.map(|i| Some(i as u32)).ok_or_else(|| bad(cmd, format!("{k}: {}", ids.join("|")))) + } + _ => Err(bad(cmd, format!("{k}: {}", ids.join("|")))), + } + }; + Ok(StereoParams { + configuration: pick("configuration", &STEREO_CONFIGURATIONS, &["stereoPair", "centerRight", "centerLeft"])?.unwrap_or(0), + scene_depth: f_p(p, "sceneDepth").unwrap_or(3.0).clamp(0.0, 100.0), + convergence: b_p(p, "convergence").unwrap_or(false), + convergence_of: pick("convergenceOf", &STEREO_CONVERGENCE, &["poi", "zoom"])?.unwrap_or(0), + z_offset: f_p(p, "zOffset").unwrap_or(0.0), + }) +} + +fn unique_item_name(proj: &Project, base: &str) -> String { + let mut name = base.to_string(); + let mut k = 2; + while proj.items.values().any(|i| i.name == name) { + name = format!("{base} {k}"); + k += 1; + } + name +} + +/// An empty comp with the settings of `src`. +fn sibling_comp(src: &Comp) -> Comp { + let mut c = src.clone(); + c.layers.clear(); + c.markers.clear(); + c +} + +fn stereo_rig(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.stereoRig"; + let cid = comp_id(s, p)?; + let sp = stereo_p(p, cmd)?; + let view3d = p.get("view3d").and_then(Value::as_u64).unwrap_or(5) as u32; + let src = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src_name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let t = s.time_of(cid); + let w = src.width as f64; + let existing = src.active_camera(t).map(|l| l.id); + let env = Env::of(s, cid); + let r = s.edit("Create Stereo 3D Rig", None, |proj, st| { + // The master camera: the comp's active camera, else a new one. + let master_id = match existing { + Some(id) => id, + None => { + let name = src.unique_layer_name("Master Camera"); + let l = build::layer(proj, &src, &name, LayerSource::Camera, (src.width, src.height), None); + insert_layer(proj, st, cid, l)? + } + }; + let controls_name = proj.comp(cid).map(|c| c.unique_layer_name("Stereo 3D Controls")).unwrap_or_default(); + let src_now = proj.comp(cid).cloned().ok_or(EngineError::NoComp)?; + let mut ctl = build::layer(proj, &src_now, &controls_name, LayerSource::Null, (src.width, src.height), None); + ctl.switches.video = false; + let spec = effectcraft_effects::find(STEREO_CONTROLS).ok_or_else(|| bad(cmd, "Stereo 3D Controls effect missing"))?; + let mut fx = effectcraft_effects::instantiate(spec, &mut Ids(&mut proj.next_id), spec.name, [100.0, 100.0]); + for (k, v) in [ + ("configuration", KV::Enum(sp.configuration)), + ("sceneDepth", KV::Scalar(sp.scene_depth)), + ("convergence", KV::Bool(sp.convergence)), + ("convergenceOf", KV::Enum(sp.convergence_of)), + ("zOffset", KV::Scalar(sp.z_offset)), + ] { + if let Some(pr) = fx.get_mut(k) { + pr.value = v; + } + } + if let Some(g) = ctl.props.sub_mut("effects") { + g.children.push(fx.into()); + } + let ctl_id = ctl.id; + let src_now = proj.comp(cid).cloned().ok_or(EngineError::NoComp)?; + let at = src_now.layers.iter().position(|l| l.id == master_id).unwrap_or(0); + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(at, ctl); + // Master camera frame at the current time. + let src_now = proj.comp(cid).cloned().ok_or(EngineError::NoComp)?; + let master = src_now.layer(master_id).cloned().ok_or(EngineError::NoComp)?; + let ctx = env.ctx(proj, cid, &src_now); + let (e, fwd, down) = layer_frame(&ctx, &master); + let right = down.cross(fwd).normalize(); + let focus = focus_distance(&ctx, &master); + let zoom = camera::layer_camera(&ctx, &master).zoom; + let two = camera::is_two_node(&master); + // Left and right eye comps: the source comp collapsed, seen by an eye camera. + let mut eye_comps = vec![]; + for (eye, label) in [(0usize, "Left"), (1, "Right")] { + let name = unique_item_name(proj, &format!("{src_name} {label} Eye")); + let ec = sibling_comp(&src_now); + let eid = proj.add_item(&name, Label::Sandstone, proj.item(cid).and_then(|i| i.parent), ItemKind::Comp(ec.clone().into())); + let mut nest = build::layer(proj, &ec, &src_name, LayerSource::Comp { item: cid }, (src.width, src.height), Some(src.duration)); + nest.switches.three_d = true; + nest.switches.collapse = true; + let (pos, poi) = eye_camera(&sp, eye, w, (e, fwd, right), focus, zoom); + let mut cam = build::layer(proj, &ec, &format!("{label} Eye Camera"), LayerSource::Camera, (src.width, src.height), None); + set(&mut cam, "transform/position", KV::Vec3(arr(pos))); + set(&mut cam, "transform/poi", KV::Vec3(arr(poi))); + set(&mut cam, "cameraOptions/zoom", KV::Scalar(zoom)); + set(&mut cam, "cameraOptions/focusDistance", KV::Scalar(focus)); + let (pe, poe, ze) = eye_exprs(&src_name, &master.name, &controls_name, two, eye); + set_expr(&mut cam, "transform/position", pe); + set_expr(&mut cam, "transform/poi", poe); + set_expr(&mut cam, "cameraOptions/zoom", ze); + cam.auto_orient = AutoOrient::TowardsPointOfInterest; + let comp = proj.comp_mut(eid).ok_or(EngineError::NoComp)?; + comp.layers = vec![cam, nest]; + eye_comps.push(eid); + } + // The output comp: both eyes through 3D Glasses. + let out_name = unique_item_name(proj, &format!("{src_name} Stereo 3D")); + let oc = sibling_comp(&src_now); + let oid = proj.add_item(&out_name, Label::Sandstone, proj.item(cid).and_then(|i| i.parent), ItemKind::Comp(oc.clone().into())); + let left_name = proj.item(eye_comps[0]).map(|i| i.name.clone()).unwrap_or_default(); + let right_name = proj.item(eye_comps[1]).map(|i| i.name.clone()).unwrap_or_default(); + let mut left = build::layer(proj, &oc, &left_name, LayerSource::Comp { item: eye_comps[0] }, (src.width, src.height), Some(src.duration)); + let mut right = build::layer(proj, &oc, &right_name, LayerSource::Comp { item: eye_comps[1] }, (src.width, src.height), Some(src.duration)); + right.switches.video = false; + let glasses = effectcraft_effects::find("ec.perspective.3dglasses").ok_or_else(|| bad(cmd, "3D Glasses effect missing"))?; + let mut g = effectcraft_effects::instantiate(glasses, &mut Ids(&mut proj.next_id), glasses.name, [w, src.height as f64]); + for (k, v) in [("leftView", KV::Layer(Some(left.id.0))), ("rightView", KV::Layer(Some(right.id.0))), ("view3d", KV::Enum(view3d.min(8)))] { + if let Some(pr) = g.get_mut(k) { + pr.value = v; + } + } + if let Some(fxg) = left.props.sub_mut("effects") { + fxg.children.push(g.into()); + } + let (lid, rid) = (left.id, right.id); + proj.comp_mut(oid).ok_or(EngineError::NoComp)?.layers = vec![left, right]; + st.project_selection = vec![oid]; + Ok(json!({ + "controls": ctl_id.0, + "master": master_id.0, + "leftComp": eye_comps[0].0, + "rightComp": eye_comps[1].0, + "stereoComp": oid.0, + "leftLayer": lid.0, + "rightLayer": rid.0, + })) + })?; + Ok(r) +} + +// ---------------------------------------------------------------- orbit null + +/// Create Orbit Null: a 3D null at the camera's point of interest becomes the camera's parent, +/// so rotating the null orbits the camera around what it looks at. +fn orbit_null(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.orbitNull"; + let cid = comp_id(s, p)?; + let cam_id = camera_p(s, p, cid, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let cam = comp.layer(cam_id).cloned().ok_or(EngineError::NoComp)?; + if cam.parent.is_some() { + return Err(bad(cmd, format!("`{}` already has a parent", cam.name))); + } + let env = Env::of(s, cid); + let ctx = env.ctx(&s.project, cid, &comp); + let (eye, fwd, _) = layer_frame(&ctx, &cam); + let target = eye + fwd * focus_distance(&ctx, &cam); + let name = comp.unique_layer_name(&format!("{} Orbit Null", cam.name)); + let r = s.edit("Create Orbit Null", None, |proj, st| { + let mut null = build::layer(proj, &comp, &name, LayerSource::Null, (comp.width, comp.height), None); + null.switches.three_d = true; + null.switches.video = true; + set(&mut null, "transform/position", KV::Vec3(arr(target))); + // World → null space (a translated, unrotated, unscaled null: p − P + anchor). + let anchor = Vec3::from(null.props.prop("transform/anchor").map(|a| a.value.as_vec3()).unwrap_or([0.0; 3])); + let to_null = |v: [f64; 3]| arr(Vec3::from(v) - target + anchor); + let nid = null.id; + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|l| l.id == cam_id).unwrap_or(0); + c.layers.insert(at, null); + let l = c.layer_mut(cam_id).ok_or(EngineError::NoComp)?; + l.parent = Some(nid); + for path in ["transform/position", "transform/poi"] { + if let Some(pr) = l.props.prop_mut(path) { + pr.value = KV::Vec3(to_null(pr.value.as_vec3())); + for k in &mut pr.keys { + k.value = KV::Vec3(to_null(k.value.as_vec3())); + } + } + } + st.selected_layers = vec![nid]; + Ok(json!({"null": nid.0, "camera": cam_id.0, "position": arr(target)})) + })?; + Ok(r) +} + +// ---------------------------------------------------------------- control light with camera + +/// Control Light with Camera: the light rides on the camera (parented at the camera's eye, +/// shining where it looks). Again (or `on: false`) releases it where it is. +fn control_light(s: &mut Session, p: &Value) -> Result { + let cmd = "light.controlWithCamera"; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lights: Vec = match p.get("layers").or_else(|| p.get("layer")) { + Some(Value::Array(a)) => a.iter().filter_map(|v| resolve_layer(&comp, v)).collect(), + Some(v) => resolve_layer(&comp, v).into_iter().collect(), + None => s.state.selected_layers.iter().copied().filter(|id| comp.layer(*id).is_some_and(Layer::is_light)).collect(), + }; + if lights.is_empty() || lights.iter().any(|id| !comp.layer(*id).is_some_and(Layer::is_light)) { + return Err(bad(cmd, "select a light layer")); + } + let cam_id = match p.get("camera") { + Some(v) => resolve_layer(&comp, v).filter(|id| comp.layer(*id).is_some_and(Layer::is_camera)).ok_or_else(|| bad(cmd, "no such camera"))?, + None => comp.active_camera(s.time_of(cid)).map(|l| l.id).ok_or_else(|| bad(cmd, "the composition has no camera (Layer ▸ New ▸ Camera)"))?, + }; + let on = b_p(p, "on").unwrap_or_else(|| comp.layer(lights[0]).is_none_or(|l| l.parent != Some(cam_id))); + let env = Env::of(s, cid); + let ctx = env.ctx(&s.project, cid, &comp); + // Released lights keep their current world placement. + let frames: Vec<(LayerId, Vec3, Vec3)> = lights + .iter() + .filter_map(|id| comp.layer(*id)) + .map(|l| { + let (e, f, _) = layer_frame(&ctx, l); + (l.id, e, f) + }) + .collect(); + let (w, h) = (comp.width as f64, comp.height as f64); + s.edit(if on { "Control Light with Camera" } else { "Release Light from Camera" }, None, |proj, _| { + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + for (id, e, f) in &frames { + let l = c.layer_mut(*id).ok_or(EngineError::NoComp)?; + if on { + l.parent = Some(cam_id); + set(l, "transform/position", KV::Vec3([0.0; 3])); + set(l, "transform/poi", KV::Vec3([0.0, 0.0, 1000.0])); + set(l, "transform/orientation", KV::Vec3([0.0; 3])); + for k in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + set(l, k, KV::Scalar(0.0)); + } + l.auto_orient = AutoOrient::Off; + } else if l.parent == Some(cam_id) { + l.parent = None; + set(l, "transform/position", KV::Vec3(arr(*e))); + set(l, "transform/poi", KV::Vec3(arr(*e + *f * w.max(h)))); + if l.transform().is_some_and(|t| t.get("poi").is_some()) { + l.auto_orient = AutoOrient::TowardsPointOfInterest; + } + } + } + Ok(()) + })?; + Ok(json!({"controlled": on, "camera": cam_id.0, "lights": lights.iter().map(|l| l.0).collect::>()})) +} + +// ---------------------------------------------------------------- cameras and lights from models + +/// Model layers a "from 3D Model" command reads: `layers`/`layer`, else the selected model layers. +fn model_layers(s: &Session, p: &Value, comp: &Comp, cmd: &str) -> Result> { + let ids: Vec = match p.get("layers").or_else(|| p.get("layer")) { + Some(Value::Array(a)) => a.iter().filter_map(|v| resolve_layer(comp, v)).collect(), + Some(v) => resolve_layer(comp, v).into_iter().collect(), + None => s.state.selected_layers.iter().copied().filter(|id| comp.layer(*id).is_some_and(|l| matches!(l.source, LayerSource::Model { .. }))).collect(), + }; + if ids.is_empty() || ids.iter().any(|id| !comp.layer(*id).is_some_and(|l| matches!(l.source, LayerSource::Model { .. }))) { + return Err(bad(cmd, "select a 3D model layer")); + } + Ok(ids) +} + +/// A camera or light found in a model: (name, world eye, world forward, the model data). +enum FromModel { + Camera { name: String, eye: Vec3, fwd: Vec3, zoom: f64 }, + Light { name: String, eye: Vec3, fwd: Vec3, light: effectcraft_model::ModelLight }, +} + +fn from_model(s: &mut Session, p: &Value, lights: bool) -> Result { + let cmd = if lights { "light.fromModel" } else { "camera.fromModel" }; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let ids = model_layers(s, p, &comp, cmd)?; + let env = Env::of(s, cid); + let ctx = env.ctx(&s.project, cid, &comp); + let mut found = vec![]; + for id in &ids { + let l = comp.layer(*id).ok_or(EngineError::NoComp)?; + let LayerSource::Model { item } = l.source else { continue }; + let Some(ItemKind::Footage(f)) = s.project.item(item).map(|i| &i.kind) else { continue }; + let Some(m) = s.footage.model(item, f) else { return Err(bad(cmd, format!("can't read the model of `{}`", l.name))) }; + let unit = l.props.sub("geometryOptions").map_or(1.0, |g| ctx.f(l, g, "unitScale", 1.0)); + let to_world = ctx.world_matrix(l) * model_to_layer(unit); + for pl in m.placed(lights, None, 0.0) { + let w: Mat4 = to_world * pl.world; + let eye = w.apply(Vec3::ZERO); + let fwd = w.apply_vec(vec3(0.0, 0.0, -1.0)).normalize(); + if lights { + let ml = m.lights[pl.index].clone(); + found.push(FromModel::Light { name: ml.name.clone(), eye, fwd, light: ml }); + } else { + let mc = &m.cameras[pl.index]; + let zoom = match mc.projection { + effectcraft_model::CameraProjection::Perspective { yfov, .. } => comp.height as f64 * 0.5 / (yfov * 0.5).tan(), + // Orthographic cameras become perspective ones framing the same height. + effectcraft_model::CameraProjection::Orthographic { .. } => effectcraft_geom::default_camera_zoom(comp.width as f64), + }; + found.push(FromModel::Camera { name: mc.name.clone(), eye, fwd, zoom }); + } + } + } + if found.is_empty() { + return Err(bad(cmd, if lights { "the model has no lights (KHR_lights_punctual)" } else { "the model has no cameras" })); + } + let made = s.edit(if lights { "Create Lights from 3D Model" } else { "Create Cameras from 3D Model" }, None, |proj, st| { + let mut made = vec![]; + for f in &found { + let c = proj.comp(cid).cloned().ok_or(EngineError::NoComp)?; + let l = match f { + FromModel::Camera { name, eye, fwd, zoom } => { + let mut l = build::layer(proj, &c, &c.unique_layer_name(name), LayerSource::Camera, (c.width, c.height), None); + set(&mut l, "transform/position", KV::Vec3(arr(*eye))); + set(&mut l, "transform/poi", KV::Vec3(arr(*eye + *fwd * *zoom))); + set(&mut l, "cameraOptions/zoom", KV::Scalar(*zoom)); + set(&mut l, "cameraOptions/focusDistance", KV::Scalar(*zoom)); + l + } + FromModel::Light { name, eye, fwd, light } => { + let kind = match light.kind { + effectcraft_model::ModelLightKind::Directional => LightKind::Parallel, + effectcraft_model::ModelLightKind::Point => LightKind::Point, + effectcraft_model::ModelLightKind::Spot { .. } => LightKind::Spot, + }; + let mut l = build::layer(proj, &c, &c.unique_layer_name(name), LayerSource::Light { kind }, (c.width, c.height), None); + set(&mut l, "transform/position", KV::Vec3(arr(*eye))); + set(&mut l, "transform/poi", KV::Vec3(arr(*eye + *fwd * c.width.max(c.height) as f64))); + let col = light.color; + set(&mut l, "lightOptions/color", KV::Color([col[0], col[1], col[2], 1.0])); + // Intensity: 1 unit of the file (lux / candela) = 100%. + set(&mut l, "lightOptions/intensity", KV::Scalar((light.intensity * 100.0).clamp(0.0, 1000.0))); + if let effectcraft_model::ModelLightKind::Spot { inner, outer } = light.kind { + set(&mut l, "lightOptions/coneAngle", KV::Scalar((outer * 2.0).to_degrees().clamp(0.0, 180.0))); + set(&mut l, "lightOptions/coneFeather", KV::Scalar(((1.0 - inner / outer.max(1e-9)) * 100.0).clamp(0.0, 100.0))); + } + if let Some(r) = light.range { + set(&mut l, "lightOptions/falloff", KV::Enum(1)); + set(&mut l, "lightOptions/falloffDistance", KV::Scalar(r * 100.0)); + } + l + } + }; + made.push(insert_layer(proj, st, cid, l)?.0); + } + Ok(made) + })?; + Ok(json!({"layers": made})) +} + +// ---------------------------------------------------------------- environment background + +/// Create Environment Light Background Layer: a layer showing the environment light's image as +/// the scene's backdrop (an equirectangular sky drawn through the camera). +fn environment_background(s: &mut Session, p: &Value) -> Result { + let cmd = "light.environmentBackground"; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let t = s.time_of(cid); + let is_env = |l: &Layer| matches!(l.source, LayerSource::Light { kind: LightKind::Environment }); + let light = match p.get("layer") { + Some(v) => resolve_layer(&comp, v).and_then(|id| comp.layer(id)).filter(|l| is_env(l)).ok_or_else(|| bad(cmd, "not an environment light"))?, + None => s + .state + .selected_layers + .iter() + .filter_map(|id| comp.layer(*id)) + .find(|l| is_env(l)) + .or_else(|| comp.layers.iter().find(|l| is_env(l))) + .ok_or_else(|| bad(cmd, "the composition has no environment light (Layer ▸ New ▸ Light ▸ Environment)"))?, + }; + let src_layer = light + .props + .prop("lightOptions/source") + .and_then(|pr| pr.value_at(light.layer_time(t)).as_layer()) + .and_then(|id| comp.layer(LayerId(id))) + .or_else(|| comp.layers.iter().find(|l| l.environment)) + .ok_or_else(|| bad(cmd, "the environment light has no Source (or set an Environment Layer)"))? + .clone(); + if !matches!(src_layer.source, LayerSource::Footage { .. } | LayerSource::Comp { .. } | LayerSource::Solid { .. }) { + return Err(bad(cmd, "the environment source must be a footage, comp or solid layer")); + } + let light_id = light.id; + let name = comp.unique_layer_name(&format!("{} Background", src_layer.name)); + let size = effectcraft_render::source_size(&s.project, &src_layer); + let id = s.edit("Create Environment Light Background Layer", None, |proj, st| { + let mut l = build::layer(proj, &comp, &name, src_layer.source.clone(), size, None); + l.switches.three_d = true; + l.environment_background = true; + // The backdrop sits at the bottom of the stack. + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let id = l.id; + c.layers.push(l); + st.selected_layers = vec![id]; + Ok(id) + })?; + Ok(json!({"layer": id.0, "light": light_id.0, "source": src_layer.id.0})) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "camera.stereoRig", + "Create Stereo 3D Rig", + ["Layer", "Camera"], + None, + "{comp?, configuration?: stereoPair|centerRight|centerLeft, sceneDepth? (% of comp width, default 3), convergence?: bool, convergenceOf?: poi|zoom, zOffset?, view3d?: 3D Glasses view index (default 5 Balanced Colored Red Blue)}", + has_comp, + stereo_rig + ), + cmd!("camera.orbitNull", "Create Orbit Null", ["Layer", "Camera"], None, "{comp?, layer?: camera (default selected/active)}", has_comp, orbit_null), + cmd!( + "camera.fromModel", + "Create Cameras from 3D Model", + ["Layer", "Camera"], + None, + "{comp?, layers?: 3D model layers (default selected)}", + has_comp, + |s, p| { from_model(s, p, false) } + ), + cmd!( + "light.fromModel", + "Create Lights from 3D Model", + ["Layer", "Light"], + None, + "{comp?, layers?: 3D model layers (default selected)}", + has_comp, + |s, p| { from_model(s, p, true) } + ), + cmd!( + "light.controlWithCamera", + "Control Light with Camera", + ["Layer", "Light"], + None, + "{comp?, layers?: lights (default selected), camera? (default active), on?: bool (default toggle)}", + has_comp, + control_light + ), + cmd!( + "light.environmentBackground", + "Create Environment Light Background Layer", + ["Layer", "Light"], + None, + "{comp?, layer?: environment light (default selected / first)}", + has_comp, + environment_background + ), + ] +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 764d6d99..c2683e60 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -26,15 +26,9 @@ pub fn specs() -> Vec { stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). - stub!("camera.stereoRig", "Create Stereo 3D Rig", ["Layer", "Camera"], "{}"), - stub!("camera.orbitNull", "Create Orbit Null", ["Layer", "Camera"], "{}"), - stub!("camera.fromModel", "Create Cameras from 3D Model", ["Layer", "Camera"], "{}"), stub!("camera.linkFocusToPoi", "Link Focus Distance to Point of Interest", ["Layer", "Camera"], "{}"), stub!("camera.linkFocusToLayer", "Link Focus Distance to Layer", ["Layer", "Camera"], "{}"), stub!("camera.setFocusToLayer", "Set Focus Distance to Layer", ["Layer", "Camera"], "{}"), - stub!("light.fromModel", "Create Lights from 3D Model", ["Layer", "Light"], "{}"), - stub!("light.controlWithCamera", "Control Light with Camera", ["Layer", "Light"], "{}"), - stub!("light.environmentBackground", "Create Environment Light Background Layer", ["Layer", "Light"], "{}"), stub!("view.3d.default", "Default", ["View", "Switch 3D View"], "{}"), stub!("view.splitLockedViewer", "Split with New Locked Viewer", ["View"], "{}"), // Color management. diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2d5beb79..c9d931d7 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -673,6 +673,8 @@ mod tests_project_items; #[cfg(test)] mod tests_proxy; #[cfg(test)] +mod tests_rig3d; +#[cfg(test)] mod tests_settings; #[cfg(test)] mod tests_stubs; diff --git a/crates/engine/src/tests_menu_cmds.rs b/crates/engine/src/tests_menu_cmds.rs index 13087ca3..4456a12c 100644 --- a/crates/engine/src/tests_menu_cmds.rs +++ b/crates/engine/src/tests_menu_cmds.rs @@ -487,7 +487,7 @@ fn frontend_commands_emit_events_and_stubs_are_disabled() { let ev = frontend_events(&mut s); assert_eq!(ev[0].0, "view.zoomIn"); assert_eq!(ev[1], ("window.panel".to_string(), json!({"panel": "align"}))); - for id in ["keys.audioToKeyframes", "camera.stereoRig", "view.displayColorManagement", "view.3d.default", "layer.create", "track.motion"] { + for id in ["keys.audioToKeyframes", "layer.autoTrace", "layer.sceneEditDetection", "layer.create", "track.motion"] { assert!(!s.is_enabled(id), "{id}"); assert!(matches!(s.execute(id, json!({})), Err(EngineError::Disabled(..))), "{id}"); } diff --git a/crates/engine/src/tests_rig3d.rs b/crates/engine/src/tests_rig3d.rs new file mode 100644 index 00000000..d3b96510 --- /dev/null +++ b/crates/engine/src/tests_rig3d.rs @@ -0,0 +1,338 @@ +//! Camera and light rigs (M7.7): Create Stereo 3D Rig, Create Orbit Null, Control Light with +//! Camera, Create Cameras / Lights from 3D Model and Create Environment Light Background Layer +//! (behaviour, values, pixels, undo/redo, serde). + +use std::sync::Arc; + +use effectcraft_geom::{Vec3, vec3}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{Footage, FootageKind, GroupKind, ItemId, Layer, LayerId, LayerSource, LightKind, Project}; +use effectcraft_render::three_d::camera::layer_frame; +use effectcraft_render::{EvalCtx, FootageSource, RenderOpts}; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::{Value, json}; + +use crate::{Importer, Session}; + +fn session() -> Session { + let mut s = Session { footage: Arc::new(DiskModels), importer: Some(Arc::new(ModelImporter)), ..Default::default() }; + s.execute("comp.new", json!({"name": "Scene", "width": 640, "height": 360, "frameRate": 30, "duration": 2})).unwrap(); + s +} + +fn cid(s: &Session) -> ItemId { + s.active_comp_id().unwrap() +} + +fn layer_in(s: &Session, comp: ItemId, id: u64) -> Layer { + s.project.comp(comp).unwrap().layer(LayerId(id)).unwrap().clone() +} + +fn v3(l: &Layer, path: &str) -> [f64; 3] { + l.props.prop(path).unwrap().value.as_vec3() +} + +fn close3(a: [f64; 3], b: [f64; 3], tol: f64) -> bool { + (0..3).all(|i| (a[i] - b[i]).abs() <= tol) +} + +/// World (eye, forward) of a layer in the active comp at time 0. +fn frame(s: &Session, id: u64) -> (Vec3, Vec3) { + let c = cid(s); + let comp = s.project.comp(c).unwrap(); + let ctx = EvalCtx { project: &s.project, comp_id: c, comp, time: Tick::ZERO, expr: None, footage: None }; + let (e, f, _) = layer_frame(&ctx, comp.layer(LayerId(id)).unwrap()); + (e, f) +} + +fn camera(s: &mut Session) -> u64 { + s.execute("layer.newCamera", json!({"name": "Cam", "position": [320, 180, -800], "poi": [320, 180, 0], "zoom": 900})).unwrap()["layer"].as_u64().unwrap() +} + +#[test] +fn stereo_rig_builds_eye_comps_and_glasses() { + let mut s = session(); + let src = cid(&s); + let cam = camera(&mut s); + let items_before = s.project.items.len(); + let r = s.execute("camera.stereoRig", json!({"sceneDepth": 4})).unwrap(); + assert_eq!(s.project.items.len(), items_before + 3, "left, right and stereo comps"); + assert_eq!(r["master"].as_u64(), Some(cam)); + // Controls on a hidden null in the source comp. + let ctl = layer_in(&s, src, r["controls"].as_u64().unwrap()); + assert_eq!(ctl.name, "Stereo 3D Controls"); + assert!(!ctl.switches.video); + let fx = ctl.effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(&fx.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::controls::STEREO_CONTROLS)); + assert_eq!(fx.get("sceneDepth").unwrap().value, KV::Scalar(4.0)); + // Eye comps: an eye camera (two-node, linked by expressions) and the collapsed source comp. + let d = 0.04 * 640.0; + for (key, name, x) in [("leftComp", "Scene Left Eye", 320.0 - d / 2.0), ("rightComp", "Scene Right Eye", 320.0 + d / 2.0)] { + let ec = ItemId(r[key].as_u64().unwrap()); + assert_eq!(s.project.item(ec).unwrap().name, name); + let c = s.project.comp(ec).unwrap(); + assert_eq!((c.width, c.height), (640, 360)); + assert_eq!(c.layers.len(), 2); + let cam = &c.layers[0]; + assert!(cam.is_camera() && effectcraft_render::three_d::camera::is_two_node(cam)); + assert!(close3(v3(cam, "transform/position"), [x, 180.0, -800.0], 1e-6), "{:?}", v3(cam, "transform/position")); + // Parallel eyes (no convergence): each looks straight ahead. + assert!(close3(v3(cam, "transform/poi"), [x, 180.0, 0.0], 1e-6), "{:?}", v3(cam, "transform/poi")); + assert_eq!(cam.props.prop("cameraOptions/zoom").unwrap().value, KV::Scalar(900.0)); + for p in ["transform/position", "transform/poi", "cameraOptions/zoom"] { + let e = cam.props.prop(p).unwrap().expr.clone().unwrap(); + assert!(e.text.contains("comp(\"Scene\")") && e.text.contains("Cam"), "{p}: {}", e.text); + } + let nest = &c.layers[1]; + assert_eq!(nest.source, LayerSource::Comp { item: src }); + assert!(nest.switches.collapse && nest.switches.three_d); + } + // The stereo comp: left eye on top with 3D Glasses fed by both eyes; right eye hidden. + let oc = ItemId(r["stereoComp"].as_u64().unwrap()); + let out = s.project.comp(oc).unwrap(); + assert_eq!(s.project.item(oc).unwrap().name, "Scene Stereo 3D"); + let (l, rr) = (r["leftLayer"].as_u64().unwrap(), r["rightLayer"].as_u64().unwrap()); + assert_eq!(out.layers[0].id.0, l); + assert!(!out.layers[1].switches.video); + let g = out.layers[0].effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(&g.kind, GroupKind::Effect { effect } if effect == "ec.perspective.3dglasses")); + assert_eq!(g.get("leftView").unwrap().value, KV::Layer(Some(l))); + assert_eq!(g.get("rightView").unwrap().value, KV::Layer(Some(rr))); + // One undo step; redo; serde. + let json = serde_json::to_string(&*s.project).unwrap(); + let back: Project = serde_json::from_str(&json).unwrap(); + assert_eq!(back, *s.project); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.project.items.len(), items_before); + assert_eq!(s.project.comp(src).unwrap().layers.len(), 1); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(s.project.items.len(), items_before + 3); +} + +#[test] +fn stereo_rig_configurations_and_convergence() { + let mut s = session(); + camera(&mut s); + let r = s.execute("camera.stereoRig", json!({"configuration": "centerRight", "sceneDepth": 5, "convergence": true})).unwrap(); + let d = 0.05 * 640.0; + let eye = |key: &str| s.project.comp(ItemId(r[key].as_u64().unwrap())).unwrap().layers[0].clone(); + let (l, rr) = (eye("leftComp"), eye("rightComp")); + assert!(close3(v3(&l, "transform/position"), [320.0, 180.0, -800.0], 1e-6)); + assert!(close3(v3(&rr, "transform/position"), [320.0 + d, 180.0, -800.0], 1e-6)); + // Converged on the master's point of interest. + assert!(close3(v3(&l, "transform/poi"), [320.0, 180.0, 0.0], 1e-6)); + assert!(close3(v3(&rr, "transform/poi"), [320.0, 180.0, 0.0], 1e-6)); + // A comp without a camera gets a Master Camera. + let mut s = session(); + let r = s.execute("camera.stereoRig", json!({"configuration": "centerLeft"})).unwrap(); + let m = layer_in(&s, cid(&s), r["master"].as_u64().unwrap()); + assert_eq!(m.name, "Master Camera"); + assert!(s.execute("camera.stereoRig", json!({"configuration": "sideways"})).is_err()); +} + +/// The eye cameras' expressions reproduce the static values (and follow the master). +#[test] +fn stereo_rig_maths() { + use crate::commands::rig3d::{StereoParams, eye_camera}; + let sp = StereoParams { configuration: 0, scene_depth: 10.0, convergence: false, convergence_of: 0, z_offset: 0.0 }; + let master = (vec3(0.0, 0.0, -100.0), vec3(0.0, 0.0, 1.0), vec3(1.0, 0.0, 0.0)); + let (p0, q0) = eye_camera(&sp, 0, 1000.0, master, 100.0, 500.0); + assert!(close3([p0.x, p0.y, p0.z], [-50.0, 0.0, -100.0], 1e-9) && close3([q0.x, q0.y, q0.z], [-50.0, 0.0, 0.0], 1e-9)); + let sp = StereoParams { convergence: true, convergence_of: 1, z_offset: 50.0, ..sp }; + let (_, q1) = eye_camera(&sp, 1, 1000.0, master, 100.0, 500.0); + // Zoom plane (500) + Z offset (50) in front of the master eye. + assert!(close3([q1.x, q1.y, q1.z], [0.0, 0.0, 450.0], 1e-9), "{q1:?}"); +} + +#[test] +fn orbit_null_parents_the_camera_at_its_point_of_interest() { + let mut s = session(); + let cam = camera(&mut s); + let (eye0, f0) = frame(&s, cam); + let r = s.execute("camera.orbitNull", json!({})).unwrap(); + let nid = r["null"].as_u64().unwrap(); + let n = layer_in(&s, cid(&s), nid); + assert!(n.is_3d() && matches!(n.source, LayerSource::Null)); + assert_eq!(n.name, "Cam Orbit Null"); + assert!(close3(v3(&n, "transform/position"), [320.0, 180.0, 0.0], 1e-9)); + let c = layer_in(&s, cid(&s), cam); + assert_eq!(c.parent, Some(LayerId(nid))); + // The camera stays where it was. + let (eye1, f1) = frame(&s, cam); + assert!((eye1 - eye0).length() < 1e-6 && (f1 - f0).length() < 1e-6, "{eye0:?} {eye1:?}"); + // Turning the null orbits the camera around the point of interest. + s.execute("prop.set", json!({"layer": nid, "path": "transform/rotationY", "value": 90})).unwrap(); + let (eye2, f2) = frame(&s, cam); + let poi = vec3(320.0, 180.0, 0.0); + assert!(((eye2 - poi).length() - 800.0).abs() < 1e-6, "{eye2:?}"); + assert!((eye2.z).abs() < 1e-6 && (eye2.x - 320.0).abs() > 700.0, "{eye2:?}"); + assert!(((poi - eye2).normalize() - f2).length() < 1e-6, "still looks at the POI: {f2:?}"); + // A parented camera can't get another orbit null; undo restores the camera. + assert!(s.execute("camera.orbitNull", json!({"layer": cam})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(layer_in(&s, cid(&s), cam).parent, None); + assert!(s.active_comp().unwrap().layer(LayerId(nid)).is_none()); +} + +#[test] +fn control_light_with_camera_follows_and_releases() { + let mut s = session(); + let cam = camera(&mut s); + let light = s.execute("layer.newLight", json!({"kind": "Spot", "position": [0, 0, -500], "poi": [100, 100, 0]})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.select", json!({"layers": [light]})).unwrap(); + let r = s.execute("light.controlWithCamera", json!({})).unwrap(); + assert_eq!(r["controlled"], true); + let l = layer_in(&s, cid(&s), light); + assert_eq!(l.parent, Some(LayerId(cam))); + let (ce, cf) = frame(&s, cam); + let (le, lf) = frame(&s, light); + assert!((le - ce).length() < 1e-6 && (lf - cf).length() < 1e-6, "light at the camera: {le:?} {lf:?}"); + // Moving the camera carries the light. + s.execute("prop.set", json!({"layer": cam, "path": "transform/position", "value": [100, 180, -800]})).unwrap(); + let (le2, _) = frame(&s, light); + assert!((le2 - vec3(100.0, 180.0, -800.0)).length() < 1e-6, "{le2:?}"); + // Again: released where it is (unparented, aimed the same way). + let r = s.execute("light.controlWithCamera", json!({"layers": [light]})).unwrap(); + assert_eq!(r["controlled"], false); + let l = layer_in(&s, cid(&s), light); + assert_eq!(l.parent, None); + let (le3, lf3) = frame(&s, light); + assert!((le3 - le2).length() < 1e-6, "{le3:?}"); + let (_, cf2) = frame(&s, cam); + assert!((lf3 - cf2).length() < 1e-6, "{lf3:?} {cf2:?}"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(layer_in(&s, cid(&s), light).parent, Some(LayerId(cam))); + // Not a light → error. + assert!(s.execute("light.controlWithCamera", json!({"layers": [cam]})).is_err()); +} + +// ---------------------------------------------------------------- from 3D model + +struct DiskModels; + +impl FootageSource for DiskModels { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn model(&self, _: ItemId, f: &Footage) -> Option> { + let bytes = std::fs::read(&f.path).ok()?; + effectcraft_model::load(&f.path, &bytes, &|_| None).ok().map(Arc::new) + } +} + +struct ModelImporter; + +impl Importer for ModelImporter { + fn probe(&self, path: &str) -> Result { + effectcraft_model::format_of(path).ok_or("not a model")?; + Ok(Footage { path: path.into(), kind: FootageKind::Model, frame_rate: FrameRate::FPS_30, codec: "glTF".into(), ..Default::default() }) + } +} + +const RIGGED: &str = r#"{ + "asset": {"version": "2.0"}, + "scenes": [{"nodes": [0, 1, 2]}], + "nodes": [ + {"name": "ShotCam", "camera": 0, "translation": [0, 0, 5]}, + {"name": "Key", "translation": [0, 3, 0], "rotation": [-0.7071068, 0, 0, 0.7071068], "extensions": {"KHR_lights_punctual": {"light": 0}}}, + {"name": "Fill", "translation": [2, 0, 0], "extensions": {"KHR_lights_punctual": {"light": 1}}} + ], + "cameras": [{"type": "perspective", "name": "Shot", "perspective": {"yfov": 0.6, "znear": 0.1}}], + "extensions": {"KHR_lights_punctual": {"lights": [ + {"type": "spot", "name": "Key Light", "color": [1, 0.8, 0.6], "intensity": 1.5, "spot": {"innerConeAngle": 0.25, "outerConeAngle": 0.5}}, + {"type": "point", "name": "Fill Light", "intensity": 0.5, "range": 4} + ]}} +}"#; + +fn rigged_model(s: &mut Session) -> u64 { + let dir = std::env::temp_dir().join(format!("ec-rig3d-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("rigged.gltf"); + std::fs::write(&path, RIGGED).unwrap(); + let r = s.execute("layer.newModel", json!({"path": path.to_string_lossy()})).unwrap(); + r["layer"].as_u64().unwrap() +} + +#[test] +fn cameras_and_lights_from_model() { + let mut s = session(); + let model = rigged_model(&mut s); + let m = layer_in(&s, cid(&s), model); + let unit = m.props.prop("geometryOptions/unitScale").unwrap().value.as_f64(); + let pos = v3(&m, "transform/position"); + s.execute("layer.select", json!({"layers": [model]})).unwrap(); + let r = s.execute("camera.fromModel", json!({})).unwrap(); + let cams = r["layers"].as_array().unwrap(); + assert_eq!(cams.len(), 1); + let c = layer_in(&s, cid(&s), cams[0].as_u64().unwrap()); + assert_eq!(c.name, "Shot"); + // glTF +Z (towards the viewer) is AE −Z; the camera looks into the scene (+Z). + let want = [pos[0], pos[1], pos[2] - 5.0 * unit]; + assert!(close3(v3(&c, "transform/position"), want, 1e-6), "{:?} vs {want:?}", v3(&c, "transform/position")); + let zoom = 180.0 / (0.3f64).tan(); + assert!((c.props.prop("cameraOptions/zoom").unwrap().value.as_f64() - zoom).abs() < 1e-6); + let (_, f) = frame(&s, c.id.0); + assert!((f - vec3(0.0, 0.0, 1.0)).length() < 1e-6, "{f:?}"); + // Lights: a spot (cone 2 × outer, feather from inner/outer) and a point light. + s.execute("layer.select", json!({"layers": [model]})).unwrap(); + let r = s.execute("light.fromModel", json!({})).unwrap(); + let ls: Vec = r["layers"].as_array().unwrap().iter().map(|v| layer_in(&s, cid(&s), v.as_u64().unwrap())).collect(); + assert_eq!(ls.len(), 2); + let key = ls.iter().find(|l| l.name == "Key Light").unwrap(); + assert_eq!(key.source, LayerSource::Light { kind: LightKind::Spot }); + assert!((key.props.prop("lightOptions/coneAngle").unwrap().value.as_f64() - 1.0f64.to_degrees()).abs() < 1e-6); + assert!((key.props.prop("lightOptions/coneFeather").unwrap().value.as_f64() - 50.0).abs() < 1e-6); + assert!((key.props.prop("lightOptions/intensity").unwrap().value.as_f64() - 150.0).abs() < 1e-9); + // Points down in glTF (−Y) = down on screen in AE (+Y). + let (ke, kf) = frame(&s, key.id.0); + assert!((kf - vec3(0.0, 1.0, 0.0)).length() < 1e-5, "{kf:?}"); + assert!((ke.y - (pos[1] - 3.0 * unit)).abs() < 1e-6, "{ke:?}"); + let fill = ls.iter().find(|l| l.name == "Fill Light").unwrap(); + assert_eq!(fill.source, LayerSource::Light { kind: LightKind::Point }); + assert!((fill.props.prop("lightOptions/falloffDistance").unwrap().value.as_f64() - 400.0).abs() < 1e-9); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.active_comp().unwrap().layer(key.id).is_none()); + // Not a model layer → error. + assert!(s.execute("camera.fromModel", json!({"layers": [c.id.0]})).is_err()); +} + +// ---------------------------------------------------------------- environment background + +fn px(img: &effectcraft_raster::Image, x: u32, y: u32) -> [f32; 4] { + img.data[(y * img.width + x) as usize] +} + +#[test] +fn environment_background_draws_the_sky_through_the_camera() { + let mut s = session(); + let scene = cid(&s); + // An equirectangular "sky": red upper hemisphere, blue lower. + let sky = s.execute("comp.new", json!({"name": "Sky", "width": 200, "height": 100, "frameRate": 30, "duration": 2})).unwrap()["comp"].as_u64().unwrap(); + s.execute("layer.newSolid", json!({"color": "#0000ff", "width": 200, "height": 100})).unwrap(); + let top = s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 200, "height": 50})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.set", json!({"layer": top, "path": "transform/position", "value": [100, 25]})).unwrap(); + s.execute("comp.open", json!({"comp": scene})).unwrap(); + let src = s.execute("layer.addItem", json!({"item": sky})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.environment", json!({"layers": [src], "on": true})).unwrap(); + // No environment light yet. + assert!(s.execute("light.environmentBackground", json!({})).is_err()); + let env = s.execute("layer.newLight", json!({"kind": "Environment"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("light.environmentBackground", json!({})).unwrap(); + assert_eq!(r["light"].as_u64(), Some(env)); + assert_eq!(r["source"].as_u64(), Some(src)); + let bg = layer_in(&s, scene, r["layer"].as_u64().unwrap()); + assert!(bg.environment_background && bg.is_3d()); + assert_eq!(bg.source, LayerSource::Comp { item: ItemId(sky) }); + assert_eq!(s.active_comp().unwrap().layers.last().unwrap().id, bg.id, "at the bottom"); + let img = s.render(scene, Tick::ZERO, RenderOpts::default()); + // Looking at the horizon: rays above it see red, below it blue. + let (t, b) = (px(&img, 320, 5), px(&img, 320, 354)); + assert!(t[0] > 0.9 && t[2] < 0.1 && t[3] > 0.99, "top {t:?}"); + assert!(b[2] > 0.9 && b[0] < 0.1, "bottom {b:?}"); + // Serde keeps the flag; undo removes the layer. + let json = serde_json::to_value(&bg).unwrap(); + assert_eq!(json["environment_background"], Value::Bool(true)); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.active_comp().unwrap().layer(bg.id).is_none()); +} diff --git a/crates/expr/src/prelude.js b/crates/expr/src/prelude.js index 1655f75d..39770ee1 100644 --- a/crates/expr/src/prelude.js +++ b/crates/expr/src/prelude.js @@ -526,6 +526,8 @@ Layer.prototype = { get opacity() { return this.transform.opacity; }, get anchorPoint() { return this.transform.anchorPoint; }, get text() { return this.__group('text'); }, + get cameraOption() { return this.__group('cameraOptions'); }, + get lightOption() { return this.__group('lightOptions'); }, get marker() { return new Markers(this.__c, this.__l); }, effect: function (k) { var g = __child(this.__c, this.__l, '', 'effects'); diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index d7668fcc..b454b5bc 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -424,4 +424,46 @@ mod tests { s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "50"})).unwrap(); assert_eq!(value(&s, a, "transform/opacity").as_f64(), 50.0); } + /// Create Stereo 3D Rig: the eye cameras' expressions follow the master camera and the + /// Stereo 3D Controls (and agree with the values the command wrote). + #[test] + fn stereo_rig_expressions_follow_the_master_camera() { + let mut s = super::session(); + s.execute("comp.new", json!({"name": "Shot", "width": 640, "height": 360, "frameRate": 30, "duration": 2})).unwrap(); + let src = s.active_comp_id().unwrap(); + let cam = s.execute("layer.newCamera", json!({"name": "Main", "position": [320, 180, -800], "poi": [320, 180, 0], "zoom": 900})).unwrap()["layer"] + .as_u64() + .unwrap(); + let r = s.execute("camera.stereoRig", json!({"sceneDepth": 4})).unwrap(); + let ctl = r["controls"].as_u64().unwrap(); + let left = effectcraft_engine::project::ItemId(r["leftComp"].as_u64().unwrap()); + // The left eye camera's evaluated value (expressions on). + let v3 = |s: &effectcraft_engine::Session, path: &str| { + let comp = s.project.comp(left).unwrap(); + let layer = &comp.layers[0]; + let ctx = effectcraft_engine::render::EvalCtx { + project: &s.project, + comp_id: left, + comp, + time: Default::default(), + expr: s.expr.as_deref(), + footage: None, + }; + ctx.value(layer, layer.props.prop(path).unwrap()).as_vec3() + }; + let close = |a: [f64; 3], b: [f64; 3]| (0..3).all(|i| (a[i] - b[i]).abs() < 1e-6); + let d = 0.04 * 640.0; + assert!(close(v3(&s, "transform/position"), [320.0 - d / 2.0, 180.0, -800.0]), "{:?}", v3(&s, "transform/position")); + assert!(close(v3(&s, "transform/poi"), [320.0 - d / 2.0, 180.0, 0.0]), "{:?}", v3(&s, "transform/poi")); + // Moving the master moves the eyes. + s.execute("comp.open", json!({"comp": src.0})).unwrap(); + s.execute("prop.set", json!({"layer": cam, "path": "transform/position", "value": [300, 180, -800]})).unwrap(); + s.execute("prop.set", json!({"layer": cam, "path": "transform/poi", "value": [300, 180, 0]})).unwrap(); + assert!(close(v3(&s, "transform/position"), [300.0 - d / 2.0, 180.0, -800.0]), "{:?}", v3(&s, "transform/position")); + // Controls: Center & Left puts the left eye a full separation left; convergence toes in. + s.execute("prop.set", json!({"layer": ctl, "path": "effects/#1/configuration", "value": 2})).unwrap(); + s.execute("prop.set", json!({"layer": ctl, "path": "effects/#1/convergence", "value": true})).unwrap(); + assert!(close(v3(&s, "transform/position"), [300.0 - d, 180.0, -800.0]), "{:?}", v3(&s, "transform/position")); + assert!(close(v3(&s, "transform/poi"), [300.0, 180.0, 0.0]), "{:?}", v3(&s, "transform/poi")); + } } diff --git a/crates/model/src/gltf.rs b/crates/model/src/gltf.rs index 375884bc..fa2e930a 100644 --- a/crates/model/src/gltf.rs +++ b/crates/model/src/gltf.rs @@ -7,8 +7,9 @@ //! (base colour, metallic-roughness, normal, occlusion and emissive textures; alpha modes; //! double-sided; `KHR_materials_emissive_strength`, `KHR_materials_unlit`), embedded or //! external PNG/JPEG images with sampler wrap modes, skins and animations (step, linear and -//! cubic-spline translation/rotation/scale). Sparse accessors, morph targets, cameras and -//! lights are ignored. +//! cubic-spline translation/rotation/scale), perspective and orthographic cameras, and +//! `KHR_lights_punctual` lights (directional, point, spot). Sparse accessors and morph targets +//! are ignored. use std::sync::Arc; @@ -363,6 +364,8 @@ pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result< matrix, mesh: idx(n, "mesh").filter(|&m| m < meshes.len()), skin: idx(n, "skin"), + camera: idx(n, "camera"), + light: n.get("extensions").and_then(|e| e.get("KHR_lights_punctual")).and_then(|e| idx(e, "light")), children: arr(n, "children").iter().filter_map(Value::as_u64).map(|c| c as usize).collect(), }); } @@ -434,7 +437,54 @@ pub fn parse(bytes: &[u8], resolve: &dyn Fn(&str) -> Option>) -> Result< animations.push(anim); } - Ok(Model { meshes, nodes, roots, materials, textures, skins, animations }) + let cameras = arr(&j, "cameras").iter().enumerate().map(|(i, c)| camera(c, i)).collect(); + let lights = j + .get("extensions") + .and_then(|e| e.get("KHR_lights_punctual")) + .map(|e| arr(e, "lights").iter().enumerate().map(|(i, l)| light(l, i)).collect()) + .unwrap_or_default(); + Ok(Model { meshes, nodes, roots, materials, textures, skins, animations, cameras, lights }) +} + +fn name_or(v: &Value, d: String) -> String { + v.get("name").and_then(Value::as_str).filter(|s| !s.is_empty()).map(str::to_string).unwrap_or(d) +} + +/// A glTF camera (glTF 2.0 §5.12): `perspective` {yfov, aspectRatio?, znear} or +/// `orthographic` {xmag, ymag}. +fn camera(c: &Value, i: usize) -> crate::ModelCamera { + let projection = match (c.get("type").and_then(Value::as_str), c.get("orthographic")) { + (Some("orthographic"), Some(o)) => crate::CameraProjection::Orthographic { xmag: num(o, "xmag", 1.0), ymag: num(o, "ymag", 1.0) }, + _ => { + let p = c.get("perspective").cloned().unwrap_or(Value::Null); + crate::CameraProjection::Perspective { + yfov: num(&p, "yfov", 0.8).clamp(1e-3, 3.1), + aspect: p.get("aspectRatio").and_then(Value::as_f64), + znear: num(&p, "znear", 0.01), + } + } + }; + crate::ModelCamera { name: name_or(c, format!("Camera {}", i + 1)), projection } +} + +/// A `KHR_lights_punctual` light: type, color, intensity, range, spot cone angles. +fn light(l: &Value, i: usize) -> crate::ModelLight { + let kind = match l.get("type").and_then(Value::as_str) { + Some("directional") => crate::ModelLightKind::Directional, + Some("spot") => { + let s = l.get("spot").cloned().unwrap_or(Value::Null); + let outer = num(&s, "outerConeAngle", std::f64::consts::FRAC_PI_4).clamp(1e-3, std::f64::consts::FRAC_PI_2); + crate::ModelLightKind::Spot { inner: num(&s, "innerConeAngle", 0.0).clamp(0.0, outer), outer } + } + _ => crate::ModelLightKind::Point, + }; + crate::ModelLight { + name: name_or(l, format!("Light {}", i + 1)), + kind, + color: nums(l, "color", [1.0; 3]), + intensity: num(l, "intensity", 1.0).max(0.0), + range: l.get("range").and_then(Value::as_f64), + } } fn primitive(d: &Doc, p: &Value, materials: &[Material]) -> Result> { diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 290e5500..93a6a9f9 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -295,6 +295,10 @@ pub struct Node { pub matrix: Option, pub mesh: Option, pub skin: Option, + /// A camera of [`Model::cameras`] at this node (looking down the node's −Z, +Y up). + pub camera: Option, + /// A light of [`Model::lights`] at this node (`KHR_lights_punctual`, shining down −Z). + pub light: Option, pub children: Vec, } @@ -308,6 +312,8 @@ impl Default for Node { matrix: None, mesh: None, skin: None, + camera: None, + light: None, children: vec![], } } @@ -495,6 +501,60 @@ pub struct Model { pub textures: Vec>, pub skins: Vec, pub animations: Vec, + /// Cameras embedded in the file (glTF `cameras`). + pub cameras: Vec, + /// Punctual lights embedded in the file (glTF `KHR_lights_punctual`). + pub lights: Vec, +} + +/// A camera embedded in a model file. +#[derive(Clone, Debug, PartialEq)] +pub struct ModelCamera { + pub name: String, + pub projection: CameraProjection, +} + +/// glTF camera projections. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum CameraProjection { + /// Vertical field of view (radians), optional aspect ratio, near plane. + Perspective { yfov: f64, aspect: Option, znear: f64 }, + /// Half the orthographic view's width and height (model units). + Orthographic { xmag: f64, ymag: f64 }, +} + +/// Kinds of `KHR_lights_punctual` lights. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ModelLightKind { + Directional, + Point, + /// Inner and outer cone angles (radians, from the axis). + Spot { + inner: f64, + outer: f64, + }, +} + +/// A punctual light embedded in a model file. +#[derive(Clone, Debug, PartialEq)] +pub struct ModelLight { + pub name: String, + pub kind: ModelLightKind, + /// Linear RGB. + pub color: [f64; 3], + /// Candela (point, spot) or lux (directional). + pub intensity: f64, + pub range: Option, +} + +/// A camera or light placed in model space by [`Model::placed`]. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Placed { + /// Index into [`Model::cameras`] or [`Model::lights`]. + pub index: usize, + pub node: usize, + /// Node → model space. + pub world: Mat4, } /// One mesh placed in model space by [`Model::instances`]. @@ -579,6 +639,31 @@ impl Model { out } + /// The scene's cameras (`lights = false`) or lights (`true`) placed at animation `clip` + /// time `t`, in node order. + pub fn placed(&self, lights: bool, clip: Option, t: f64) -> Vec { + let world = self.world_pose(&self.local_pose(clip, t)); + let mut reach = vec![false; self.nodes.len()]; + let mut stack: Vec = self.roots.clone(); + while let Some(i) = stack.pop() { + if i >= self.nodes.len() || reach[i] { + continue; + } + reach[i] = true; + stack.extend(self.nodes[i].children.iter().copied()); + } + let count = if lights { self.lights.len() } else { self.cameras.len() }; + self.nodes + .iter() + .enumerate() + .filter(|(i, _)| reach[*i]) + .filter_map(|(i, n)| { + let index = if lights { n.light } else { n.camera }?; + (index < count).then_some(Placed { index, node: i, world: world[i] }) + }) + .collect() + } + /// Model-space bounds of the rest pose (None for an empty model). pub fn bounds(&self) -> Option<([f64; 3], [f64; 3])> { let mut lo = [f64::INFINITY; 3]; diff --git a/crates/model/src/tests.rs b/crates/model/src/tests.rs index 31040b7c..1a3b1322 100644 --- a/crates/model/src/tests.rs +++ b/crates/model/src/tests.rs @@ -316,3 +316,47 @@ fn contours_group_into_outlines_by_nesting() { assert_eq!(g.len(), 3); assert_eq!(g.iter().map(|o| o.holes.len()).sum::(), 2); } + +#[test] +fn gltf_cameras_and_punctual_lights() { + let doc = r#"{ + "asset": {"version": "2.0"}, + "scene": 0, + "scenes": [{"nodes": [0, 2]}], + "nodes": [ + {"name": "Rig", "translation": [0, 0, 10], "children": [1]}, + {"name": "Cam", "camera": 0, "translation": [1, 0, 0]}, + {"name": "Sun", "rotation": [-0.7071068, 0, 0, 0.7071068], "extensions": {"KHR_lights_punctual": {"light": 1}}}, + {"name": "Orphan", "camera": 1} + ], + "cameras": [ + {"type": "perspective", "name": "Shot", "perspective": {"yfov": 0.5, "znear": 0.1, "aspectRatio": 1.5}}, + {"type": "orthographic", "orthographic": {"xmag": 2, "ymag": 1, "znear": 0, "zfar": 10}} + ], + "extensions": {"KHR_lights_punctual": {"lights": [ + {"type": "spot", "name": "Key", "color": [1, 0.5, 0.25], "intensity": 3, "spot": {"innerConeAngle": 0.2, "outerConeAngle": 0.4}}, + {"type": "directional", "intensity": 2} + ]}} + }"#; + let m = crate::gltf::parse(doc.as_bytes(), &|_| None).unwrap(); + assert_eq!(m.cameras.len(), 2); + assert_eq!(m.cameras[0].name, "Shot"); + assert_eq!(m.cameras[0].projection, CameraProjection::Perspective { yfov: 0.5, aspect: Some(1.5), znear: 0.1 }); + assert_eq!(m.cameras[1].projection, CameraProjection::Orthographic { xmag: 2.0, ymag: 1.0 }); + assert_eq!(m.lights.len(), 2); + assert_eq!(m.lights[0].kind, ModelLightKind::Spot { inner: 0.2, outer: 0.4 }); + assert_eq!(m.lights[0].color, [1.0, 0.5, 0.25]); + assert_eq!(m.lights[1].kind, ModelLightKind::Directional); + assert_eq!(m.lights[1].name, "Light 2"); + // Only nodes of the scene are placed; the camera node inherits its parent's translation. + let cams = m.placed(false, None, 0.0); + assert_eq!(cams.len(), 1); + assert_eq!((cams[0].index, cams[0].node), (0, 1)); + let eye = cams[0].world.apply(vec3(0.0, 0.0, 0.0)); + assert!((eye.x - 1.0).abs() < 1e-9 && (eye.z - 10.0).abs() < 1e-9, "{eye:?}"); + let lights = m.placed(true, None, 0.0); + assert_eq!(lights.len(), 1); + // −90° about X: the light's −Z axis points down (−Y in glTF). + let d = lights[0].world.apply_vec(vec3(0.0, 0.0, -1.0)); + assert!((d.y + 1.0).abs() < 1e-6, "{d:?}"); +} diff --git a/crates/project/src/build.rs b/crates/project/src/build.rs index a201879d..a75fac72 100644 --- a/crates/project/src/build.rs +++ b/crates/project/src/build.rs @@ -818,6 +818,7 @@ pub fn layer(project: &mut Project, comp: &Comp, name: &str, source: LayerSource markers: vec![], markers_locked: false, environment: false, + environment_background: false, auto_orient, props: root, } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 409ae741..fb7f7208 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -542,6 +542,10 @@ pub struct Layer { /// reflections) of Advanced 3D comps instead of being drawn. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub environment: bool, + /// Layer ▸ Light ▸ Create Environment Light Background Layer: the layer's (equirectangular) + /// image is drawn as the 3D scene's backdrop, seen through the camera, instead of as a card. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub environment_background: bool, /// The property tree (Masks, Effects, Transform, Text, Contents, Camera/Light options…). pub props: PropGroup, } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index ff71fb57..86d1bd9a 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -1016,7 +1016,7 @@ impl<'a> Renderer<'a> { /// The region of interest of a top-level 2D frame: (x, y) offset of the output in output /// pixels and its size. `None` renders the whole comp (no ROI, or a nested comp). - fn roi_offset(&self) -> Option<(f64, f64, u32, u32)> { + pub(crate) fn roi_offset(&self) -> Option<(f64, f64, u32, u32)> { let r = self.opts.roi.filter(|_| self.depth == 0)?; let s = self.opts.scale; if r[2] <= 0.0 || r[3] <= 0.0 { @@ -1218,17 +1218,29 @@ impl<'a> Renderer<'a> { } /// Apply the track matte / preserve transparency to an isolated layer render, then blend. - fn composite_iso(&self, ctx: &EvalCtx, layer: &Layer, mut iso: Image, canvas: &mut Image, opacity: f32) { + fn composite_iso(&self, ctx: &EvalCtx, layer: &Layer, iso: Image, canvas: &mut Image, opacity: f32) { + self.composite_iso_with(ctx, layer, iso, canvas, opacity, None); + } + + /// [`Self::composite_iso`] with the track matte's pixels already drawn (`matte`, e.g. a 3D + /// matte seen through the camera); otherwise the matte layer is placed in 2D. + pub(crate) fn composite_iso_with(&self, ctx: &EvalCtx, layer: &Layer, mut iso: Image, canvas: &mut Image, opacity: f32, matte_img: Option) { let matte = layer.track_matte.and_then(|tm| ctx.comp.layer(tm.layer).filter(|m| m.id != layer.id).map(|m| (m, tm.kind))); let preserve = layer.preserve_transparency; if let Some((m, kind)) = matte { - let mut mimg = Image::new(canvas.width, canvas.height); - if m.is_active_at(ctx.time) - && let Some(mb) = self.layer_buf(ctx, m) - { - let mo = ctx.opacity(m) as f32; - self.place(ctx, m, &mb, &mut mimg, BlendMode::Normal, mo); - } + let mimg = match matte_img { + Some(i) if i.width == canvas.width && i.height == canvas.height => i, + _ => { + let mut mimg = Image::new(canvas.width, canvas.height); + if m.is_active_at(ctx.time) + && let Some(mb) = self.layer_buf(ctx, m) + { + let mo = ctx.opacity(m) as f32; + self.place(ctx, m, &mb, &mut mimg, BlendMode::Normal, mo); + } + mimg + } + }; iso.data.par_iter_mut().zip(mimg.data.par_iter()).for_each(|(p, q)| { let k = match kind { MatteKind::Alpha => q[3], diff --git a/crates/render/src/three_d/adv/mod.rs b/crates/render/src/three_d/adv/mod.rs index 04f23885..f8690b8f 100644 --- a/crates/render/src/three_d/adv/mod.rs +++ b/crates/render/src/three_d/adv/mod.rs @@ -140,25 +140,135 @@ pub fn scene_of(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> scene::build(r, ctx, run, out) } -/// Draw a run of 3D layers with the Advanced 3D renderer into `canvas`. -pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut Image) { - let s = scene::build(r, ctx, run, (canvas.width, canvas.height)); - if s.indices.is_empty() { - return; +/// Rasterise one scene (GPU when available, else CPU) and resolve it. +fn raster_resolved(r: &Renderer, s: &Scene) -> (Image, Vec) { + let target = r.active_accel().and_then(|a| a.raster_3d(s)).filter(|t| t.width == s.width && t.height == s.height).unwrap_or_else(|| raster::render(s)); + resolve(&target, s.ssaa.max(1)) +} + +/// Number of motion-blur sub-samples for a set of layers (1 = none blurred). +fn mb_samples(r: &Renderer, ctx: &EvalCtx, layers: &[&Layer]) -> usize { + if !layers.iter().any(|l| scene::motion_blurred(r, ctx, l)) { + return 1; } - let target = r.active_accel().and_then(|a| a.raster_3d(&s)).filter(|t| t.width == s.width && t.height == s.height).unwrap_or_else(|| raster::render(&s)); - let (mut img, depth) = resolve(&target, s.ssaa.max(1)); + if r.opts.draft { 4 } else { ctx.comp.motion_blur_samples.clamp(2, 64) as usize } +} + +/// Render `layers` as one Advanced 3D scene: premultiplied pixels in the canvas's encoding and +/// the camera depth of each pixel (∞ where nothing was drawn). With motion blur the scene is +/// rendered at sub-samples spread over the comp's shutter (angle and phase) and averaged; the +/// depth is the nearest over the samples. +pub(crate) fn render_layers(r: &Renderer, ctx: &EvalCtx, layers: &[&Layer], out: (u32, u32)) -> Option<(Image, Vec)> { + let n = mb_samples(r, ctx, layers); + let (mut img, depth, linear) = if n <= 1 { + let s = scene::build(r, ctx, layers, out); + if s.indices.is_empty() { + return None; + } + let (img, depth) = raster_resolved(r, &s); + (img, depth, s.linear_io) + } else { + let fd = ctx.comp.frame_duration().seconds(); + let (angle, phase) = (ctx.comp.shutter_angle / 360.0, ctx.comp.shutter_phase / 360.0); + let mut acc = Image::new(out.0, out.1); + let mut depth = vec![f32::INFINITY; (out.0 * out.1) as usize]; + let mut linear = false; + let mut any = false; + let k = 1.0 / n as f32; + for i in 0..n { + let f = phase + angle * i as f64 / (n - 1) as f64; + let sub = ctx.at(ctx.time + effectcraft_time::Tick::from_seconds_f64(f * fd)); + let s = scene::build_at(r, ctx, Some(&sub), layers, out); + linear = s.linear_io; + if s.indices.is_empty() { + continue; + } + any = true; + let (img, d) = raster_resolved(r, &s); + acc.data.par_iter_mut().zip(img.data.par_iter()).for_each(|(a, p)| { + for c in 0..4 { + a[c] += p[c] * k; + } + }); + depth.par_iter_mut().zip(d.par_iter()).for_each(|(a, b)| *a = a.min(*b)); + } + if !any { + return None; + } + (acc, depth, linear) + }; let cam = camera_for(r, ctx); if let Some(dof) = cam.dof.filter(|_| !ctx.comp.draft_3d && !r.opts.draft) { img = depth_of_field(&img, &depth, &dof, r.opts.scale); } - if !s.linear_io { + if !linear { encode(&mut img); } - canvas.data.par_iter_mut().zip(img.data.par_iter()).for_each(|(d, s)| { - let k = 1.0 - s[3]; - *d = [s[0] + d[0] * k, s[1] + d[1] * k, s[2] + d[2] * k, s[3] + d[3] * k]; - }); + Some((img, depth)) +} + +/// Layers whose blend mode, track matte or Preserve Transparency needs the 2D compositing path. +fn needs_2d_composite(ctx: &EvalCtx, l: &Layer) -> bool { + l.blend_mode != effectcraft_color::BlendMode::Normal + || l.preserve_transparency + || l.track_matte.is_some_and(|tm| tm.layer != l.id && ctx.comp.layer(tm.layer).is_some()) +} + +/// Draw a run of 3D layers with the Advanced 3D renderer into `canvas`. +/// +/// Layers with a blend mode, a track matte or Preserve Transparency can't blend inside the +/// depth-buffered scene: they are rendered on their own (lit by the same lights, through the +/// same camera), hidden where the rest of the run is nearer, then composited through the 2D +/// path (matte, Preserve Transparency, blend mode) from the farthest to the nearest. +pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut Image) { + let out = (canvas.width, canvas.height); + let (special, plain): (Vec<&Layer>, Vec<&Layer>) = run.iter().copied().partition(|l| needs_2d_composite(ctx, l)); + let main = render_layers(r, ctx, &plain, out); + if let Some((img, _)) = &main { + canvas.data.par_iter_mut().zip(img.data.par_iter()).for_each(|(d, s)| { + let k = 1.0 - s[3]; + *d = [s[0] + d[0] * k, s[1] + d[1] * k, s[2] + d[2] * k, s[3] + d[3] * k]; + }); + } + if special.is_empty() { + return; + } + let cam = camera_for(r, ctx); + let mut order: Vec<(f64, &Layer)> = special + .into_iter() + .map(|l| { + let anchor = l.transform().map_or([0.0; 3], |tr| ctx.v3(l, tr, "anchor", [0.0; 3])); + (cam.depth(ctx.world_matrix(l).apply(effectcraft_geom::Vec3::from(anchor))), l) + }) + .collect(); + order.sort_by(|a, b| b.0.total_cmp(&a.0)); + for (_, l) in order { + let Some((mut iso, depth)) = render_layers(r, ctx, &[l], out) else { continue }; + if let Some((m, md)) = &main { + iso.data.par_iter_mut().zip(depth.par_iter()).zip(m.data.par_iter().zip(md.par_iter())).for_each(|((p, z), (q, mz))| { + if *mz < *z - 0.01 * z.abs().max(1.0) { + let k = 1.0 - q[3].clamp(0.0, 1.0); + for c in p.iter_mut() { + *c *= k; + } + } + }); + } + // A 3D matte layer is drawn through the camera too. + let matte3d = l.track_matte.and_then(|tm| ctx.comp.layer(tm.layer).filter(|m| m.id != l.id && m.is_3d())).map(|m| { + let mut solo = m.clone(); + solo.track_matte = None; + solo.blend_mode = effectcraft_color::BlendMode::Normal; + solo.preserve_transparency = false; + solo.switches.video = true; + if m.is_active_at(ctx.time) { + render_layers(r, ctx, &[&solo], out).map(|(i, _)| i).unwrap_or_else(|| Image::new(out.0, out.1)) + } else { + Image::new(out.0, out.1) + } + }); + r.composite_iso_with(ctx, l, iso, canvas, 1.0, matte3d); + } } #[cfg(test)] diff --git a/crates/render/src/three_d/adv/scene.rs b/crates/render/src/three_d/adv/scene.rs index 14251745..2c3b03bc 100644 --- a/crates/render/src/three_d/adv/scene.rs +++ b/crates/render/src/three_d/adv/scene.rs @@ -806,6 +806,20 @@ fn ssaa_for(r: &Renderer, out: (u32, u32)) -> u32 { /// Build the scene of a run of 3D layers. pub(crate) fn build(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Scene { + build_at(r, ctx, None, run, out) +} + +/// Whether a layer's motion is blurred (Motion Blur switch, comp setting and render option). +pub(crate) fn motion_blurred(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> bool { + r.opts.motion_blur && ctx.comp.enable_motion_blur && layer.switches.motion_blur +} + +/// [`build`] at a motion-blur sub-sample: layers with Motion Blur on, the camera and the lights +/// are evaluated at `sub`'s time; the other layers stay at `ctx`'s. +pub(crate) fn build_at(r: &Renderer, ctx: &EvalCtx, sub: Option<&EvalCtx>, run: &[&Layer], out: (u32, u32)) -> Scene { + let base = ctx; + let at = sub.unwrap_or(ctx); + let ctx = at; let cam = camera_for(r, ctx); let lin = r.pipe.linear || r.pipe.linear_blend; let ssaa = ssaa_for(r, out); @@ -816,6 +830,7 @@ pub(crate) fn build(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32) cam, }; for layer in run { + let ctx = if sub.is_some() && motion_blurred(r, base, layer) { at } else { base }; if layer.environment || layer.switches.adjustment || ctx.opacity(layer) <= 0.0 || !layer.has_video() { continue; } diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs index b1d94413..cccc1ec9 100644 --- a/crates/render/src/three_d/adv/tests.rs +++ b/crates/render/src/three_d/adv/tests.rs @@ -312,3 +312,116 @@ fn extruded_text_renders_as_a_mesh() { let img = render(&p, cid); assert!(img.data.iter().filter(|q| q[3] > 0.5).count() > 200); } + +// ---------------------------------------------------------------- motion blur, blend modes, mattes + +fn solid(p: &mut Project, color: [f32; 3], w: u32, h: u32) -> ItemId { + p.add_item("S", Default::default(), None, ItemKind::Solid(effectcraft_project::Solid { color, width: w, height: h, pixel_aspect: 1.0 })) +} + +/// Pixels of row `y` whose alpha is partial (motion-blur smear or anti-aliasing). +fn partial(img: &crate::Image, y: u32) -> usize { + (0..img.width).filter(|x| (0.05..0.95).contains(&img.data[(y * img.width + x) as usize][3])).count() +} + +#[test] +fn motion_blur_smears_moving_meshes() { + let (mut p, cid) = project(160, 120); + add(&mut p, cid, LayerSource::Primitive { kind: PrimitiveKind::Cube }, |l| { + set(l, "geometryOptions/width", Value::Scalar(30.0)); + set(l, "geometryOptions/height", Value::Scalar(30.0)); + set(l, "geometryOptions/depth", Value::Scalar(30.0)); + // 60 px per frame to the right. + let pr = l.props.prop_mut("transform/position").unwrap(); + pr.keys = vec![ + effectcraft_keyframe::Keyframe::new(Tick::ZERO, Value::Vec3([50.0, 60.0, 0.0])), + effectcraft_keyframe::Keyframe::new(Tick::from_seconds_f64(1.0), Value::Vec3([50.0 + 30.0 * 60.0, 60.0, 0.0])), + ]; + }); + let sharp = render(&p, cid); + let hard = partial(&sharp, 60); + assert!(hard <= 4, "no blur without the switch: {hard}"); + p.comp_mut(cid).unwrap().layers[0].switches.motion_blur = true; + let blurred = render(&p, cid); + let smear = partial(&blurred, 60); + // A 180° shutter smears over half a frame of motion (≈ 30 px). + assert!(smear > 20, "smeared: {smear}"); + // Coverage is conserved: the summed alpha along the row stays about the cube's width. + let sum = |im: &crate::Image| (0..160).map(|x| im.data[60 * 160 + x][3]).sum::(); + assert!((sum(&blurred) - sum(&sharp)).abs() < 4.0, "{} vs {}", sum(&blurred), sum(&sharp)); + // The comp's motion blur switch and the render option turn it off. + p.comp_mut(cid).unwrap().enable_motion_blur = false; + assert!(partial(&render(&p, cid), 60) <= 4); + p.comp_mut(cid).unwrap().enable_motion_blur = true; + let r = Renderer::new(&p, &NoFootage, RenderOpts { motion_blur: false, ..Default::default() }); + assert!(partial(&r.comp_frame(cid, Tick::ZERO), 60) <= 4); + // Shutter angle 0 has no exposure time: no smear. + p.comp_mut(cid).unwrap().shutter_angle = 0.0; + assert!(partial(&render(&p, cid), 60) <= 4); +} + +#[test] +fn blend_modes_and_occlusion_in_advanced_3d() { + let (mut p, cid) = project(120, 90); + let red = solid(&mut p, [1.0, 0.0, 0.0], 120, 90); + let gray = solid(&mut p, [0.5, 0.5, 0.5], 60, 40); + let blue = solid(&mut p, [0.0, 0.0, 1.0], 20, 20); + // Back: a red card; middle: a gray Multiply card; front: a small blue card on the left. + add(&mut p, cid, LayerSource::Solid { item: red }, |l| { + l.switches.three_d = true; + }); + add(&mut p, cid, LayerSource::Solid { item: gray }, |l| { + l.switches.three_d = true; + l.blend_mode = effectcraft_color::BlendMode::Multiply; + set(l, "transform/position", Value::Vec3([60.0, 45.0, -20.0])); + set(l, "transform/anchor", Value::Vec3([30.0, 20.0, 0.0])); + }); + add(&mut p, cid, LayerSource::Solid { item: blue }, |l| { + l.switches.three_d = true; + set(l, "transform/position", Value::Vec3([45.0, 45.0, -40.0])); + set(l, "transform/anchor", Value::Vec3([10.0, 10.0, 0.0])); + }); + let img = render(&p, cid); + let at = |x: usize, y: usize| img.data[y * 120 + x]; + // Multiply over red: half red, no gray. + let c = at(70, 45); + assert!((c[0] - 0.5).abs() < 0.02 && c[1] < 0.02 && c[2] < 0.02 && c[3] > 0.99, "multiplied {c:?}"); + // Outside the gray card: plain red. + let r = at(115, 5); + assert!(r[0] > 0.98 && r[1] < 0.02, "{r:?}"); + // The nearer blue card hides the Multiply card behind it. + let b = at(40, 45); + assert!(b[2] > 0.98 && b[0] < 0.02, "occluded {b:?}"); + // Normal mode for comparison: gray covers the red. + p.comp_mut(cid).unwrap().layers[1].blend_mode = effectcraft_color::BlendMode::Normal; + let n = render(&p, cid).data[45 * 120 + 70]; + assert!((n[0] - 0.5).abs() < 0.02 && (n[1] - 0.5).abs() < 0.02, "{n:?}"); +} + +#[test] +fn track_mattes_in_advanced_3d() { + let (mut p, cid) = project(120, 90); + let white = solid(&mut p, [1.0, 1.0, 1.0], 120, 90); + let small = solid(&mut p, [1.0, 1.0, 1.0], 40, 30); + // Layer order (top first): the matte, then the matted layer. + add(&mut p, cid, LayerSource::Solid { item: white }, |l| l.switches.three_d = true); + add(&mut p, cid, LayerSource::Solid { item: small }, |l| { + l.switches.three_d = true; + l.switches.video = false; + // A 3D matte is seen through the camera. + set(l, "transform/position", Value::Vec3([30.0, 30.0, 0.0])); + set(l, "transform/anchor", Value::Vec3([20.0, 15.0, 0.0])); + }); + let matte = p.comp(cid).unwrap().layers[0].id; + p.comp_mut(cid).unwrap().layers[1].track_matte = Some(effectcraft_project::TrackMatte { layer: matte, kind: effectcraft_project::MatteKind::Alpha }); + let img = render(&p, cid); + let inside = img.data[30 * 120 + 30]; + let outside = img.data[70 * 120 + 100]; + assert!(inside[3] > 0.99 && inside[0] > 0.99, "{inside:?}"); + assert!(outside[3] < 0.01, "{outside:?}"); + // An inverted alpha matte keeps the outside instead. + p.comp_mut(cid).unwrap().layers[1].track_matte = + Some(effectcraft_project::TrackMatte { layer: matte, kind: effectcraft_project::MatteKind::AlphaInverted }); + let img = render(&p, cid); + assert!(img.data[30 * 120 + 30][3] < 0.01 && img.data[70 * 120 + 100][3] > 0.99); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 2647fadd..7be24f1e 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -313,6 +313,15 @@ fn scene_lights(ctx: &EvalCtx) -> Vec { /// Draw a run of consecutive 3D layers (bottom-to-top order) into `canvas`. pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut Image) { + // Environment background layers: an infinitely distant sky behind the rest of the run. + if run.iter().any(|l| l.environment_background) { + for l in run.iter().filter(|l| l.environment_background) { + draw_sky(r, ctx, l, canvas); + } + let rest: Vec<&Layer> = run.iter().copied().filter(|l| !l.environment_background).collect(); + draw_run(r, ctx, &rest, canvas); + return; + } // 3D adjustment layers act on everything below them in the stack (like 2D ones): split. if let Some(k) = run.iter().position(|l| l.switches.adjustment) { draw_run(r, ctx, &run[..k], canvas); @@ -768,3 +777,49 @@ pub(crate) fn aux_pass(r: &Renderer, ctx: &EvalCtx) -> effectcraft_raster::AuxCh aux.manifests = vec![("CryptoObject".into(), manifest_obj), ("CryptoMaterial".into(), manifest_mat)]; aux } + +/// Rotation (radians) of the comp's first Environment light, which also turns its backdrop. +pub(crate) fn environment_rotation(ctx: &EvalCtx) -> f32 { + ctx.comp + .layers + .iter() + .find(|l| matches!(l.source, effectcraft_project::LayerSource::Light { kind: LightKind::Environment }) && l.is_active_at(ctx.time)) + .and_then(|l| l.props.sub("lightOptions").map(|g| ctx.f(l, g, "rotation", 0.0))) + .unwrap_or(0.0) + .to_radians() as f32 +} + +/// Draw an Environment Light Background layer: its (equirectangular) image looked up by each +/// pixel's view direction through the camera, as an infinitely distant backdrop. +pub(crate) fn draw_sky(r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image) { + let Some(buf) = r.blend_layer_buf(ctx, layer) else { return }; + let img = &buf.img; + if img.width == 0 || img.height == 0 { + return; + } + let op = (ctx.opacity(layer) as f32 * r.opacity_mul()).clamp(0.0, 1.0); + if op <= 0.0 { + return; + } + let cam = camera_for(r, ctx); + let (fwd, right, down) = (cam.forward(), cam.right(), cam.down()); + let s = r.opts.scale.max(1e-9); + let (cw, ch) = (ctx.comp.width as f64, ctx.comp.height as f64); + let (ox, oy) = r.roi_offset().map_or((0.0, 0.0), |(x, y, _, _)| (x, y)); + let rot = environment_rotation(ctx); + let (iw, ih) = (img.width as f64, img.height as f64); + canvas.rows_mut().for_each(|(y, row)| { + let cy = (y as f64 + 0.5 + oy) / s; + for (x, d) in row.iter_mut().enumerate() { + let cx = (x as f64 + 0.5 + ox) / s; + let dir = if cam.ortho { fwd } else { (fwd * cam.zoom + right * (cx - cw / 2.0) + down * (cy - ch / 2.0)).normalize() }; + let (u, v) = super::adv::shade::equirect_uv([dir.x as f32, dir.y as f32, dir.z as f32], rot); + // Wrap horizontally across the seam. + let sx = (u as f64 * iw).clamp(0.5, iw - 0.5); + let sy = (v as f64 * ih).clamp(0.5, ih - 0.5); + let p = img.sample_bilinear(sx, sy); + let k = 1.0 - p[3] * op; + *d = [p[0] * op + d[0] * k, p[1] * op + d[1] * k, p[2] * op + d[2] * k, p[3] * op + d[3] * k]; + } + }); +} diff --git a/docs/architecture.md b/docs/architecture.md index 7acd3922..7a2a7a28 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,9 +101,15 @@ transparent ones are sorted back to front and blended. The software rasteriser `Accelerator::raster_3d` with a wgpu render pipeline (`advanced3d.wgsl`) that repeats the shading step for step, and tests compare the two. Resolve, depth of field (a gather blur by each pixel's circle of confusion from the camera's Depth of Field settings) and the conversion -back to the working encoding run on the CPU for both. Not yet: motion blur, track mattes and -blend modes inside an Advanced 3D run, collapsed 3D precomps (drawn flattened), morph targets, -extruded strokes. +back to the working encoding run on the CPU for both. **Motion blur** renders the scene at +sub-samples spread over the comp's shutter (angle, phase, samples per frame) — layers with the +Motion Blur switch, the camera and the lights at the sub-sample time, other layers at the frame +time — and averages them. Layers with a **blend mode, a track matte or Preserve Transparency** +are rendered on their own through the same camera and lights, hidden where the rest of the run +is nearer, and composited from the farthest to the nearest through the 2D path (a 3D matte is +itself drawn through the camera). **Environment Light Background** layers (Layer ▸ Light) draw +their equirectangular image behind the run, looked up by each pixel's view direction. Not yet: +collapsed 3D precomps (drawn flattened), morph targets, extruded strokes. A **layer cache** keeps each layer's finished pixels (source, masks and effects) keyed by a hash of its evaluated inputs, excluding the transform. Static and transform-only layers render once; From aafe46e091556336f764a7ca54eec34202e1cd56 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:15:01 -0400 Subject: [PATCH 02/36] M12.7: Classic 3D runs on the GPU (per-pixel depth sort, lights, ray-cast shadows, DOF); Auto sends 3D comps to the GPU Co-Authored-By: Claude Opus 5.5 --- apps/effectcraft-cli/src/main.rs | 15 +- crates/gpu/src/classic3d.rs | 244 ++++++++++++++ crates/gpu/src/context.rs | 69 +++- crates/gpu/src/effects.rs | 2 +- crates/gpu/src/lib.rs | 3 + crates/gpu/src/shaders/classic3d.wgsl | 460 ++++++++++++++++++++++++++ crates/gpu/src/tests.rs | 54 +-- crates/gpu/src/tests_3d.rs | 220 ++++++++++++ crates/gpu/src/walk.rs | 21 +- crates/render/src/lib.rs | 12 +- crates/render/src/tests.rs | 9 +- crates/render/src/three_d/compose.rs | 179 ++++++++-- crates/render/src/three_d/mod.rs | 1 + 13 files changed, 1214 insertions(+), 75 deletions(-) create mode 100644 crates/gpu/src/classic3d.rs create mode 100644 crates/gpu/src/shaders/classic3d.wgsl create mode 100644 crates/gpu/src/tests_3d.rs diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 4f2192fc..9e1a4a8b 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -761,7 +761,10 @@ fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> { None => s.project.comps().map(|(id, _)| *id).collect(), }; eprintln!("GPU: {} — median of {n} runs (ms/frame)", effectcraft_render::Accelerator::name(&gpu)); - eprintln!("{:<28} {:>5} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9}", "comp", "res", "size", "cpu cold", "gpu cold", "cpu warm", "gpu warm", "gpu view"); + eprintln!( + "{:<28} {:>5} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9} {:>8} {:>9}", + "comp", "res", "size", "cpu cold", "gpu cold", "cpu warm", "gpu warm", "gpu view", "speedup", "gpu≠cpu" + ); for cid in comps { let Some(comp) = s.project.comp(cid) else { continue }; let name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); @@ -797,7 +800,15 @@ fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> { gpu.wait(); }); let size = format!("{}x{}", (comp.width as f64 * scale).round(), (comp.height as f64 * scale).round()); - eprintln!("{name:<28} {label:>5} {size:>10} {cpu_cold:>9.2} {gpu_cold:>9.2} {cpu_warm:>9.2} {gpu_warm:>9.2} {view:>9.2}"); + // Agreement: share of pixels where the GPU frame differs from the CPU reference by + // more than 1/255 on any channel. + let (a, b) = (mk(Backend::Cpu, Some(&cache)), mk(Backend::Gpu, Some(&gcache))); + let off = a.data.iter().zip(&b.data).filter(|(p, q)| (0..4).any(|c| (p[c] - q[c]).abs() > 1.0 / 255.0 + 1e-6)).count(); + let pct = 100.0 * off as f64 / a.data.len().max(1) as f64; + let speedup = cpu_warm / gpu_warm.max(1e-9); + eprintln!( + "{name:<28} {label:>5} {size:>10} {cpu_cold:>9.2} {gpu_cold:>9.2} {cpu_warm:>9.2} {gpu_warm:>9.2} {view:>9.2} {speedup:>7.2}x {pct:>8.3}%" + ); } } Ok(()) diff --git a/crates/gpu/src/classic3d.rs b/crates/gpu/src/classic3d.rs new file mode 100644 index 00000000..0d141c92 --- /dev/null +++ b/crates/gpu/src/classic3d.rs @@ -0,0 +1,244 @@ +//! Classic 3D runs on the GPU (`classic3d.wgsl`): the CPU prepares the planes +//! ([`Renderer::prepare_3d_run`](effectcraft_render::Renderer::prepare_3d_run): layer buffers, +//! homographies, materials, lights, track mattes), the GPU blurs depth of field, packs the +//! buffers into one atlas texture and runs the per-pixel depth sort, lighting, ray-cast shadows +//! and blending of `render::three_d::compose` in one kernel. + +use std::collections::HashMap; + +use effectcraft_geom::Mat4; +use effectcraft_project::LightKind; +use effectcraft_render::three_d::{LightState, PlaneGeo, Run3d}; + +use crate::context::{Enc, GpuImage, Params}; +use crate::ops::mode_id; + +/// Fragments kept per pixel (`MAX_FRAGS` in the shader). Runs that could stack more planes on +/// one pixel render on the CPU. +const MAX_FRAGS: usize = 32; + +/// Little-endian words for the shader's storage structs. +#[derive(Default)] +struct Words(Vec); + +impl Words { + fn f(&mut self, v: [f32; 4]) { + for x in v { + self.0.extend_from_slice(&x.to_le_bytes()); + } + } + fn u(&mut self, v: [u32; 4]) { + for x in v { + self.0.extend_from_slice(&x.to_le_bytes()); + } + } + fn d(&mut self, v: [f64; 4]) { + self.f(v.map(|x| x as f32)); + } + /// Pad an empty array to one element (`size` bytes) so its binding is valid. + fn at_least(mut self, size: usize) -> Vec { + if self.0.len() < size { + self.0.resize(size, 0); + } + self.0 + } +} + +fn affine_rows(w: &mut Words, m: &Mat4) { + for r in 0..3 { + w.d(m.0[r]); + } +} + +/// Shelf-pack rectangles into an atlas no larger than `max` on a side: (width, height, +/// positions). +fn pack(sizes: &[(u32, u32)], max: u32) -> Option<(u32, u32, Vec<(u32, u32)>)> { + let maxw = sizes.iter().map(|s| s.0).max()?; + if maxw > max { + return None; + } + let area: f64 = sizes.iter().map(|s| s.0 as f64 * s.1 as f64).sum(); + let limit = maxw.max((area.sqrt() * 1.25).ceil() as u32).min(max); + let mut order: Vec = (0..sizes.len()).collect(); + order.sort_by_key(|&i| std::cmp::Reverse(sizes[i].1)); + let mut pos = vec![(0, 0); sizes.len()]; + let (mut x, mut y, mut shelf, mut width) = (0u32, 0u32, 0u32, 0u32); + for i in order { + let (w, h) = sizes[i]; + if x + w > limit { + y += shelf; + x = 0; + shelf = 0; + } + pos[i] = (x, y); + x += w; + shelf = shelf.max(h); + width = width.max(x); + } + let height = y + shelf; + (height <= max).then_some((width.max(1), height.max(1), pos)) +} + +/// Upper bound of the planes drawn on any one pixel: a pixel covered by a set of planes lies in +/// each one's box, so every plane of the set overlaps the box of any member. +fn max_overlap(boxes: &[[i64; 4]]) -> usize { + boxes.iter().map(|a| boxes.iter().filter(|b| a[0] < b[2] && b[0] < a[2] && a[1] < b[3] && b[1] < a[3]).count()).max().unwrap_or(0) +} + +fn light_kind(k: LightKind) -> u32 { + match k { + LightKind::Ambient | LightKind::Environment => 0, + LightKind::Parallel => 1, + LightKind::Point => 2, + LightKind::Spot => 3, + } +} + +fn write_geo(w: &mut Words, g: &PlaneGeo) { + // The homography is defined up to scale: normalise it (by a positive factor, keeping the + // sign tests) into f32's comfortable range. + let m = g.hinv.0.iter().flatten().fold(0.0f64, |a, v| a.max(v.abs())); + let k = if m > 0.0 { 1.0 / m } else { 1.0 }; + for r in 0..3 { + w.d([g.hinv.0[r][0] * k, g.hinv.0[r][1] * k, g.hinv.0[r][2] * k, 0.0]); + } + w.d([g.depth[0], g.depth[1], g.depth[2], 0.0]); + affine_rows(w, &g.world); + w.d([g.normal.x, g.normal.y, g.normal.z, 0.0]); + w.d([g.cam.eye.x, g.cam.eye.y, g.cam.eye.z, if g.cam.ortho { 1.0 } else { 0.0 }]); + let f = g.cam.forward(); + w.d([f.x, f.y, f.z, 0.0]); +} + +fn write_light(w: &mut Words, l: &LightState) { + w.u([light_kind(l.kind), l.falloff, l.casts_shadows as u32, 0]); + w.d([l.pos.x, l.pos.y, l.pos.z, 0.0]); + w.d([l.dir.x, l.dir.y, l.dir.z, 0.0]); + w.f([l.color[0], l.color[1], l.color[2], 0.0]); + w.d([l.cone_half, l.feather, l.radius, l.falloff_distance]); + w.d([l.shadow_darkness, l.shadow_diffusion, 0.0, 0.0]); +} + +const PLANE_BYTES: usize = 12 * 16; +const GEO_BYTES: usize = 10 * 16; +const LIGHT_BYTES: usize = 6 * 16; + +/// Composite a prepared Classic 3D run over `canvas`. `None` when it cannot run here (the +/// caller draws it on the CPU). +pub(crate) fn draw_run(e: &mut Enc, run: &Run3d, canvas: &GpuImage) -> Option { + let (w, h) = (canvas.width, canvas.height); + let boxes: Vec<[i64; 4]> = run.planes.iter().filter(|p| p.draw).map(|p| p.bbox).collect(); + if boxes.is_empty() { + return Some(canvas.clone()); + } + if max_overlap(&boxes) > MAX_FRAGS + || run.planes.iter().any(|p| p.winv.0[3] != [0.0, 0.0, 0.0, 1.0] || p.geos.iter().any(|g| g.world.0[3] != [0.0, 0.0, 0.0, 1.0])) + { + return None; + } + // Plane textures: the (cached) layer upload, depth-of-field blurred when asked. + let mut texs: Vec<(GpuImage, [f64; 2])> = Vec::with_capacity(run.planes.len()); + for p in &run.planes { + let mut img = e.g.upload_buf(&p.buf)?; + let mut off = p.buf.offset; + if let Some((sigma, pad)) = p.dof { + let (pw, ph) = (img.width + 2 * pad, img.height + 2 * pad); + if !e.g.fits(pw, ph) { + return None; + } + let padded = e.image(pw, ph); + e.copy_into(&img, &padded, pad, pad); + img = crate::effects::gaussian_blur(e, &padded, sigma, sigma, false); + off = [off[0] + pad as f64, off[1] + pad as f64]; + } + texs.push((img, off)); + } + let sizes: Vec<(u32, u32)> = texs.iter().map(|(t, _)| (t.width, t.height)).collect(); + let (aw, ah, pos) = pack(&sizes, e.g.max_dim)?; + if (aw as u64) * (ah as u64) * 16 > 2 << 30 { + return None; + } + let atlas = e.image(aw, ah); + for ((t, _), &(x, y)) in texs.iter().zip(&pos) { + e.copy_into(t, &atlas, x, y); + } + // Track mattes (shared by the planes of one layer). + let mut mattes: Vec = vec![]; + let mut matte_ix: HashMap = HashMap::new(); + let n_px = w as usize * h as usize; + let mut planes = Words::default(); + let mut geos = Words::default(); + let mut n_geos = 0u32; + for ((p, (t, off)), &(ax, ay)) in run.planes.iter().zip(&texs).zip(&pos) { + let matte = match &p.matte { + Some(m) if m.len() == n_px => { + let key = std::sync::Arc::as_ptr(m) as usize; + let next = matte_ix.len() as u32 + 1; + let ix = *matte_ix.entry(key).or_insert(next); + if ix == next { + mattes.extend_from_slice(m); + } + ix + } + Some(_) => return None, + None => 0, + }; + planes.f([ax as f32, ay as f32, t.width as f32, t.height as f32]); + planes.d([off[0], off[1], p.buf.scale, if p.bicubic { 1.0 } else { 0.0 }]); + planes.d(p.bbox.map(|v| v as f64)); + planes.d(p.bounds); + affine_rows(&mut planes, &p.winv); + planes.f([p.mat.ambient, p.mat.diffuse, p.mat.specular, p.mat.exponent() as f32]); + planes.f([p.mat.metal, p.mat.light_transmission, p.opacity, 0.0]); + let flags = p.draw as u32 | (p.preserve as u32) << 1 | (p.mat.accepts_lights as u32) << 2; + planes.u([n_geos, p.geos.len() as u32, flags, mode_id(p.mode)]); + planes.u([p.mat.casts_shadows, p.mat.accepts_shadows, p.layer_id, p.order as u32]); + planes.u([matte, run.seed ^ p.layer_id, 0, 0]); + for g in &p.geos { + write_geo(&mut geos, g); + } + n_geos += p.geos.len() as u32; + } + if (mattes.len() * 4) as u64 > e.g.device.limits().max_storage_buffer_binding_size as u64 { + return None; + } + let mut lights = Words::default(); + for l in &run.lights { + write_light(&mut lights, l); + } + let mut casters = Words::default(); + for c in &run.casters { + casters.0.extend_from_slice(&(*c as u32).to_le_bytes()); + } + let pb = e.bytes(planes.at_least(PLANE_BYTES)); + let gb = e.bytes(geos.at_least(GEO_BYTES)); + let lb = e.bytes(lights.at_least(LIGHT_BYTES)); + let cb = e.bytes(casters.at_least(16)); + let mb = e.data(&mattes); + let mut p = Params::default(); + p.u[0] = [run.planes.len() as u32, run.lights.len() as u32, run.casters.len() as u32, 0]; + let out = e.image(w, h); + e.dispatch_ext("classic3d", &p, &atlas, Some(canvas), &out, Some(&mb), (w.div_ceil(16), h.div_ceil(16)), Some([&pb, &gb, &lb, &cb])); + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shelf_packing_fits_and_does_not_overlap() { + let sizes = [(100, 40), (30, 90), (60, 60), (100, 10), (5, 5)]; + let (w, h, pos) = pack(&sizes, 256).unwrap(); + for (i, (&(x, y), &(sw, sh))) in pos.iter().zip(&sizes).enumerate() { + assert!(x + sw <= w && y + sh <= h); + for (j, (&(x2, y2), &(sw2, sh2))) in pos.iter().zip(&sizes).enumerate() { + if i != j { + assert!(x + sw <= x2 || x2 + sw2 <= x || y + sh <= y2 || y2 + sh2 <= y, "{i} and {j} overlap"); + } + } + } + assert!(pack(&[(300, 10)], 256).is_none()); + assert_eq!(max_overlap(&[[0, 0, 10, 10], [5, 5, 15, 15], [20, 20, 30, 30]]), 2); + } +} diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index 46868104..d549149d 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -28,6 +28,10 @@ const ENTRIES: &[&str] = &[ "pointwise", ]; +/// Entry points that also bind group 1 (four read-only storage buffers; see +/// [`Enc::dispatch_ext`]). +const EXT_ENTRIES: &[&str] = &["classic3d"]; + /// Pixel format of every working texture: premultiplied RGBA, 32-bit float (32 bpc headroom; /// 8/16 bpc are emulated by clamping and quantising, as on the CPU). pub const FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba32Float; @@ -88,6 +92,7 @@ pub struct GpuContext { pub(crate) name: String, pub(crate) max_dim: u32, bgl: wgpu::BindGroupLayout, + bgl_ext: wgpu::BindGroupLayout, pipelines: HashMap<&'static str, wgpu::ComputePipeline>, display_bgl: wgpu::BindGroupLayout, display: wgpu::ComputePipeline, @@ -135,7 +140,7 @@ impl GpuContext { } } let name = format!("{} ({:?})", info.name, info.backend); - let src = [include_str!("shaders/common.wgsl"), include_str!("shaders/kernels.wgsl")].concat(); + let src = [include_str!("shaders/common.wgsl"), include_str!("shaders/kernels.wgsl"), include_str!("shaders/classic3d.wgsl")].concat(); let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("effectcraft kernels"), source: wgpu::ShaderSource::Wgsl(src.into()) }); let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { @@ -167,12 +172,29 @@ impl GpuContext { bind_group_layouts: &[Some(&bgl)], immediate_size: 0, }); + let storage = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: true }, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }; + let bgl_ext = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effectcraft kernels (ext)"), + entries: &[storage(0), storage(1), storage(2), storage(3)], + }); + let layout_ext = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effectcraft kernels (ext)"), + bind_group_layouts: &[Some(&bgl), Some(&bgl_ext)], + immediate_size: 0, + }); let pipelines = ENTRIES .iter() - .map(|e| { + .map(|e| (e, &layout)) + .chain(EXT_ENTRIES.iter().map(|e| (e, &layout_ext))) + .map(|(e, layout)| { let p = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { label: Some(e), - layout: Some(&layout), + layout: Some(layout), module: &module, entry_point: Some(e), compilation_options: Default::default(), @@ -221,6 +243,7 @@ impl GpuContext { name, max_dim, bgl, + bgl_ext, pipelines, display_bgl, display, @@ -249,7 +272,7 @@ impl GpuContext { let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { label: Some("effectcraft gpu"), required_limits, ..Default::default() })) .ok()?; - device.on_uncaptured_error(Arc::new(|e| log::error!("wgpu: {e}"))); + device.on_uncaptured_error(Arc::new(|e| eprintln!("wgpu: {e}"))); GpuContext::new(&adapter, device, queue).map_err(|e| log::info!("gpu: {e}")).ok() } @@ -374,6 +397,22 @@ impl<'g> Enc<'g> { out: &GpuImage, data: Option<&wgpu::Buffer>, groups: (u32, u32), + ) { + self.dispatch_ext(entry, p, src, aux, out, data, groups, None); + } + + /// [`Enc::dispatch`] for the kernels in `EXT_ENTRIES`, with their group 1 storage buffers. + #[allow(clippy::too_many_arguments)] + pub fn dispatch_ext( + &mut self, + entry: &str, + p: &Params, + src: &GpuImage, + aux: Option<&GpuImage>, + out: &GpuImage, + data: Option<&wgpu::Buffer>, + groups: (u32, u32), + ext: Option<[&wgpu::Buffer; 4]>, ) { let g = self.g; let Some(pipe) = g.pipelines.get(entry) else { @@ -395,11 +434,26 @@ impl<'g> Enc<'g> { wgpu::BindGroupEntry { binding: 4, resource: data.unwrap_or(&g.dummy_buf).as_entire_binding() }, ], }); + let bg_ext = ext.map(|b| { + g.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &g.bgl_ext, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: b[0].as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: b[1].as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: b[2].as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: b[3].as_entire_binding() }, + ], + }) + }); let enc = self.encoder(); { let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some(entry), timestamp_writes: None }); pass.set_pipeline(pipe); pass.set_bind_group(0, &bg, &[]); + if let Some(b) = &bg_ext { + pass.set_bind_group(1, b, &[]); + } pass.dispatch_workgroups(groups.0.max(1), groups.1.max(1), 1); } self.pending += 1; @@ -525,7 +579,12 @@ impl<'g> Enc<'g> { for x in v { bytes.extend_from_slice(&x.to_le_bytes()); } - while bytes.len() < 16 { + self.bytes(bytes) + } + + /// A read-only storage buffer holding `bytes` (padded to 16 bytes). + pub fn bytes(&self, mut bytes: Vec) -> wgpu::Buffer { + while bytes.len() < 16 || bytes.len() % 4 != 0 { bytes.push(0); } self.g.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("data"), contents: &bytes, usage: wgpu::BufferUsages::STORAGE }) diff --git a/crates/gpu/src/effects.rs b/crates/gpu/src/effects.rs index fdc317b3..b1cc308f 100644 --- a/crates/gpu/src/effects.rs +++ b/crates/gpu/src/effects.rs @@ -111,7 +111,7 @@ fn box_passes(e: &mut Enc, img: &GpuImage, rx: &[usize], ry: &[usize], repeat: b } /// raster::gaussian_blur (3 box passes per axis). -fn gaussian_blur(e: &mut Enc, img: &GpuImage, sx: f64, sy: f64, repeat: bool) -> GpuImage { +pub(crate) fn gaussian_blur(e: &mut Enc, img: &GpuImage, sx: f64, sy: f64, repeat: bool) -> GpuImage { let rx = if sx > 0.05 { box_radii(sx, 3) } else { vec![] }; let ry = if sy > 0.05 { box_radii(sy, 3) } else { vec![] }; box_passes(e, img, &rx, &ry, repeat) diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 43474e9d..7a2634b3 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -22,6 +22,7 @@ //! egui-wgpu to draw. mod adv3d; +mod classic3d; mod context; mod effects; mod ops; @@ -140,4 +141,6 @@ impl Accelerator for Gpu { #[cfg(test)] mod tests; #[cfg(test)] +mod tests_3d; +#[cfg(test)] mod tests_adv3d; diff --git a/crates/gpu/src/shaders/classic3d.wgsl b/crates/gpu/src/shaders/classic3d.wgsl new file mode 100644 index 00000000..4e8d2139 --- /dev/null +++ b/crates/gpu/src/shaders/classic3d.wgsl @@ -0,0 +1,460 @@ +// Classic 3D on the GPU: the per-pixel compositor of `render::three_d::compose` (draw_run). +// +// Every plane covering an output pixel is inverse-mapped through its homography (one per +// motion-blur sub-sample), sampled from the plane atlas (`src`), shaded by the comp's lights +// (Blinn-Phong, ray-cast shadows against the caster planes) and multiplied by its track matte; +// the pixel's fragments are insertion-sorted far → near (coplanar fragments in stack order) and +// blended over the canvas (`aux`) with each layer's mode and opacity. +// +// Bindings: group 0 as every kernel (P, src = atlas, aux = canvas, out, data = track mattes); +// group 1 = planes, geometries, lights, caster indices. +// P.u[0] = (planes, lights, casters, _) + +struct Plane { + // Atlas rectangle (x, y, w, h) in texels. + atlas: vec4, + // Buffer offset (x, y), scale, bicubic (0 / 1). + tex: vec4, + // Output pixel bounds [x0, y0, x1, y1). + bbox: vec4, + // Layer-space bounds of the buffer. + bounds: vec4, + // World → layer (affine rows). + winv0: vec4, + winv1: vec4, + winv2: vec4, + // Ambient, diffuse, specular, Blinn-Phong exponent. + mat0: vec4, + // Metal, light transmission, opacity. + mat1: vec4, + // First geometry, geometry count, flags (1 draw, 2 preserve transparency, 4 accepts + // lights), blend mode. + ints: vec4, + // Casts shadows, accepts shadows (0 off, 1 on, 2 only), layer id, stack order. + ints2: vec4, + // Track matte (index + 1; 0 = none), dissolve seed. + ints3: vec4, +} + +struct Geo { + // Output pixel → homogeneous layer coordinates (rows). + h0: vec4, + h1: vec4, + h2: vec4, + // Camera depth = d.x·u + d.y·v + d.z. + depth: vec4, + // Layer → world (affine rows). + w0: vec4, + w1: vec4, + w2: vec4, + // Unit world normal. + normal: vec4, + // Camera eye (xyz) and orthographic flag (w). + eye: vec4, + // Camera forward (xyz). + fwd: vec4, +} + +struct Light { + // Kind (0 ambient, 1 parallel, 2 point, 3 spot), falloff, casts shadows. + ints: vec4, + pos: vec4, + dir: vec4, + color: vec4, + // Cone half-angle, feather, radius, falloff distance. + cone: vec4, + // Shadow darkness, shadow diffusion. + shadow: vec4, +} + +@group(1) @binding(0) var planes: array; +@group(1) @binding(1) var geos: array; +@group(1) @binding(2) var lights: array; +@group(1) @binding(3) var casters: array; + +const NEAR3: f32 = 1.0; +const MAX_FRAGS: u32 = 32u; + +fn atlas_get(i: u32, x: i32, y: i32) -> vec4 { + let r = planes[i].atlas; + if (x < 0 || y < 0 || x >= i32(r.z) || y >= i32(r.w)) { + return vec4(0.0); + } + return textureLoad(src, vec2(i32(r.x) + x, i32(r.y) + y), 0); +} + +// Image::sample_bilinear / sample_bicubic on a plane's buffer at layer coordinates (u, v). +fn texel(i: u32, u: f32, v: f32) -> vec4 { + let t = planes[i].tex; + let x = u * t.z + t.x; + let y = v * t.z + t.y; + let fx = x - 0.5; + let fy = y - 0.5; + let x0 = floor(fx); + let y0 = floor(fy); + let xi = i32(x0); + let yi = i32(y0); + if (t.w > 0.5) { + let wx = cubic_w(fx - x0); + let wy = cubic_w(fy - y0); + var o = vec4(0.0); + for (var j = 0; j < 4; j++) { + for (var k = 0; k < 4; k++) { + o += atlas_get(i, xi - 1 + k, yi - 1 + j) * (wx[k] * wy[j]); + } + } + return max(o, vec4(0.0)); + } + let tx = fx - x0; + let ty = fy - y0; + let a = atlas_get(i, xi, yi); + let b = atlas_get(i, xi + 1, yi); + let c = atlas_get(i, xi, yi + 1); + let d = atlas_get(i, xi + 1, yi + 1); + let top = a + (b - a) * tx; + let bot = c + (d - c) * tx; + return top + (bot - top) * ty; +} + +fn affine(r0: vec4, r1: vec4, r2: vec4, p: vec3) -> vec3 { + return vec3(dot(r0.xyz, p) + r0.w, dot(r1.xyz, p) + r1.w, dot(r2.xyz, p) + r2.w); +} + +fn linear3(r0: vec4, r1: vec4, r2: vec4, v: vec3) -> vec3 { + return vec3(dot(r0.xyz, v), dot(r1.xyz, v), dot(r2.xyz, v)); +} + +// compose::unproject: (u, v, depth, ok). +fn unproject(g: u32, sx: f32, sy: f32) -> vec4 { + let gm = geos[g]; + let ortho = gm.eye.w > 0.5; + let qz = gm.h2.x * sx + gm.h2.y * sy + gm.h2.z; + if ((!ortho && qz <= 0.0) || qz == 0.0) { + return vec4(0.0); + } + let u = (gm.h0.x * sx + gm.h0.y * sy + gm.h0.z) / qz; + let v = (gm.h1.x * sx + gm.h1.y * sy + gm.h1.z) / qz; + let z = gm.depth.x * u + gm.depth.y * v + gm.depth.z; + if (!ortho && z < NEAR3) { + return vec4(0.0); + } + return vec4(u, v, z, 1.0); +} + +// LightState::to_light: unit vector towards the light (xyz) and the distance (w; -1 = ∞). +fn to_light(l: u32, p: vec3) -> vec4 { + let li = lights[l]; + if (li.ints.x == 1u) { + return vec4(-li.dir.xyz, -1.0); + } + let d = li.pos.xyz - p; + let len = length(d); + if (len < 1e-9) { + return vec4(0.0, 0.0, -1.0, 0.0); + } + return vec4(d / len, len); +} + +fn smoothstep01(t: f32) -> f32 { + return 1.0 - t * t * (3.0 - 2.0 * t); +} + +// LightState::attenuation. +fn attenuation(l: u32, p: vec3) -> f32 { + let li = lights[l]; + let tl = to_light(l, p); + let dist = tl.w; + let finite = dist >= 0.0; + let radius = li.cone.z; + let fd = li.cone.w; + var k = 1.0; + if (li.ints.y == 1u && finite) { + if (dist <= radius) { + k = 1.0; + } else if (fd <= 0.0) { + k = 0.0; + } else { + k = smoothstep01(clamp((dist - radius) / fd, 0.0, 1.0)); + } + } else if (li.ints.y == 2u && finite) { + if (dist <= radius || (radius <= 0.0 && dist <= 1.0)) { + k = 1.0; + } else { + let q = max(radius, 1.0) / dist; + k = q * q; + } + } + if (li.ints.x == 3u) { + let cos_a = clamp(dot(-tl.xyz, li.dir.xyz), -1.0, 1.0); + let a = acos(cos_a); + let outer = li.cone.x; + let inner = outer * (1.0 - li.cone.y); + if (a <= inner) { + } else if (a >= outer) { + k = 0.0; + } else { + k *= smoothstep01((a - inner) / max(outer - inner, 1e-9)); + } + } + return k; +} + +// compose::occlusion: RGB transmittance of light `l` reaching `p` past caster `c`. +fn occlusion(c: u32, p: vec3, l: u32) -> vec3 { + let pl = planes[c]; + let li = lights[l]; + let a = affine(pl.winv0, pl.winv1, pl.winv2, p); + var hit: vec3; + var d_pc: f32; + var d_cl: f32; + if (li.ints.x == 1u) { + // Towards the light, 1e6 away (in direction form: exact in f32). + let dl = linear3(pl.winv0, pl.winv1, pl.winv2, -li.dir.xyz); + if (dl.z == 0.0) { + return vec3(1.0); + } + let s = -a.z / dl.z; + if (s <= 1.0 || s >= 1.0e6) { + return vec3(1.0); + } + hit = a + dl * s; + d_pc = length(li.dir.xyz) * s; + d_cl = 1000.0; + } else { + let b = affine(pl.winv0, pl.winv1, pl.winv2, li.pos.xyz); + if (a.z == b.z || (a.z < 0.0) == (b.z < 0.0)) { + return vec3(1.0); + } + let t = a.z / (a.z - b.z); + if (t <= 1e-6 || t >= 1.0) { + return vec3(1.0); + } + hit = a + (b - a) * t; + let seg = length(li.pos.xyz - p); + d_pc = seg * t; + d_cl = max(seg * (1.0 - t), 1.0); + } + let r = li.shadow.y * min(d_pc / d_cl, 4.0); + let bd = pl.bounds; + if (hit.x < bd.x - r || hit.y < bd.y - r || hit.x > bd.z + r || hit.y > bd.w + r) { + return vec3(1.0); + } + var px = vec4(0.0); + if (r * pl.tex.z < 0.5) { + px = texel(c, hit.x, hit.y); + } else { + for (var j = -1; j <= 1; j++) { + for (var i = -1; i <= 1; i++) { + px += texel(c, hit.x + f32(i) * r * 0.66, hit.y + f32(j) * r * 0.66) / 9.0; + } + } + } + let alpha = clamp(px.w * pl.mat1.z, 0.0, 1.0); + if (alpha <= 0.0) { + return vec3(1.0); + } + let tr = pl.mat1.y; + let dark = li.shadow.x; + var o = vec3(1.0); + for (var k = 0; k < 3; k++) { + var st = 0.0; + if (px.w > 0.0) { + st = clamp(px[k] / px.w, 0.0, 1.0); + } + let through = (1.0 - alpha) + alpha * tr * st; + o[k] = 1.0 - dark * (1.0 - through); + } + return o; +} + +fn shadow_of(it: u32, l: u32, p: vec3) -> vec3 { + var tr = vec3(1.0); + let n = P.u[0].z; + let lid = planes[it].ints2.z; + for (var k = 0u; k < n; k++) { + let c = casters[k]; + if (c == it || planes[c].ints2.z == lid) { + continue; + } + tr *= occlusion(c, p, l); + } + return tr; +} + +// compose::shade_texel (light::shade). +fn shade_texel(it: u32, g: u32, u: f32, v: f32, t: vec4) -> vec4 { + let pl = planes[it]; + let n_lights = P.u[0].y; + let accepts_shadows = pl.ints2.y; + if (n_lights == 0u || (pl.ints.z & 4u) == 0u) { + if (accepts_shadows == 2u) { + return vec4(0.0); + } + return t; + } + let gm = geos[g]; + let a = t.w; + let rgb = t.xyz / a; + let p = affine(gm.w0, gm.w1, gm.w2, vec3(u, v, 0.0)); + var to_viewer: vec3; + if (gm.eye.w > 0.5) { + to_viewer = -gm.fwd.xyz; + } else { + to_viewer = normalize(gm.eye.xyz - p); + } + var n = gm.normal.xyz; + if (dot(n, to_viewer) < 0.0) { + n = -n; + } + var amb = vec3(0.0); + var diff = vec3(0.0); + var spec = vec3(0.0); + var shadow_amt = 0.0; + let ex = pl.mat0.w; + let metal = pl.mat1.x; + let transmission = pl.mat1.y; + let specular = pl.mat0.z; + for (var i = 0u; i < n_lights; i++) { + let li = lights[i]; + if (li.ints.x == 0u) { + amb += li.color.xyz; + continue; + } + let k = attenuation(i, p); + if (k <= 0.0) { + continue; + } + let lv = to_light(i, p).xyz; + let ndl = dot(n, lv); + let front = ndl > 0.0; + var lambert = ndl; + if (!front) { + lambert = -ndl * transmission; + } + if (lambert <= 0.0) { + continue; + } + var sh = vec3(1.0); + if (li.ints.z != 0u && accepts_shadows != 0u) { + sh = shadow_of(it, i, p); + } + shadow_amt = max(shadow_amt, 1.0 - (sh.x + sh.y + sh.z) / 3.0); + diff += li.color.xyz * (lambert * k) * sh; + if (front && specular > 0.0) { + let h = normalize(lv + to_viewer); + let s = powz(max(dot(n, h), 0.0), ex) * k; + let tint = vec3(1.0) + (rgb - vec3(1.0)) * metal; + spec += li.color.xyz * s * sh * tint; + } + } + let lit = rgb * (pl.mat0.x * amb + pl.mat0.y * diff) + specular * spec; + if (accepts_shadows == 2u) { + return vec4(0.0, 0.0, 0.0, a * shadow_amt); + } + return vec4(lit * a, a); +} + +// compose::before: a draws before b (farther; coplanar → lower stack order first). +fn before(za: f32, oa: u32, zb: f32, ob: u32) -> bool { + let tol = 1e-5 * max(max(abs(za), abs(zb)), 1.0); + if (abs(za - zb) <= tol) { + return oa < ob; + } + return za > zb; +} + +@compute @workgroup_size(16, 16) +fn classic3d(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let x = i32(gid.x); + let y = i32(gid.y); + if (x >= dims.x || y >= dims.y) { + return; + } + let pix = vec2(x, y); + var d = textureLoad(aux, pix, 0); + let fx = f32(x); + let fy = f32(y); + let sx = fx + 0.5; + let sy = fy + 0.5; + var fz: array; + var fo: array; + var fi: array; + var fp: array, MAX_FRAGS>; + var nf = 0u; + let n_planes = P.u[0].x; + for (var i = 0u; i < n_planes; i++) { + let pl = planes[i]; + if ((pl.ints.z & 1u) == 0u) { + continue; + } + if (fx < pl.bbox.x || fx >= pl.bbox.z || fy < pl.bbox.y || fy >= pl.bbox.w) { + continue; + } + // compose::fragment: average of the motion-blur sub-samples. + let g0 = pl.ints.x; + let ng = pl.ints.y; + var acc = vec4(0.0); + var depth = 0.0; + var hit = false; + for (var k = 0u; k < ng; k++) { + let q = unproject(g0 + k, sx, sy); + if (q.w == 0.0) { + continue; + } + let t = texel(i, q.x, q.y); + if (t.w <= 1e-6) { + continue; + } + if (!hit) { + depth = q.z; + hit = true; + } + acc += shade_texel(i, g0 + k, q.x, q.y, t); + } + if (!hit) { + continue; + } + if (ng > 1u) { + acc *= 1.0 / f32(ng); + } + let m = pl.ints3.x; + if (m != 0u) { + let f = data[(m - 1u) * u32(dims.x) * u32(dims.y) + u32(y) * u32(dims.x) + u32(x)]; + acc *= f; + } + if (!(acc.w > 0.0 || acc.x != 0.0 || acc.y != 0.0 || acc.z != 0.0) || nf >= MAX_FRAGS) { + continue; + } + // Insertion: far → near, coplanar in stack order (stable, as the CPU's insertion sort). + let order = pl.ints2.w; + var j = nf; + while (j > 0u && before(depth, order, fz[j - 1u], fo[j - 1u])) { + fz[j] = fz[j - 1u]; + fo[j] = fo[j - 1u]; + fi[j] = fi[j - 1u]; + fp[j] = fp[j - 1u]; + j--; + } + fz[j] = depth; + fo[j] = order; + fi[j] = i; + fp[j] = acc; + nf++; + } + for (var k = 0u; k < nf; k++) { + let pl = planes[fi[k]]; + var op = pl.mat1.z; + if ((pl.ints.z & 2u) != 0u) { + op *= d.w; + } + let s = fp[k] * op; + let mode = pl.ints.w; + var n = 0.5; + if (mode == 1u || mode == 2u) { + n = hash_noise(u32(x), u32(y), pl.ints3.y); + } + d = blend_pixel(mode, d, s, n); + } + textureStore(out, pix, d); +} diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index c6fd84f0..137fdec3 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -26,7 +26,7 @@ use effectcraft_time::{FrameRate, Tick}; use crate::Gpu; -fn gpu() -> Option<&'static Gpu> { +pub(crate) fn gpu() -> Option<&'static Gpu> { static G: OnceLock> = OnceLock::new(); G.get_or_init(|| { let g = Gpu::headless(); @@ -40,9 +40,9 @@ fn gpu() -> Option<&'static Gpu> { /// Procedural footage: smooth colour gradients, fine noise and an alpha with soft, opaque and /// fully transparent regions (different per item). -struct Pattern; +pub(crate) struct Pattern; -fn pattern(seed: u32, w: u32, h: u32) -> Image { +pub(crate) fn pattern(seed: u32, w: u32, h: u32) -> Image { let mut img = Image::new(w, h); let k = seed as f32 * 0.37; for y in 0..h { @@ -66,14 +66,14 @@ impl FootageSource for Pattern { } } -struct Scene { - p: Project, - cid: ItemId, - comp: Comp, +pub(crate) struct Scene { + pub(crate) p: Project, + pub(crate) cid: ItemId, + pub(crate) comp: Comp, } impl Scene { - fn new(depth: BitDepth) -> Scene { + pub(crate) fn new(depth: BitDepth) -> Scene { let mut p = Project::default(); p.settings.bit_depth = depth; let comp = Comp::new(97, 61, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); @@ -81,7 +81,7 @@ impl Scene { Scene { p, cid, comp } } - fn footage(&mut self, w: u32, h: u32) -> Layer { + pub(crate) fn footage(&mut self, w: u32, h: u32) -> Layer { let f = Footage { path: "pattern.png".into(), kind: FootageKind::Still, @@ -106,19 +106,19 @@ impl Scene { build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) } - fn solid(&mut self, color: [f32; 3], w: u32, h: u32) -> Layer { + pub(crate) fn solid(&mut self, color: [f32; 3], w: u32, h: u32) -> Layer { let sid = self.p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: 1.0 })); build::layer(&mut self.p, &self.comp, "Solid", LayerSource::Solid { item: sid }, (w, h), None) } /// Add on top of the stack. - fn push(&mut self, l: Layer) -> effectcraft_project::LayerId { + pub(crate) fn push(&mut self, l: Layer) -> effectcraft_project::LayerId { let id = l.id; self.p.comp_mut(self.cid).unwrap().layers.insert(0, l); id } - fn effect(&mut self, l: &mut Layer, id: &str, vals: &[(&str, Value)]) { + pub(crate) fn effect(&mut self, l: &mut Layer, id: &str, vals: &[(&str, Value)]) { let spec = effectcraft_effects::find(id).unwrap(); let mut next = self.p.next_id; let size = effectcraft_render::source_size(&self.p, l); @@ -131,27 +131,27 @@ impl Scene { } } -fn set(l: &mut Layer, path: &str, v: Value) { +pub(crate) fn set(l: &mut Layer, path: &str, v: Value) { l.props.prop_mut(path).unwrap_or_else(|| panic!("no {path}")).value = v; } -fn v3(x: f64, y: f64) -> Value { +pub(crate) fn v3(x: f64, y: f64) -> Value { Value::Vec3([x, y, 0.0]) } /// Differences of a GPU render against the CPU render. #[derive(Debug)] -struct Diff { - max: f32, +pub(crate) struct Diff { + pub(crate) max: f32, /// Pixels over the tolerance. - over: usize, - total: usize, - worst: (u32, u32, [f32; 4], [f32; 4]), + pub(crate) over: usize, + pub(crate) total: usize, + pub(crate) worst: (u32, u32, [f32; 4], [f32; 4]), /// 8/16 bpc render. - quantized: bool, + pub(crate) quantized: bool, } -fn diff(cpu: &Image, gpu: &Image, tol: f32) -> Diff { +pub(crate) fn diff(cpu: &Image, gpu: &Image, tol: f32) -> Diff { assert_eq!((cpu.width, cpu.height), (gpu.width, gpu.height), "size"); let mut d = Diff { max: 0.0, over: 0, total: cpu.data.len(), worst: (0, 0, [0.0; 4], [0.0; 4]), quantized: false }; for (i, (a, b)) in cpu.data.iter().zip(&gpu.data).enumerate() { @@ -169,7 +169,7 @@ fn diff(cpu: &Image, gpu: &Image, tol: f32) -> Diff { d } -fn tolerance(depth: BitDepth) -> f32 { +pub(crate) fn tolerance(depth: BitDepth) -> f32 { match depth { BitDepth::Bpc8 => 1.0 / 255.0 + 1e-6, _ => 1e-3, @@ -177,7 +177,7 @@ fn tolerance(depth: BitDepth) -> f32 { } /// Render on the CPU and the GPU; `Some(diff)` (None without an adapter). -fn compare_at(s: &Scene, opts: RenderOpts, t: Tick) -> Option { +pub(crate) fn compare_at(s: &Scene, opts: RenderOpts, t: Tick) -> Option { let g = gpu()?; let cpu = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Cpu, ..opts }).comp_frame_cpu(s.cid, t); let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..opts }); @@ -188,12 +188,12 @@ fn compare_at(s: &Scene, opts: RenderOpts, t: Tick) -> Option { Some(d) } -fn opts() -> RenderOpts { +pub(crate) fn opts() -> RenderOpts { RenderOpts { motion_blur: true, ..Default::default() } } /// Assert agreement; `allow` = fraction of pixels allowed over the tolerance. -fn check(label: &str, d: Option, allow: f64) { +pub(crate) fn check(label: &str, d: Option, allow: f64) { let Some(d) = d else { return }; let frac = d.over as f64 / d.total as f64; // Quantised depths: rounding-boundary flips (see the module docs). @@ -417,11 +417,11 @@ fn effect_case(id: &str, vals: &[(&str, Value)]) { } } -fn n(v: f64) -> Value { +pub(crate) fn n(v: f64) -> Value { Value::Scalar(v) } -fn c(r: f64, g: f64, b: f64) -> Value { +pub(crate) fn c(r: f64, g: f64, b: f64) -> Value { Value::Color([r, g, b, 1.0]) } diff --git a/crates/gpu/src/tests_3d.rs b/crates/gpu/src/tests_3d.rs new file mode 100644 index 00000000..f10368d2 --- /dev/null +++ b/crates/gpu/src/tests_3d.rs @@ -0,0 +1,220 @@ +//! Classic 3D on the GPU vs the CPU compositor: intersecting planes, blend modes, lights, +//! ray-cast shadows, depth of field, motion blur, track mattes, collapsed precomps and +//! adjustment layers splitting a run. Same tolerances as `tests.rs`; pixels on the exact +//! intersection line of two planes may sort differently (f32 vs f64 depths), so intersecting +//! scenes allow 0.5 % of pixels over. + +use effectcraft_color::{BlendMode, Label}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::build; +use effectcraft_project::{BitDepth, Comp, ItemKind, Layer, LayerSource, LightKind, MatteKind, TrackMatte}; +use effectcraft_render::{Backend, RenderOpts, Renderer}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::tests::{Pattern, Scene, check, compare_at, gpu, n, opts, set}; + +const W: u32 = 160; +const H: u32 = 100; + +fn scene(depth: BitDepth) -> Scene { + let mut s = Scene::new(depth); + let comp = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + *s.p.comp_mut(s.cid).unwrap() = comp.clone(); + s.comp = comp; + s +} + +fn three_d(mut l: Layer, pos: [f64; 3], rot_y: f64) -> Layer { + l.switches.three_d = true; + set(&mut l, "transform/position", Value::Vec3(pos)); + set(&mut l, "transform/rotationY", Value::Scalar(rot_y)); + l +} + +fn camera(s: &mut Scene, pos: [f64; 3], dof: Option<(f64, f64)>) { + let mut cam = build::layer(&mut s.p, &s.comp, "Camera", LayerSource::Camera, (W, H), None); + set(&mut cam, "cameraOptions/zoom", Value::Scalar(W as f64 * 50.0 / 36.0)); + set(&mut cam, "transform/position", Value::Vec3(pos)); + set(&mut cam, "transform/poi", Value::Vec3([W as f64 / 2.0, H as f64 / 2.0, 0.0])); + if let Some((focus, aperture)) = dof { + set(&mut cam, "cameraOptions/dof", Value::Bool(true)); + set(&mut cam, "cameraOptions/focusDistance", Value::Scalar(focus)); + set(&mut cam, "cameraOptions/aperture", Value::Scalar(aperture)); + } + s.push(cam); +} + +fn light(s: &mut Scene, kind: LightKind, pos: [f64; 3], vals: &[(&str, Value)]) { + let mut l = build::layer(&mut s.p, &s.comp, "Light", LayerSource::Light { kind }, (W, H), None); + if kind != LightKind::Ambient { + set(&mut l, "transform/position", Value::Vec3(pos)); + } + if l.props.prop_mut("transform/poi").is_some() { + set(&mut l, "transform/poi", Value::Vec3([W as f64 / 2.0, H as f64 / 2.0, 0.0])); + } + for (k, v) in vals { + set(&mut l, &format!("lightOptions/{k}"), v.clone()); + } + s.push(l); +} + +/// Background, a far plane, and two planes crossing each other (one in `mode`). +fn crossing(depth: BitDepth, mode: BlendMode) -> Scene { + let mut s = scene(depth); + let bg = s.solid([0.1, 0.12, 0.2], W, H); + s.push(bg); + let far = three_d(s.footage(150, 90), [80.0, 50.0, 120.0], 0.0); + s.push(far); + let a = three_d(s.footage(90, 70), [80.0, 50.0, 0.0], 35.0); + s.push(a); + let mut b = three_d(s.solid([0.9, 0.5, 0.2], 90, 70), [80.0, 52.0, 0.0], -35.0); + b.blend_mode = mode; + set(&mut b, "transform/opacity", Value::Scalar(80.0)); + s.push(b); + s +} + +#[test] +fn intersecting_planes_and_blend_modes() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + for mode in [BlendMode::Normal, BlendMode::Screen, BlendMode::Multiply, BlendMode::Add, BlendMode::Dissolve] { + let mut s = crossing(depth, mode); + check(&format!("crossing {mode:?} {depth:?} (default camera)"), compare_at(&s, opts(), Tick::ZERO), 0.005); + camera(&mut s, [30.0, 20.0, -260.0], None); + check(&format!("crossing {mode:?} {depth:?} (camera)"), compare_at(&s, opts(), Tick::ZERO), 0.005); + } + } +} + +#[test] +fn lights_specular_transmission_and_shadows() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = crossing(depth, BlendMode::Normal); + // Every layer casts shadows; the crossing pair transmits light. + for l in &mut s.p.comp_mut(s.cid).unwrap().layers { + if l.switches.three_d { + set(l, "materialOptions/castsShadows", Value::Enum(1)); + set(l, "materialOptions/specularIntensity", Value::Scalar(60.0)); + set(l, "materialOptions/specularShininess", Value::Scalar(30.0)); + set(l, "materialOptions/lightTransmission", Value::Scalar(40.0)); + } + } + camera(&mut s, [-20.0, -10.0, -250.0], None); + light(&mut s, LightKind::Ambient, [0.0, 0.0, 0.0], &[("intensity", n(25.0))]); + light( + &mut s, + LightKind::Spot, + [20.0, -60.0, -150.0], + &[("castsShadows", Value::Bool(true)), ("shadowDarkness", n(80.0)), ("coneAngle", n(70.0)), ("coneFeather", n(40.0)), ("intensity", n(120.0))], + ); + light( + &mut s, + LightKind::Point, + [150.0, 20.0, -80.0], + &[ + ("castsShadows", Value::Bool(true)), + ("shadowDiffusion", n(6.0)), + ("falloff", Value::Enum(1)), + ("radius", n(120.0)), + ("falloffDistance", n(200.0)), + ], + ); + light(&mut s, LightKind::Parallel, [-100.0, -100.0, -100.0], &[("castsShadows", Value::Bool(true)), ("color", Value::Color([0.5, 0.7, 1.0, 1.0]))]); + check(&format!("lights + shadows {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + // Accepts Shadows: Only, Accepts Lights off, Casts Shadows: Only. + for l in &mut s.p.comp_mut(s.cid).unwrap().layers { + match l.name.as_str() { + "Pattern" => set(l, "materialOptions/acceptsShadows", Value::Enum(2)), + "Solid" if l.switches.three_d => set(l, "materialOptions/acceptsLights", Value::Bool(false)), + _ => {} + } + } + check(&format!("shadow-only materials {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + } +} + +#[test] +fn depth_of_field_and_motion_blur() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = crossing(depth, BlendMode::Normal); + camera(&mut s, [80.0, 50.0, -220.0], Some((220.0, 30.0))); + check(&format!("dof {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + // Animated plane with motion blur (sub-sampled geometry, equal weights). + let mut s = scene(depth); + let mut l = three_d(s.footage(60, 40), [40.0, 50.0, 0.0], 20.0); + l.switches.motion_blur = true; + l.props.prop_mut("transform/position").unwrap().keys = + vec![Keyframe::new(Tick::ZERO, Value::Vec3([40.0, 50.0, 0.0])), Keyframe::new(Tick::from_seconds_f64(1.0), Value::Vec3([140.0, 40.0, -60.0]))]; + s.push(l); + s.p.comp_mut(s.cid).unwrap().enable_motion_blur = true; + light(&mut s, LightKind::Point, [80.0, 0.0, -100.0], &[]); + check(&format!("3d motion blur {depth:?}"), compare_at(&s, opts(), Tick::from_seconds_f64(0.5)), 0.005); + } +} + +#[test] +fn mattes_preserve_transparency_precomps_and_adjustment_splits() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = crossing(depth, BlendMode::Normal); + // A 2D luma matte for the top plane, preserve transparency on the far plane. + let m = s.solid([0.7, 0.7, 0.7], 70, 50); + let mid = s.push(m); + let layers = &mut s.p.comp_mut(s.cid).unwrap().layers; + layers[1].track_matte = Some(TrackMatte { layer: mid, kind: MatteKind::Luma }); + if let Some(far) = layers.iter_mut().find(|l| l.switches.three_d && l.name == "Pattern") { + far.preserve_transparency = true; + } + // Move the matte below the bottom so the 3D run stays one run. + let ml = layers.remove(0); + layers.push(ml); + layers.last_mut().unwrap().switches.video = false; + check(&format!("matte + preserve {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + + // An adjustment layer in the middle of the run splits it. + let mut s = crossing(depth, BlendMode::Normal); + let mut adj = s.solid([1.0, 1.0, 1.0], W, H); + adj.switches.adjustment = true; + adj.switches.three_d = true; + s.effect(&mut adj, "ec.channel.invert", &[]); + s.p.comp_mut(s.cid).unwrap().layers.insert(1, adj); + check(&format!("adjustment split {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + + // A collapsed 3D precomp: its layers join the parent's 3D space. + let mut s = crossing(depth, BlendMode::Normal); + let inner = Comp::new(80, 60, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let inner_id = s.p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into())); + let mut a = s.footage(50, 40); + a.switches.three_d = true; + set(&mut a, "transform/rotationX", Value::Scalar(40.0)); + let b = s.solid([0.2, 0.8, 0.4], 30, 20); + s.p.comp_mut(inner_id).unwrap().layers = vec![b, a]; + let mut pre = build::layer(&mut s.p, &s.comp, "Inner", LayerSource::Comp { item: inner_id }, (80, 60), None); + pre.switches.collapse = true; + pre.switches.three_d = true; + set(&mut pre, "transform/position", Value::Vec3([70.0, 45.0, -10.0])); + s.push(pre); + check(&format!("collapsed 3d precomp {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.005); + } +} + +#[test] +fn the_gpu_draws_classic_3d_runs_itself() { + let Some(g) = gpu() else { return }; + let s = crossing(BitDepth::Bpc32, BlendMode::Normal); + let r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..opts() }); + let ctx = r.eval_ctx(s.cid, Tick::ZERO).unwrap(); + let vis = r.visible_layers(&ctx); + let run: Vec<_> = vis.iter().copied().filter(|l| l.is_3d()).collect(); + let prep = r.prepare_3d_run(&ctx, &run, (W, H)).expect("a classic run is prepared for the GPU"); + assert_eq!(prep.planes.iter().filter(|p| p.draw).count(), 3); + let mut e = crate::context::Enc::new(g.context()); + let canvas = e.image(W, H); + let out = crate::classic3d::draw_run(&mut e, &prep, &canvas).expect("the kernel draws the run"); + let img = e.download(&out).unwrap(); + let mut cpu = effectcraft_raster::Image::new(W, H); + r.draw_3d_run(&ctx, &run, &mut cpu); + let covered = cpu.data.iter().filter(|p| p[3] > 0.5).count(); + assert!(covered > (W * H / 3) as usize, "the planes cover the frame ({covered} px)"); + let d = crate::tests::diff(&cpu, &img, 1e-3); + assert!(d.over * 200 <= d.total, "{d:?}"); +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index ea56df7a..a0b51f1f 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -69,7 +69,7 @@ fn draw_comp<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, canvas: &mut while j < visible.len() && visible[j].is_3d() { j += 1; } - on_cpu(e, canvas, |img| r.draw_3d_run(ctx, &visible[i..j], img))?; + draw_3d(e, r, ctx, &visible[i..j], canvas)?; i = j; } else { match r.collapsed(ctx, visible[i]) { @@ -86,6 +86,25 @@ fn draw_comp<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, canvas: &mut Some(()) } +/// A run of consecutive 3D layers: Classic 3D on the GPU (`classic3d`), with 3D adjustment +/// layers splitting the run as on the CPU; Advanced 3D, wireframes and runs the kernel cannot +/// hold draw on the CPU. +fn draw_3d<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, run: &[&'a Layer], canvas: &mut GpuImage) -> Option<()> { + if let Some(k) = run.iter().position(|l| l.switches.adjustment) { + draw_3d(e, r, ctx, &run[..k], canvas)?; + draw_layer(e, r, ctx, run[k], canvas)?; + return draw_3d(e, r, ctx, &run[k + 1..], canvas); + } + if run.is_empty() { + return Some(()); + } + match r.prepare_3d_run(ctx, run, (canvas.width, canvas.height)).and_then(|prep| crate::classic3d::draw_run(e, &prep, canvas)) { + Some(img) => *canvas = img, + None => on_cpu(e, canvas, |img| r.draw_3d_run(ctx, run, img))?, + } + Some(()) +} + fn draw_collapsed<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, layer: &Layer, item: ItemId, canvas: &mut GpuImage) -> Option<()> { let opacity = r.layer_opacity(ctx, layer); if opacity <= 0.0 { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index ff71fb57..2e4028d4 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -494,11 +494,8 @@ impl<'a> Renderer<'a> { /// Render a composition at comp time `t` (transparent background, comp size × scale). /// Top-level frames go to the GPU when [`Self::active_accel`] is set and handles them. pub fn comp_frame(&self, comp_id: ItemId, t: Tick) -> Image { - // Auto keeps comps with 3D layers on the CPU compositor: 3D runs are still composited on - // the CPU, and the GPU path's readback/upload around them makes such comps slower. - let auto_3d = self.opts.backend == Backend::Auto && self.project.comp(comp_id).is_some_and(|c| c.has_3d()); + // Classic 3D runs composite on the GPU too (M12.7), so Auto sends 3D comps there as well. if self.depth == 0 - && !auto_3d && let Some(a) = self.active_accel() && let Some(img) = a.comp_frame(self, comp_id, t) { @@ -1347,6 +1344,13 @@ impl<'a> Renderer<'a> { three_d::compose::draw_run(self, ctx, run, canvas); } + /// A run of consecutive 3D layers prepared for an accelerator (planes, lights, shadow + /// casters; see [`three_d::Run3d`]) at output size `out`. `None` = draw it with + /// [`Self::draw_3d_run`] (adjustment or wireframe layers in the run, Advanced 3D). + pub fn prepare_3d_run(&self, ctx: &EvalCtx<'a>, run: &[&Layer], out: (u32, u32)) -> Option { + three_d::compose::gpu_run(self, ctx, run, out) + } + /// Draw one 2D layer into `canvas` on the CPU (any kind: adjustment, wireframe, styled…). pub fn draw_layer_cpu(&self, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image) { self.draw_layer(ctx, layer, canvas, false); diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 02c7f496..55373714 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -390,7 +390,7 @@ impl crate::Accelerator for ProbeAccel { } #[test] -fn auto_backend_keeps_3d_comps_on_the_cpu() { +fn auto_backend_sends_2d_and_3d_comps_to_the_gpu() { let (mut p, cid, comp) = setup(); p.settings.gpu_acceleration = true; let l = solid(&mut p, &comp, [1.0, 0.0, 0.0], 50, 50); @@ -404,8 +404,11 @@ fn auto_backend_keeps_3d_comps_on_the_cpu() { }; // 2D comp: Auto uses the GPU compositor. assert!(asked(&p, crate::Backend::Auto)); - // A 3D layer: Auto stays on the CPU; an explicit Gpu request still goes to the GPU. + // A 3D layer: Classic 3D runs on the GPU too (M12.7), so Auto still asks it. p.comp_mut(cid).unwrap().layers[0].switches.three_d = true; - assert!(!asked(&p, crate::Backend::Auto)); + assert!(asked(&p, crate::Backend::Auto)); assert!(asked(&p, crate::Backend::Gpu)); + // Software Only: never. + p.settings.gpu_acceleration = false; + assert!(!asked(&p, crate::Backend::Auto)); } diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 2647fadd..37fe5911 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -29,16 +29,18 @@ use crate::{EvalCtx, Renderer}; /// One position of a plane (several with motion blur). #[derive(Clone, Copy, Debug)] -pub(crate) struct Geo { +pub struct Geo { /// Output pixel → homogeneous layer coordinates. - hinv: Mat3, + pub hinv: Mat3, /// Camera-space depth as a function of layer (u, v): `d0·u + d1·v + d2`. - depth: [f64; 3], - world: Mat4, - normal: Vec3, + pub depth: [f64; 3], + /// Layer → world. + pub world: Mat4, + /// Unit world normal. + pub normal: Vec3, /// Output pixel bounds [x0, y0, x1, y1) covered by the plane. - bbox: [i64; 4], - cam: CameraState, + pub bbox: [i64; 4], + pub cam: CameraState, } /// A prepared 3D layer. @@ -60,6 +62,9 @@ pub(crate) struct Item<'a> { winv: Mat4, bounds: [f64; 4], bbox: [i64; 4], + /// Depth of field left to an accelerator: Gaussian sigma and padding (buffer pixels) of the + /// blur `buf` still needs (`None` = `buf` is final). + dof: Option<(f64, u32)>, } impl Item<'_> { @@ -72,7 +77,7 @@ impl Item<'_> { } /// Build the plane geometry of a layer for a camera. -fn geo(cam: &CameraState, world: Mat4, buf: &Buf, comp: (f64, f64), scale: f64, out: (u32, u32)) -> Option { +fn geo(cam: &CameraState, world: Mat4, b: [f64; 4], comp: (f64, f64), scale: f64, out: (u32, u32)) -> Option { let proj = Mat4::scale(vec3(scale, scale, 1.0)) * cam.projection(comp.0, comp.1); let full = proj * world; let h = full.plane_to_mat3(); @@ -89,7 +94,6 @@ fn geo(cam: &CameraState, world: Mat4, buf: &Buf, comp: (f64, f64), scale: f64, return None; } let normal = normal.normalize(); - let b = buf_bounds(buf); let corners = [(b[0], b[1]), (b[2], b[1]), (b[2], b[3]), (b[0], b[3])]; let (mut x0, mut y0, mut x1, mut y1) = (f64::INFINITY, f64::INFINITY, f64::NEG_INFINITY, f64::NEG_INFINITY); let mut behind = 0; @@ -170,20 +174,25 @@ fn mb_samples(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> usize { } /// Prepare a layer (buffer, geometry, material). `None` when it shows nothing. -pub(crate) fn prepare<'a>(r: &Renderer, ctx: &EvalCtx<'a>, layer: &'a Layer, order: usize, out: (u32, u32), in_run: bool) -> Option> { +pub(crate) fn prepare<'a>(r: &Renderer, ctx: &EvalCtx<'a>, layer: &'a Layer, order: usize, out: (u32, u32), in_run: bool, defer_dof: bool) -> Option> { let buf = r.blend_layer_buf(ctx, layer)?; - prepare_with(r, ctx, layer, order, out, in_run, buf, Mat4::IDENTITY) + prepare_with(r, ctx, layer, order, out, in_run, buf, Mat4::IDENTITY, defer_dof) } /// Prepare a layer as planes: one per character for per-character 3D text, else one. pub(crate) fn prepare_all<'a>(r: &Renderer, ctx: &EvalCtx<'a>, layer: &'a Layer, order: usize, out: (u32, u32), in_run: bool) -> Vec> { + prepare_all_opt(r, ctx, layer, order, out, in_run, false) +} + +/// [`prepare_all`], optionally leaving the depth-of-field blur to an accelerator. +fn prepare_all_opt<'a>(r: &Renderer, ctx: &EvalCtx<'a>, layer: &'a Layer, order: usize, out: (u32, u32), in_run: bool, defer_dof: bool) -> Vec> { if crate::text::per_char_3d(ctx, layer) { return crate::text::per_char_planes(ctx, layer, r.opts.scale) .into_iter() - .filter_map(|(buf, m)| prepare_with(r, ctx, layer, order, out, in_run, r.to_blend(Arc::new(buf), None), m)) + .filter_map(|(buf, m)| prepare_with(r, ctx, layer, order, out, in_run, r.to_blend(Arc::new(buf), None), m, defer_dof)) .collect(); } - prepare(r, ctx, layer, order, out, in_run).into_iter().collect() + prepare(r, ctx, layer, order, out, in_run, defer_dof).into_iter().collect() } /// Prepare a plane showing `buf`, whose space maps into the layer through `local`. @@ -197,6 +206,7 @@ fn prepare_with<'a>( in_run: bool, mut buf: Arc, local: Mat4, + defer_dof: bool, ) -> Option> { if buf.img.is_empty() { return None; @@ -207,6 +217,8 @@ fn prepare_with<'a>( let outer = r.collapse3d.map_or(Mat4::IDENTITY, |c| c.world); let world = outer * ctx.world_matrix(layer) * local; // Depth of field: blur the layer by its circle of confusion at its centre. + let mut bounds = buf_bounds(&buf); + let mut deferred = None; if let Some(dof) = cam.dof.filter(|_| !ctx.comp.draft_3d && !r.opts.draft) { let b = buf_bounds(&buf); let centre = world.apply(vec3((b[0] + b[2]) * 0.5, (b[1] + b[3]) * 0.5, 0.0)); @@ -218,18 +230,27 @@ fn prepare_with<'a>( if radius > 0.3 { let sigma = radius / 1.5; let pad = (sigma * 3.0).ceil() as u32 + 1; - let padded = buf.img.padded(pad); - // The (cached, shared) layer buffer stays untouched; the blurred copy is per frame. - buf = Arc::new(Buf { - img: effectcraft_raster::gaussian_blur(&padded, sigma, sigma, false), - offset: [buf.offset[0] + pad as f64, buf.offset[1] + pad as f64], - scale: buf.scale, - }); + if defer_dof { + let s = buf.scale.max(1e-9); + let (w, h) = ((buf.img.width + 2 * pad) as f64, (buf.img.height + 2 * pad) as f64); + let o = [buf.offset[0] + pad as f64, buf.offset[1] + pad as f64]; + bounds = [-o[0] / s, -o[1] / s, (w - o[0]) / s, (h - o[1]) / s]; + deferred = Some((sigma, pad)); + } else { + let padded = buf.img.padded(pad); + // The (cached, shared) layer buffer stays untouched; the blurred copy is per frame. + buf = Arc::new(Buf { + img: effectcraft_raster::gaussian_blur(&padded, sigma, sigma, false), + offset: [buf.offset[0] + pad as f64, buf.offset[1] + pad as f64], + scale: buf.scale, + }); + bounds = buf_bounds(&buf); + } } } let n = if in_run { mb_samples(r, ctx, layer) } else { 1 }; let geos: Vec = if n <= 1 { - geo(&cam, world, &buf, comp, s, out).into_iter().collect() + geo(&cam, world, bounds, comp, s, out).into_iter().collect() } else { let fd = ctx.comp.frame_duration().seconds(); let angle = ctx.comp.shutter_angle / 360.0; @@ -239,7 +260,7 @@ fn prepare_with<'a>( let f = phase + angle * i as f64 / (n - 1) as f64; let sub = ctx.at(ctx.time + Tick::from_seconds_f64(f * fd)); let c = camera_for(r, &sub); - geo(&c, outer * sub.world_matrix(layer) * local, &buf, comp, s, out) + geo(&c, outer * sub.world_matrix(layer) * local, bounds, comp, s, out) }) .collect() }; @@ -251,7 +272,6 @@ fn prepare_with<'a>( bbox = [bbox[0].min(g.bbox[0]), bbox[1].min(g.bbox[1]), bbox[2].max(g.bbox[2]), bbox[3].max(g.bbox[3])]; } let mat = Material::of(ctx, layer); - let bounds = buf_bounds(&buf); Some(Item { layer, order, @@ -266,6 +286,7 @@ fn prepare_with<'a>( winv: world.inverse().unwrap_or(Mat4::IDENTITY), bounds, bbox, + dof: deferred, }) } @@ -330,6 +351,23 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut return; } let out = (canvas.width, canvas.height); + let g = gather(r, ctx, run, out, false); + let casters: Vec<&Item> = g.items.iter().chain(g.extra.iter()).filter(|i| i.mat.casts_shadows != 0 && !i.geos.is_empty()).collect(); + composite(canvas, &g.items, &g.lights, &casters, run[0].id.0 as u32); + for l in run.iter().filter(|l| l.switches.quality == effectcraft_project::Quality::Wireframe) { + r.draw_layer(ctx, l, canvas, false); + } +} + +/// The prepared planes of a run: drawn items (stack order), shadow casters outside the run, and +/// the lights. +struct Gathered<'a> { + items: Vec>, + extra: Vec>, + lights: Vec, +} + +fn gather<'a>(r: &Renderer<'a>, ctx: &EvalCtx<'a>, run: &[&'a Layer], out: (u32, u32), defer_dof: bool) -> Gathered<'a> { // A collapsed precomp's 3D layers are lit by the outer comp's lights. let lights = scene_lights(&r.collapse3d.map_or(*ctx, |c| c.parent)); let mut items: Vec = run @@ -340,9 +378,9 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut // Stack order with room for a collapsed precomp's layers in between. let i = i * 1024; if let Some(item) = r.collapsed(ctx, l) { - return collapsed_items(r, ctx, l, item, i, out); + return collapsed_items(r, ctx, l, item, i, out, defer_dof); } - let mut its = prepare_all(r, ctx, l, i, out, true); + let mut its = prepare_all_opt(r, ctx, l, i, out, true, defer_dof); if !its.is_empty() { let m = matte_for(r, ctx, l, out); for it in &mut its { @@ -362,15 +400,11 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut .par_iter() .filter(|l| l.switches.three_d && l.has_video() && l.is_active_at(ctx.time) && !in_run.contains(&l.id) && !l.switches.adjustment) .filter(|l| Material::of(ctx, l).casts_shadows != 0) - .flat_map_iter(|l| prepare_all(r, ctx, l, 0, out, false)) + .flat_map_iter(|l| prepare_all_opt(r, ctx, l, 0, out, false, defer_dof)) .collect(); } items.sort_by_key(|i| i.order); - let casters: Vec<&Item> = items.iter().chain(extra.iter()).filter(|i| i.mat.casts_shadows != 0 && !i.geos.is_empty()).collect(); - composite(canvas, &items, &lights, &casters, run[0].id.0 as u32); - for l in run.iter().filter(|l| l.switches.quality == effectcraft_project::Quality::Wireframe) { - r.draw_layer(ctx, l, canvas, false); - } + Gathered { items, extra, lights } } /// A collapsed 3D precomp layer inside a 3D run: its nested layers become planes in this comp's @@ -384,6 +418,7 @@ fn collapsed_items<'a>( item: effectcraft_project::ItemId, order: usize, out: (u32, u32), + defer_dof: bool, ) -> Vec> { let op = ctx.opacity(layer) as f32 * r.opacity_mul(); let Some((sub, nctx)) = r.collapse_into(ctx, layer, item, op) else { return vec![] }; @@ -401,7 +436,7 @@ fn collapsed_items<'a>( if nctx.opacity(l) <= 0.0 { continue; } - let mut its = prepare_all(&sub, &nctx, l, order + 1 + k.min(1022), out, true); + let mut its = prepare_all_opt(&sub, &nctx, l, order + 1 + k.min(1022), out, true, defer_dof); if !its.is_empty() { let m = matte_for(&sub, &nctx, l, out); for it in &mut its { @@ -590,6 +625,86 @@ fn occlusion(c: &Item, p: Vec3, l: &LightState) -> [f32; 3] { out } +// ------------------------------------------------------------------ accelerator export + +/// A Classic 3D run prepared for an accelerator ([`Renderer::prepare_3d_run`]): every plane +/// with its geometry, material and buffer, ready for the per-pixel depth sort, lighting and +/// ray-cast shadows of [`draw_run`] on another device. +#[derive(Clone, Debug)] +pub struct Run3d { + /// Drawn planes in stack order, then shadow casters from outside the run (`draw` false). + pub planes: Vec, + /// Indices (into `planes`) of the shadow casters, in the CPU's order. + pub casters: Vec, + pub lights: Vec, + /// Dissolve seed of the run. + pub seed: u32, +} + +/// One plane of a [`Run3d`]. +#[derive(Clone, Debug)] +pub struct Plane3d { + /// The layer buffer (blending space). When `dof` is set, the plane shows this buffer padded + /// by `dof.1` pixels on every side and Gaussian-blurred with sigma `dof.0` + /// (`raster::gaussian_blur`, no edge repeat). + pub buf: Arc, + pub dof: Option<(f64, u32)>, + pub bicubic: bool, + /// One per motion-blur sub-sample (equal weights). + pub geos: Vec, + pub mat: Material, + pub opacity: f32, + pub draw: bool, + /// Screen-space track matte factor (output size, row-major). + pub matte: Option>>, + pub preserve: bool, + /// World → layer, and the (blurred) buffer's layer-space bounds. + pub winv: Mat4, + pub bounds: [f64; 4], + /// Output pixel bounds of all geometries. + pub bbox: [i64; 4], + /// Stack order (coplanar fragments draw lower orders first). + pub order: usize, + pub layer_id: u32, + pub mode: BlendMode, +} + +/// Prepare a run of 3D layers for an accelerator. `None` when the run must be drawn on the CPU +/// (adjustment or wireframe layers, Advanced 3D). +pub(crate) fn gpu_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Option { + if run.is_empty() || run.iter().any(|l| l.switches.adjustment || l.switches.quality == effectcraft_project::Quality::Wireframe) { + return None; + } + if ctx.comp.renderer == effectcraft_project::Renderer::Advanced3D && r.collapse3d.is_none() { + return None; + } + let g = gather(r, ctx, run, out, true); + let planes: Vec = g + .items + .into_iter() + .chain(g.extra) + .map(|it| Plane3d { + layer_id: it.layer.id.0 as u32, + mode: it.layer.blend_mode, + buf: it.buf, + dof: it.dof, + bicubic: it.bicubic, + geos: it.geos, + mat: it.mat, + opacity: it.opacity, + draw: it.draw, + matte: it.matte, + preserve: it.preserve, + winv: it.winv, + bounds: it.bounds, + bbox: it.bbox, + order: it.order, + }) + .collect(); + let casters = planes.iter().enumerate().filter(|(_, p)| p.mat.casts_shadows != 0 && !p.geos.is_empty()).map(|(i, _)| i).collect(); + Some(Run3d { planes, casters, lights: g.lights, seed: run[0].id.0 as u32 }) +} + // ------------------------------------------------------------------ auxiliary render pass /// One surface seen through an output pixel (aux pass). diff --git a/crates/render/src/three_d/mod.rs b/crates/render/src/three_d/mod.rs index e97c67fe..4bd7dcf4 100644 --- a/crates/render/src/three_d/mod.rs +++ b/crates/render/src/three_d/mod.rs @@ -16,6 +16,7 @@ mod tests; mod tests_orient; pub use camera::{CameraState, Dof, Rig, View3D, ViewCam, Views3D, active_camera, default_camera, default_view_cam}; +pub use compose::{Geo as PlaneGeo, Plane3d, Run3d}; pub use light::{LightState, Material, lights_at}; use effectcraft_geom::Mat3; From b8a56eb4a934c4d945156a7296435e7f46a734ae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:21:14 -0400 Subject: [PATCH 03/36] M13.6: scopes, content-aware fill (PatchMatch/flow propagation/membrane), cut detection and auto-trace algorithms Co-Authored-By: Claude Opus 5.5 --- crates/path/src/lib.rs | 1 + crates/path/src/trace.rs | 417 +++++++++++++++ crates/raster/src/cuts.rs | 206 ++++++++ crates/raster/src/inpaint.rs | 958 +++++++++++++++++++++++++++++++++++ crates/raster/src/lib.rs | 3 + crates/raster/src/scopes.rs | 552 ++++++++++++++++++++ 6 files changed, 2137 insertions(+) create mode 100644 crates/path/src/trace.rs create mode 100644 crates/raster/src/cuts.rs create mode 100644 crates/raster/src/inpaint.rs create mode 100644 crates/raster/src/scopes.rs diff --git a/crates/path/src/lib.rs b/crates/path/src/lib.rs index b4216e88..169a7428 100644 --- a/crates/path/src/lib.rs +++ b/crates/path/src/lib.rs @@ -11,6 +11,7 @@ mod fit; pub mod interp; pub mod offset; pub mod ops; +pub mod trace; pub mod varstroke; use effectcraft_geom::Mat3; diff --git a/crates/path/src/trace.rs b/crates/path/src/trace.rs new file mode 100644 index 00000000..42973803 --- /dev/null +++ b/crates/path/src/trace.rs @@ -0,0 +1,417 @@ +//! Auto-trace: outlines of a scalar field (a layer's alpha, a colour channel or luminance) as +//! Bezier paths (Layer ▸ Auto-trace). +//! +//! 1. **Marching squares** over the samples at pixel centres, with linear interpolation of the +//! threshold crossing on each cell edge (saddles resolved by the cell's mean), on a field padded +//! with "outside" so every contour closes. +//! 2. **Douglas–Peucker** simplification of each closed loop to the Tolerance (pixels). +//! 3. **Bezier fitting**: each simplified span is fitted with one cubic (least squares with fixed +//! end tangents, Newton reparameterisation; [`crate::fit`]). The tangent at each vertex is the +//! chord direction at Corner Roundness 0 (straight segments, sharp corners) and the smooth +//! (Catmull–Rom) direction at 100 %, blended between. +//! +//! Loops smaller than Minimum Area are dropped. Each loop gets its nesting depth (0 = outermost), +//! so callers can add outer outlines and subtract holes. + +use std::collections::HashMap; + +use effectcraft_keyframe::ShapePath; +use kurbo::{Point, Vec2}; + +use crate::fit::{fit_single, unit}; + +/// Auto-trace options. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct TraceOpts { + /// Inside where the field is at least this (0…1). + pub threshold: f32, + /// Douglas–Peucker tolerance in pixels. + pub tolerance: f64, + /// Loops with a smaller area (square pixels) are dropped. + pub min_area: f64, + /// 0 = polygons, 1 = fully smooth. + pub corner_roundness: f64, +} + +impl Default for TraceOpts { + fn default() -> Self { + TraceOpts { threshold: 0.5, tolerance: 1.0, min_area: 10.0, corner_roundness: 0.5 } + } +} + +/// A traced outline. +#[derive(Clone, Debug, PartialEq)] +pub struct Traced { + pub path: ShapePath, + /// Number of other outlines that contain this one (even = filled, odd = hole). + pub depth: usize, + /// Signed area of the simplified polygon (pixels²). + pub area: f64, +} + +/// Trace `field` (`w × h`, row-major, sample (x, y) at pixel centre (x + 0.5, y + 0.5)). +pub fn trace(field: &[f32], w: usize, h: usize, o: &TraceOpts) -> Vec { + if w == 0 || h == 0 || field.len() < w * h { + return vec![]; + } + let loops = contours(field, w, h, o.threshold); + let mut polys: Vec<(Vec<[f64; 2]>, Vec<[f64; 2]>, f64)> = loops + .into_iter() + .filter_map(|raw| { + let a = signed_area(&raw); + if a.abs() < o.min_area.max(0.0) || raw.len() < 3 { + return None; + } + let simp = simplify_closed(&raw, o.tolerance.max(0.01)); + if simp.len() < 3 { + return None; + } + Some((raw, simp, a)) + }) + .collect(); + // Depth: how many other loops contain a point of this loop. + let depths: Vec = (0..polys.len()) + .map(|i| { + let p = polys[i].1[0]; + (0..polys.len()).filter(|&j| j != i && polys[j].2.abs() > polys[i].2.abs() && point_in_poly(p, &polys[j].1)).count() + }) + .collect(); + let mut out: Vec = polys + .drain(..) + .zip(depths) + .map(|((raw, simp, a), depth)| Traced { path: fit_loop(&raw, &simp, o.corner_roundness.clamp(0.0, 1.0)), depth, area: a }) + .collect(); + // Outer outlines first, then holes, islands…; larger first within a depth. + out.sort_by(|a, b| a.depth.cmp(&b.depth).then(b.area.abs().total_cmp(&a.area.abs()))); + out +} + +/// Marching squares: closed polylines (pixel coordinates). +fn contours(field: &[f32], w: usize, h: usize, thr: f32) -> Vec> { + // Padded grid: sample (i, j) of the padded grid is pixel (i - 1, j - 1), outside = -inf. + let (gw, gh) = (w + 2, h + 2); + let val = |i: usize, j: usize| -> f32 { if i == 0 || j == 0 || i > w || j > h { f32::NEG_INFINITY } else { field[(j - 1) * w + (i - 1)] } }; + let inside = |i: usize, j: usize| val(i, j) >= thr; + // Edge ids: horizontal edge (i, j)-(i+1, j) = 2 * (j * gw + i); vertical (i, j)-(i, j+1) = +1. + let pos = |i: usize, j: usize| [i as f64 - 0.5, j as f64 - 0.5]; + let cross = |a: (usize, usize), b: (usize, usize)| -> [f64; 2] { + let (va, vb) = (val(a.0, a.1), val(b.0, b.1)); + let t = if va.is_finite() && vb.is_finite() && (vb - va).abs() > 1e-12 { + ((thr - va) / (vb - va)).clamp(0.0, 1.0) as f64 + } else if va.is_finite() { + // b is padding: the crossing sits half a pixel out. + if va >= thr { 0.5 } else { 0.0 } + } else if vb >= thr { + 0.5 + } else { + 1.0 + }; + let (pa, pb) = (pos(a.0, a.1), pos(b.0, b.1)); + [pa[0] + (pb[0] - pa[0]) * t, pa[1] + (pb[1] - pa[1]) * t] + }; + let hid = |i: usize, j: usize| 2 * (j * gw + i); + let vid = |i: usize, j: usize| 2 * (j * gw + i) + 1; + // Directed segments (from edge, to edge) keeping the inside on the left. + let mut next: HashMap = HashMap::new(); + let mut point: HashMap = HashMap::new(); + for j in 0..gh - 1 { + for i in 0..gw - 1 { + // Corners: tl (i, j), tr (i+1, j), br (i+1, j+1), bl (i, j+1). + let (tl, tr, br, bl) = (inside(i, j), inside(i + 1, j), inside(i + 1, j + 1), inside(i, j + 1)); + let code = (tl as u8) << 3 | (tr as u8) << 2 | (br as u8) << 1 | bl as u8; + if code == 0 || code == 15 { + continue; + } + let top = hid(i, j); + let bottom = hid(i, j + 1); + let left = vid(i, j); + let right = vid(i + 1, j); + let mut seg = |a: usize, b: usize| { + next.insert(a, b); + }; + // Walk with the inside on the right in image coordinates (y down), which is + // counter-clockwise on screen for outer loops… orientation is fixed afterwards. + match code { + 1 => seg(left, bottom), + 2 => seg(bottom, right), + 3 => seg(left, right), + 4 => seg(right, top), + 5 => { + // tr + bl inside (saddle). + let centre = (val(i, j) + val(i + 1, j) + val(i + 1, j + 1) + val(i, j + 1)) / 4.0; + if centre >= thr { + seg(left, top); + seg(right, bottom); + } else { + seg(left, bottom); + seg(right, top); + } + } + 6 => seg(bottom, top), + 7 => seg(left, top), + 8 => seg(top, left), + 9 => seg(top, bottom), + 10 => { + // tl + br inside (saddle). + let centre = (val(i, j) + val(i + 1, j) + val(i + 1, j + 1) + val(i, j + 1)) / 4.0; + if centre >= thr { + seg(top, right); + seg(bottom, left); + } else { + seg(top, left); + seg(bottom, right); + } + } + 11 => seg(top, right), + 12 => seg(right, left), + 13 => seg(right, bottom), + 14 => seg(bottom, left), + _ => {} + } + for (e, a, b) in [(top, (i, j), (i + 1, j)), (bottom, (i, j + 1), (i + 1, j + 1)), (left, (i, j), (i, j + 1)), (right, (i + 1, j), (i + 1, j + 1))] + { + if inside(a.0, a.1) != inside(b.0, b.1) { + point.entry(e).or_insert_with(|| cross(a, b)); + } + } + } + } + let mut out = vec![]; + let mut keys: Vec = next.keys().copied().collect(); + keys.sort_unstable(); + for start in keys { + if !next.contains_key(&start) { + continue; + } + let mut poly = vec![]; + let mut e = start; + while let Some(n) = next.remove(&e) { + if let Some(p) = point.get(&e) { + poly.push(*p); + } + e = n; + if e == start { + break; + } + } + if poly.len() >= 3 { + out.push(poly); + } + } + out +} + +/// Shoelace area (positive = clockwise on screen with y down). +pub fn signed_area(p: &[[f64; 2]]) -> f64 { + let n = p.len(); + let mut a = 0.0; + for i in 0..n { + let (q, r) = (p[i], p[(i + 1) % n]); + a += q[0] * r[1] - r[0] * q[1]; + } + a * 0.5 +} + +fn point_in_poly(p: [f64; 2], poly: &[[f64; 2]]) -> bool { + let mut inside = false; + let n = poly.len(); + let mut j = n - 1; + for i in 0..n { + let (a, b) = (poly[i], poly[j]); + if (a[1] > p[1]) != (b[1] > p[1]) && p[0] < (b[0] - a[0]) * (p[1] - a[1]) / (b[1] - a[1]) + a[0] { + inside = !inside; + } + j = i; + } + inside +} + +fn seg_dist(p: [f64; 2], a: [f64; 2], b: [f64; 2]) -> f64 { + let (dx, dy) = (b[0] - a[0], b[1] - a[1]); + let l2 = dx * dx + dy * dy; + let t = if l2 > 0.0 { (((p[0] - a[0]) * dx + (p[1] - a[1]) * dy) / l2).clamp(0.0, 1.0) } else { 0.0 }; + ((p[0] - a[0] - t * dx).powi(2) + (p[1] - a[1] - t * dy).powi(2)).sqrt() +} + +/// Douglas–Peucker on an open polyline: kept indices. +fn dp(p: &[[f64; 2]], lo: usize, hi: usize, tol: f64, keep: &mut Vec) { + if hi <= lo + 1 { + return; + } + let (mut best, mut bi) = (0.0, lo); + for (k, q) in p.iter().enumerate().take(hi).skip(lo + 1) { + let d = seg_dist(*q, p[lo], p[hi]); + if d > best { + best = d; + bi = k; + } + } + if best > tol { + dp(p, lo, bi, tol, keep); + keep.push(bi); + dp(p, bi, hi, tol, keep); + } +} + +/// Douglas–Peucker on a closed loop: indices into `p` of the kept vertices, in order. +fn simplify_idx(p: &[[f64; 2]], tol: f64) -> Vec { + let n = p.len(); + // Split at the two mutually farthest-ish points (first point and the point farthest from it). + let far = (0..n).max_by(|&a, &b| { + let d = |i: usize| (p[i][0] - p[0][0]).powi(2) + (p[i][1] - p[0][1]).powi(2); + d(a).total_cmp(&d(b)) + }); + let far = far.unwrap_or(0); + let mut ext: Vec<[f64; 2]> = p.to_vec(); + ext.push(p[0]); + let mut keep = vec![0]; + dp(&ext, 0, far, tol, &mut keep); + keep.push(far); + dp(&ext, far, n, tol, &mut keep); + keep.sort_unstable(); + keep.dedup(); + keep.retain(|&i| i < n); + keep +} + +fn simplify_closed(p: &[[f64; 2]], tol: f64) -> Vec<[f64; 2]> { + simplify_idx(p, tol).into_iter().map(|i| p[i]).collect() +} + +/// Fit cubics through the simplified vertices of a loop (`raw` holds the full contour). +fn fit_loop(raw: &[[f64; 2]], simp: &[[f64; 2]], round: f64) -> ShapePath { + let n = simp.len(); + if round <= 1e-6 { + return ShapePath::polygon(simp, true); + } + // Raw index of each simplified vertex (they are raw points). + let idx: Vec = simp.iter().map(|s| raw.iter().position(|r| r == s).unwrap_or(0)).collect(); + let pt = |a: [f64; 2]| Point::new(a[0], a[1]); + let v = |a: [f64; 2], b: [f64; 2]| Vec2::new(b[0] - a[0], b[1] - a[1]); + // Per vertex: (direction leaving, direction arriving), blended between chord and smooth. + let dirs: Vec<(Vec2, Vec2)> = (0..n) + .map(|i| { + let (prev, cur, nxt) = (simp[(i + n - 1) % n], simp[i], simp[(i + 1) % n]); + let smooth = unit(v(prev, nxt)).unwrap_or(Vec2::new(1.0, 0.0)); + let out_chord = unit(v(cur, nxt)).unwrap_or(smooth); + let in_chord = unit(v(prev, cur)).unwrap_or(smooth); + let leave = unit(out_chord * (1.0 - round) + smooth * round).unwrap_or(out_chord); + let arrive = unit(in_chord * (1.0 - round) + smooth * round).unwrap_or(in_chord); + (leave, arrive) + }) + .collect(); + let mut path = ShapePath::polygon(simp, true); + for i in 0..n { + let j = (i + 1) % n; + let (a, b) = (idx[i], idx[j]); + let mut pts: Vec = + if b > a { raw[a..=b].iter().map(|q| pt(*q)).collect() } else { raw[a..].iter().chain(raw[..=b].iter()).map(|q| pt(*q)).collect() }; + if pts.len() < 2 { + pts = vec![pt(simp[i]), pt(simp[j])]; + } + let chord = pts[0].distance(pts[pts.len() - 1]); + let (t0, t1) = (dirs[i].0, dirs[j].1); + let (c, _, _) = if pts.len() >= 3 { fit_single(&pts, t0, t1) } else { (kurbo::CubicBez::new(pts[0], pts[0], pts[1], pts[1]), 0.0, 0) }; + // Keep handles pointing along the tangents and within the chord. + let max = chord * 0.5 * round.max(0.34); + let l0 = (c.p1 - c.p0).dot(t0).clamp(0.0, max); + let l1 = (c.p3 - c.p2).dot(t1).clamp(0.0, max); + let (l0, l1) = if pts.len() < 3 { (chord / 3.0 * round, chord / 3.0 * round) } else { (l0, l1) }; + path.out_tangents[i] = [t0.x * l0, t0.y * l0]; + path.in_tangents[j] = [-t1.x * l1, -t1.y * l1]; + } + path +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{FillRule, fill_coverage, to_kurbo}; + use effectcraft_geom::Mat3; + + fn iou(a: &[bool], b: &[f32]) -> f64 { + let (mut i, mut u) = (0.0, 0.0); + for (x, y) in a.iter().zip(b) { + let y = *y >= 0.5; + if *x && y { + i += 1.0; + } + if *x || y { + u += 1.0; + } + } + i / u + } + + /// Rasterise traced outlines with even-odd fill (as the Add / Subtract mask stack does). + fn raster(t: &[Traced], w: usize, h: usize) -> Vec { + let paths: Vec<_> = t.iter().map(|t| to_kurbo(&t.path)).collect(); + fill_coverage(&paths, &Mat3::IDENTITY, w as u32, h as u32, FillRule::EvenOdd).data + } + + #[test] + fn traces_a_disc_and_a_square_with_a_hole() { + let (w, h) = (120, 80); + let mut field = vec![0.0f32; w * h]; + let mut truth = vec![false; w * h]; + for y in 0..h { + for x in 0..w { + let (fx, fy) = (x as f64 + 0.5, y as f64 + 0.5); + let disc = (fx - 30.0).powi(2) + (fy - 40.0).powi(2) <= 20.0f64.powi(2); + let square = (60.0..110.0).contains(&fx) && (15.0..65.0).contains(&fy); + let hole = (75.0..95.0).contains(&fx) && (30.0..50.0).contains(&fy); + let on = disc || (square && !hole); + truth[y * w + x] = on; + field[y * w + x] = if on { 1.0 } else { 0.0 }; + } + } + for round in [0.0, 0.5, 1.0] { + let t = trace(&field, w, h, &TraceOpts { corner_roundness: round, ..Default::default() }); + assert_eq!(t.len(), 3, "disc, square, hole"); + assert_eq!(t.iter().filter(|t| t.depth == 1).count(), 1, "one hole"); + let r = raster(&t, w, h); + let score = iou(&truth, &r); + // Full roundness rounds the square's corners off as well. + let min = if round >= 1.0 { 0.88 } else { 0.96 }; + assert!(score > min, "IoU {score} at roundness {round}"); + } + // A disc stays a disc when smooth. + let disc: Vec = + (0..w * h).map(|i| if ((i % w) as f64 + 0.5 - 30.0).powi(2) + ((i / w) as f64 + 0.5 - 40.0).powi(2) <= 400.0 { 1.0 } else { 0.0 }).collect(); + let t = trace(&disc, w, h, &TraceOpts { corner_roundness: 1.0, tolerance: 1.5, ..Default::default() }); + let truth_disc: Vec = disc.iter().map(|v| *v > 0.5).collect(); + let score = iou(&truth_disc, &raster(&t, w, h)); + assert!(score > 0.97, "disc IoU {score}"); + assert!(t[0].path.vertices.len() < 40, "simplified to {}", t[0].path.vertices.len()); + // The square traces to 4 corners at zero roundness. + let t = trace(&field, w, h, &TraceOpts { corner_roundness: 0.0, tolerance: 1.0, ..Default::default() }); + let sq = t.iter().find(|t| t.depth == 0 && t.path.vertices.iter().all(|v| v[0] > 55.0)).expect("square"); + assert!(sq.path.vertices.len() <= 8, "{:?}", sq.path.vertices); + } + + #[test] + fn min_area_and_threshold() { + let (w, h) = (40, 40); + let mut field = vec![0.0f32; w * h]; + // A 2×2 speck and a 20×20 block at 0.6. + for y in 2..4 { + for x in 2..4 { + field[y * w + x] = 1.0; + } + } + for y in 10..30 { + for x in 10..30 { + field[y * w + x] = 0.6; + } + } + assert_eq!(trace(&field, w, h, &TraceOpts { min_area: 10.0, ..Default::default() }).len(), 1); + assert_eq!(trace(&field, w, h, &TraceOpts { min_area: 1.0, ..Default::default() }).len(), 2); + // Above 0.6 only the speck remains (and it is too small). + assert!(trace(&field, w, h, &TraceOpts { threshold: 0.7, ..Default::default() }).is_empty()); + // Edge-touching shapes close along the border. + let full = vec![1.0f32; 100]; + let t = trace(&full, 10, 10, &TraceOpts::default()); + assert_eq!(t.len(), 1); + assert!((t[0].area.abs() - 100.0).abs() < 2.0, "{}", t[0].area); + } +} diff --git a/crates/raster/src/cuts.rs b/crates/raster/src/cuts.rs new file mode 100644 index 00000000..28ca93f3 --- /dev/null +++ b/crates/raster/src/cuts.rs @@ -0,0 +1,206 @@ +//! Scene edit (cut) detection (Layer ▸ Scene Edit Detection). +//! +//! Each frame is reduced to a [`Signature`]: a joint RGB colour histogram (4 × 4 × 4 bins, +//! normalised) and a small luma thumbnail. Two consecutive frames are compared with +//! +//! - the **histogram distance** (half the L1 distance, 0…1): large when the colour content +//! changes, insensitive to motion; +//! - the **motion-compensated difference**: block optical flow ([`crate::flow::block_flow`]) +//! from the earlier thumbnail to the later one, the earlier one warped along it, and the mean +//! absolute luma error that remains. Inside a shot motion explains the change and the error +//! stays small; across a cut nothing matches. +//! +//! The dissimilarity is the geometric mean of the two (both must agree, which rejects flashes and +//! fast pans). A frame starts a new shot when its dissimilarity exceeds the absolute threshold +//! and is a clear local peak: several times the median of its neighbours (the adaptive threshold +//! of the classic shot-boundary literature), with a minimum shot length. + +use rayon::prelude::*; + +use crate::Image; +use crate::flow::block_flow; + +const BINS: usize = 4; +const THUMB: u32 = 96; + +/// The per-frame data compared by [`dissimilarity`]. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Signature { + pub hist: Vec, + pub thumb: Image, +} + +/// Compute a frame's signature. +pub fn signature(img: &Image) -> Signature { + let mut hist = vec![0f32; BINS * BINS * BINS]; + let mut n = 0.0; + for p in &img.data { + let a = p[3]; + let c = if a > 1e-6 { [p[0] / a * a.min(1.0), p[1] / a * a.min(1.0), p[2] / a * a.min(1.0)] } else { [0.0; 3] }; + let b = |v: f32| ((v.clamp(0.0, 1.0) * BINS as f32) as usize).min(BINS - 1); + hist[b(c[0]) * BINS * BINS + b(c[1]) * BINS + b(c[2])] += 1.0; + n += 1.0; + } + if n > 0.0 { + for h in &mut hist { + *h /= n; + } + } + let long = img.width.max(img.height).max(1); + let thumb = if long > THUMB { + let s = THUMB as f64 / long as f64; + crate::resample(img, ((img.width as f64 * s).round() as u32).max(8), ((img.height as f64 * s).round() as u32).max(8)) + } else { + img.clone() + }; + Signature { hist, thumb } +} + +fn luma(p: [f32; 4]) -> f32 { + 0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2] +} + +/// Dissimilarity of consecutive frames, 0 (same) … 1 (unrelated). +pub fn dissimilarity(a: &Signature, b: &Signature) -> f32 { + let hist: f32 = a.hist.iter().zip(&b.hist).map(|(x, y)| (x - y).abs()).sum::() * 0.5; + let (ta, tb) = (&a.thumb, &b.thumb); + let motion = if ta.width == tb.width && ta.height == tb.height && !ta.is_empty() { + let f = block_flow(tb, ta, 8, 4); + // For each pixel of B, where it came from in A. + let mut err = 0.0f32; + let mut n = 0.0f32; + for y in 0..tb.height { + for x in 0..tb.width { + let v = f.at(x as f64 + 0.5, y as f64 + 0.5); + let pa = ta.sample_bilinear_clamped(x as f64 + 0.5 + v[0] as f64, y as f64 + 0.5 + v[1] as f64); + err += (luma(pa) - luma(tb.data[tb.idx(x, y)])).abs(); + n += 1.0; + } + } + // Mean absolute error of 0.25 (a quarter of full scale) counts as completely different. + (err / n.max(1.0) * 4.0).min(1.0) + } else { + 1.0 + }; + (hist * motion).sqrt() +} + +/// Cut detection options. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CutOpts { + /// Absolute dissimilarity a cut must exceed (0…1). + pub threshold: f32, + /// A cut must be this many times the median of its neighbours. + pub peak_ratio: f32, + /// Neighbours on each side used for the local median. + pub window: usize, + /// Minimum frames between cuts. + pub min_shot: usize, +} + +impl Default for CutOpts { + fn default() -> Self { + CutOpts { threshold: 0.25, peak_ratio: 3.0, window: 4, min_shot: 2 } + } +} + +/// Dissimilarities of every frame to the previous one (`scores[0]` = 0). +pub fn scores(frames: &[Image]) -> Vec { + let sigs: Vec = frames.par_iter().map(signature).collect(); + let mut out = vec![0.0]; + out.extend((1..sigs.len()).into_par_iter().map(|i| dissimilarity(&sigs[i - 1], &sigs[i])).collect::>()); + out +} + +/// Frames that start a new shot (indices into `scores`; never 0). +pub fn detect(scores: &[f32], o: &CutOpts) -> Vec { + let mut cuts: Vec = vec![]; + for i in 1..scores.len() { + let s = scores[i]; + if s < o.threshold { + continue; + } + let lo = i.saturating_sub(o.window).max(1); + let hi = (i + o.window).min(scores.len() - 1); + let mut nb: Vec = (lo..=hi).filter(|&j| j != i).map(|j| scores[j]).collect(); + nb.sort_by(f32::total_cmp); + let med = nb.get(nb.len() / 2).copied().unwrap_or(0.0); + if s < o.peak_ratio * med.max(0.02) { + continue; + } + match cuts.last() { + Some(&last) if i - last < o.min_shot.max(1) => { + // Keep the stronger of two close candidates. + if s > scores[last] { + cuts.pop(); + cuts.push(i); + } + } + _ => cuts.push(i), + } + } + cuts +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash_noise; + + /// A textured "shot": `kind` picks the palette and texture, panned by `dx` pixels. + fn frame(kind: u32, dx: f32, w: u32, h: u32) -> Image { + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let fx = x as f32 + dx; + let v = match kind { + 0 => 0.5 + 0.4 * (fx * 0.15).sin() * (y as f32 * 0.11).cos(), + 1 => { + if ((fx / 12.0).floor() as i32 + (y / 12) as i32) % 2 == 0 { + 0.85 + } else { + 0.15 + } + } + _ => hash_noise((fx.floor() as i64 / 4) as u32, y / 4, 7), + }; + let c = match kind { + 0 => [v, v * 0.6, 0.2], + 1 => [0.1, v * 0.8, v], + _ => [v * 0.9, v * 0.9, v * 0.4 + 0.5], + }; + img.set(x, y, [c[0], c[1], c[2], 1.0]); + } + } + img + } + + #[test] + fn finds_synthetic_cuts_exactly() { + // Three shots with camera pans (12, 9 and 11 frames) and a brightness flicker inside the + // second. + let mut frames = vec![]; + for i in 0..12 { + frames.push(frame(0, i as f32 * 2.0, 128, 72)); + } + for i in 0..9 { + let mut f = frame(1, i as f32 * 3.0, 128, 72); + if i == 4 { + f.map_straight(|c| [c[0] * 1.08, c[1] * 1.08, c[2] * 1.08]); + } + frames.push(f); + } + for i in 0..11 { + frames.push(frame(2, -(i as f32) * 4.0, 128, 72)); + } + let s = scores(&frames); + let cuts = detect(&s, &CutOpts::default()); + assert_eq!(cuts, vec![12, 21], "scores {s:?}"); + } + + #[test] + fn a_static_shot_has_no_cuts() { + let frames: Vec = (0..8).map(|i| frame(2, i as f32, 64, 48)).collect(); + assert!(detect(&scores(&frames), &CutOpts::default()).is_empty()); + } +} diff --git a/crates/raster/src/inpaint.rs b/crates/raster/src/inpaint.rs new file mode 100644 index 00000000..a4ae40ce --- /dev/null +++ b/crates/raster/src/inpaint.rs @@ -0,0 +1,958 @@ +//! Content-Aware Fill: filling the transparent area of a layer across a range of frames. +//! +//! Written from the published methods: +//! +//! - **Membrane (harmonic) fill**: the hole takes the smoothest values that match its boundary +//! (Laplace's equation with Dirichlet boundary values), solved coarse to fine with +//! known-weighted pyramids and Jacobi relaxation. Edge Blend uses it directly; it also +//! initialises the other methods and completes flow fields. +//! - **PatchMatch** (Barnes, Shechtman, Finkelstein, Goldman 2009) nearest-neighbour fields +//! inside the **space-time completion** EM loop of Wexler, Shechtman and Irani (2007), applied +//! per frame: every patch overlapping the hole is matched to a fully known patch (random +//! initialisation, propagation, random search); hole pixels become the weighted vote of the +//! overlapping matches; repeated over a multi-scale pyramid, coarse to fine. +//! - **Flow-guided propagation** (the propagation stage of flow-guided video inpainting, e.g. Xu +//! et al. 2019 / Huang et al. 2016): optical flow between neighbouring frames is computed on +//! membrane-filled frames, *completed* inside the hole from the surrounding motion, and each +//! missing pixel follows the flow forwards and backwards until it lands on a frame where that +//! point of the scene is visible; the two candidates are blended by temporal distance. +//! **Lighting Correction** adds the membrane interpolation of the mismatch along the hole's +//! border (Poisson-style seamless blending), scaled by the chosen strength. +//! +//! Methods (After Effects' Content-Aware Fill panel): +//! - **Object**: propagation, then PatchMatch synthesis for what no frame shows; each frame's +//! synthesis starts from the previous frame's result warped along the flow, so the fill stays +//! temporally stable. +//! - **Surface**: propagation, then one synthesised fill carried through the range by the flow +//! (flat, rigid surfaces). +//! - **Edge Blend**: the membrane fill of each frame from its own border (fast; textureless +//! surfaces). +//! +//! Reference frames are complete images of given frames (for example a clean plate): they count as +//! frames where every pixel is visible, so propagation pulls their content into the hole. + +use rayon::prelude::*; + +use crate::Image; +use crate::flow::{Flow, block_flow}; + +/// Fill method. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FillMethod { + #[default] + Object, + Surface, + EdgeBlend, +} + +impl FillMethod { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "object" => Some(FillMethod::Object), + "surface" => Some(FillMethod::Surface), + "edgeblend" | "edge" => Some(FillMethod::EdgeBlend), + _ => None, + } + } + pub fn name(self) -> &'static str { + match self { + FillMethod::Object => "object", + FillMethod::Surface => "surface", + FillMethod::EdgeBlend => "edgeBlend", + } + } +} + +/// Fill options. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FillOpts { + pub method: FillMethod, + /// Lighting Correction strength 0 (None) … 1 (Strong). + pub lighting: f32, + /// Patch size (odd). + pub patch: usize, + /// Seed of the PatchMatch random search. + pub seed: u32, +} + +impl Default for FillOpts { + fn default() -> Self { + FillOpts { method: FillMethod::Object, lighting: 0.0, patch: 7, seed: 1 } + } +} + +// ------------------------------------------------------------------ helpers + +#[inline] +fn rnd(seed: u32, a: u32, b: u32) -> u32 { + let mut h = seed.wrapping_mul(0x9E37_79B9) ^ a.wrapping_mul(0x85EB_CA6B) ^ b.wrapping_mul(0xC2B2_AE35); + h ^= h >> 16; + h = h.wrapping_mul(0x7FEB_352D); + h ^= h >> 15; + h = h.wrapping_mul(0x846C_A68B); + h ^ (h >> 16) +} + +/// Grow a hole mask by `r` pixels (square structuring element, separable). +pub fn dilate(hole: &[bool], w: usize, h: usize, r: usize) -> Vec { + if r == 0 { + return hole.to_vec(); + } + let mut tmp = vec![false; w * h]; + for y in 0..h { + for x in 0..w { + let lo = x.saturating_sub(r); + let hi = (x + r).min(w - 1); + tmp[y * w + x] = (lo..=hi).any(|k| hole[y * w + k]); + } + } + let mut out = vec![false; w * h]; + for y in 0..h { + for x in 0..w { + let lo = y.saturating_sub(r); + let hi = (y + r).min(h - 1); + out[y * w + x] = (lo..=hi).any(|k| tmp[k * w + x]); + } + } + out +} + +/// Peak signal-to-noise ratio (dB) of the RGB of `a` against `b` over `region` (all pixels when +/// `None`), for values in 0…1. +pub fn psnr(a: &Image, b: &Image, region: Option<&[bool]>) -> f64 { + let (mut se, mut n) = (0.0f64, 0.0f64); + for i in 0..a.data.len().min(b.data.len()) { + if region.is_some_and(|r| !r[i]) { + continue; + } + for c in 0..3 { + let d = (a.data[i][c] - b.data[i][c]) as f64; + se += d * d; + } + n += 3.0; + } + if se <= 0.0 || n == 0.0 { + return 99.0; + } + 10.0 * (1.0 / (se / n)).log10() +} + +// ------------------------------------------------------------------ membrane + +/// Harmonic fill of `data` (`w × h`, 4 channels) where `known` is false. +fn membrane_core(data: &mut [[f32; 4]], known: &[bool], w: usize, h: usize) { + if !known.iter().any(|k| !*k) { + return; + } + if !known.iter().any(|k| *k) { + return; + } + // Coarse level: known-weighted 2×2 averages. + if w > 8 && h > 8 { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut cd = vec![[0.0f32; 4]; cw * ch]; + let mut ck = vec![false; cw * ch]; + for y in 0..ch { + for x in 0..cw { + let mut s = [0.0f32; 4]; + let mut n = 0.0; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (fx, fy) = (2 * x + dx, 2 * y + dy); + if fx < w && fy < h && known[fy * w + fx] { + let p = data[fy * w + fx]; + for c in 0..4 { + s[c] += p[c]; + } + n += 1.0; + } + } + if n > 0.0 { + cd[y * cw + x] = s.map(|v| v / n); + ck[y * cw + x] = true; + } + } + } + membrane_core(&mut cd, &ck, cw, ch); + for y in 0..h { + for x in 0..w { + if !known[y * w + x] { + data[y * w + x] = cd[(y / 2).min(ch - 1) * cw + (x / 2).min(cw - 1)]; + } + } + } + } + // Jacobi relaxation of the unknown pixels (in place, Gauss–Seidel order). + let iters = if w * h <= 4096 { 200 } else { 40 }; + let unknown: Vec = (0..w * h).filter(|&i| !known[i]).collect(); + for _ in 0..iters { + let mut delta = 0.0f32; + for &i in &unknown { + let (x, y) = (i % w, i / w); + let mut s = [0.0f32; 4]; + let mut n = 0.0; + for (nx, ny) in [(x.wrapping_sub(1), y), (x + 1, y), (x, y.wrapping_sub(1)), (x, y + 1)] { + if nx < w && ny < h { + let p = data[ny * w + nx]; + for c in 0..4 { + s[c] += p[c]; + } + n += 1.0; + } + } + let v = s.map(|v| v / n); + delta = delta.max((v[0] - data[i][0]).abs()); + data[i] = v; + } + if delta < 1e-5 { + break; + } + } +} + +/// Membrane fill: `img` with the `hole` pixels replaced by the harmonic interpolation of their +/// border. +pub fn membrane_fill(img: &Image, hole: &[bool]) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + let mut out = img.clone(); + let known: Vec = hole.iter().map(|h| !*h).collect(); + membrane_core(&mut out.data, &known, w, h); + out +} + +// ------------------------------------------------------------------ PatchMatch + +struct Level { + w: usize, + h: usize, + img: Vec<[f32; 4]>, + hole: Vec, +} + +fn downsample(l: &Level) -> Level { + let (w, h) = (l.w.div_ceil(2), l.h.div_ceil(2)); + let mut img = vec![[0.0f32; 4]; w * h]; + let mut hole = vec![false; w * h]; + for y in 0..h { + for x in 0..w { + let mut s = [0.0f32; 4]; + let mut n = 0.0; + let mut any_hole = false; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let (fx, fy) = ((2 * x + dx).min(l.w - 1), (2 * y + dy).min(l.h - 1)); + let i = fy * l.w + fx; + if l.hole[i] { + any_hole = true; + } else { + for c in 0..4 { + s[c] += l.img[i][c]; + } + n += 1.0; + } + } + hole[y * w + x] = any_hole; + if n > 0.0 { + img[y * w + x] = s.map(|v| v / n); + } + } + } + Level { w, h, img, hole } +} + +/// Summed-area table of hole pixels, for "is this patch fully known" queries. +struct HoleSat { + w: usize, + s: Vec, +} + +impl HoleSat { + fn new(hole: &[bool], w: usize, h: usize) -> HoleSat { + let sw = w + 1; + let mut s = vec![0u32; sw * (h + 1)]; + for y in 0..h { + for x in 0..w { + s[(y + 1) * sw + x + 1] = hole[y * w + x] as u32 + s[y * sw + x + 1] + s[(y + 1) * sw + x] - s[y * sw + x]; + } + } + HoleSat { w: sw, s } + } + /// Hole pixels in [x0, x1) × [y0, y1). + fn count(&self, x0: usize, y0: usize, x1: usize, y1: usize) -> u32 { + self.s[y1 * self.w + x1] + self.s[y0 * self.w + x0] - self.s[y0 * self.w + x1] - self.s[y1 * self.w + x0] + } +} + +struct Pm<'a> { + w: usize, + h: usize, + r: usize, + img: &'a [[f32; 4]], + sat: HoleSat, +} + +impl Pm<'_> { + fn valid_source(&self, x: i64, y: i64) -> bool { + let r = self.r as i64; + x >= r + && y >= r + && x + r < self.w as i64 + && y + r < self.h as i64 + && self.sat.count((x - r) as usize, (y - r) as usize, (x + r + 1) as usize, (y + r + 1) as usize) == 0 + } + /// SSD between the patch at target (tx, ty) (clipped to the image) and source (sx, sy). + fn dist(&self, tx: i64, ty: i64, sx: i64, sy: i64, cap: f32) -> f32 { + let r = self.r as i64; + let mut d = 0.0f32; + let mut n = 0.0f32; + for dy in -r..=r { + let (ty2, sy2) = (ty + dy, sy + dy); + if ty2 < 0 || ty2 >= self.h as i64 { + continue; + } + for dx in -r..=r { + let tx2 = tx + dx; + if tx2 < 0 || tx2 >= self.w as i64 { + continue; + } + let a = self.img[ty2 as usize * self.w + tx2 as usize]; + let b = self.img[sy2 as usize * self.w + (sx + dx) as usize]; + for c in 0..4 { + let e = a[c] - b[c]; + d += e * e; + } + n += 1.0; + } + if d > cap * n.max(1.0) { + return f32::INFINITY; + } + } + d / n.max(1.0) + } +} + +/// One Wexler/PatchMatch level: refine `img` (hole pixels) with `em` EM rounds, starting from the +/// nearest-neighbour field `nnf` (per pixel, source centre; `None` = random). +fn pm_level(l: &mut Level, nnf: &mut Vec>, em: usize, pm_iters: usize, patch: usize, seed: u32) { + let (w, h) = (l.w, l.h); + let r = (patch / 2).min((w.min(h).saturating_sub(1)) / 2).max(1); + // Target centres: pixels whose patch touches the hole. + let near = dilate(&l.hole, w, h, r); + let targets: Vec = (0..w * h).filter(|&i| near[i]).collect(); + if targets.is_empty() { + return; + } + let sat = HoleSat::new(&l.hole, w, h); + // Sources: centres of fully known patches. + let sources: Vec<[i32; 2]> = { + let pm = Pm { w, h, r, img: &l.img, sat: HoleSat::new(&l.hole, w, h) }; + (0..w * h).filter(|&i| pm.valid_source((i % w) as i64, (i / w) as i64)).map(|i| [(i % w) as i32, (i / w) as i32]).collect() + }; + if sources.is_empty() { + return; + } + nnf.resize(w * h, None); + for e in 0..em { + let pm = Pm { w, h, r, img: &l.img, sat: HoleSat { w: sat.w, s: sat.s.clone() } }; + // Initialise / validate. + let mut field: Vec<([i32; 2], f32)> = vec![([0, 0], f32::INFINITY); w * h]; + targets + .par_iter() + .map(|&i| { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let s = nnf[i] + .filter(|s| pm.valid_source(s[0] as i64, s[1] as i64)) + .unwrap_or_else(|| sources[rnd(seed ^ e as u32, i as u32, 0) as usize % sources.len()]); + (i, (s, pm.dist(x, y, s[0] as i64, s[1] as i64, f32::INFINITY))) + }) + .collect::>() + .into_iter() + .for_each(|(i, v)| field[i] = v); + let is_target = &near; + for it in 0..pm_iters { + let fwd = it % 2 == 0; + let snap = field.clone(); + let rows: Vec> = (0..h) + .into_par_iter() + .map(|y| { + let mut row: Vec<([i32; 2], f32)> = snap[y * w..(y + 1) * w].to_vec(); + let mut out = vec![]; + let xs: Box> = if fwd { Box::new(0..w) } else { Box::new((0..w).rev()) }; + for x in xs { + let i = y * w + x; + if !is_target[i] { + continue; + } + let (mut best, mut bd) = row[x]; + let (xi, yi) = (x as i64, y as i64); + let try_c = |c: [i32; 2], best: &mut [i32; 2], bd: &mut f32| { + if c != *best && pm.valid_source(c[0] as i64, c[1] as i64) { + let d = pm.dist(xi, yi, c[0] as i64, c[1] as i64, *bd); + if d < *bd { + *bd = d; + *best = c; + } + } + }; + // Propagation from the scan-order neighbours. + let step: i64 = if fwd { -1 } else { 1 }; + let nx = xi + step; + if nx >= 0 && nx < w as i64 && is_target[y * w + nx as usize] { + let s = row[nx as usize].0; + try_c([s[0] - step as i32, s[1]], &mut best, &mut bd); + } + let ny = yi + step; + if ny >= 0 && ny < h as i64 && is_target[ny as usize * w + x] { + let s = snap[ny as usize * w + x].0; + try_c([s[0], s[1] - step as i32], &mut best, &mut bd); + } + // Random search around the best match, halving the radius. + let mut rad = w.max(h) as i64; + let mut k = 0u32; + while rad >= 1 { + let a = rnd(seed.wrapping_add(it as u32 * 977 + e as u32 * 31), i as u32, k); + let ox = (a % (2 * rad as u32 + 1)) as i64 - rad; + let oy = ((a >> 16) % (2 * rad as u32 + 1)) as i64 - rad; + try_c([(best[0] as i64 + ox) as i32, (best[1] as i64 + oy) as i32], &mut best, &mut bd); + rad /= 2; + k += 1; + } + // And a random source anywhere (helps escape bad starts). + let any = sources[rnd(seed ^ 0xABCD, i as u32, it as u32 + e as u32 * 64) as usize % sources.len()]; + try_c(any, &mut best, &mut bd); + row[x] = (best, bd); + out.push((i, (best, bd))); + } + out + }) + .collect(); + for row in rows { + for (i, v) in row { + field[i] = v; + } + } + } + // Vote. + let mut ds: Vec = targets.iter().map(|&i| field[i].1).filter(|d| d.is_finite()).collect(); + ds.sort_by(f32::total_cmp); + let sigma2 = ds.get(ds.len() * 3 / 4).copied().unwrap_or(1.0).max(1e-5); + let ri = r as i64; + let new: Vec<(usize, [f32; 4])> = (0..w * h) + .into_par_iter() + .filter(|&i| l.hole[i]) + .map(|i| { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let mut acc = [0.0f32; 4]; + let mut wsum = 0.0f32; + for dy in -ri..=ri { + for dx in -ri..=ri { + let (px, py) = (x + dx, y + dy); + if px < 0 || py < 0 || px >= w as i64 || py >= h as i64 { + continue; + } + let (s, d) = field[py as usize * w + px as usize]; + if !d.is_finite() { + continue; + } + let (sx, sy) = (s[0] as i64 - dx, s[1] as i64 - dy); + let wt = (-d / (2.0 * sigma2)).exp(); + let v = l.img[sy as usize * w + sx as usize]; + for c in 0..4 { + acc[c] += v[c] * wt; + } + wsum += wt; + } + } + (i, if wsum > 0.0 { acc.map(|v| v / wsum) } else { l.img[i] }) + }) + .collect(); + for (i, v) in new { + l.img[i] = v; + } + for &i in &targets { + nnf[i] = Some(field[i].0); + } + } +} + +/// Fill the `hole` of `img` with PatchMatch synthesis (multi-scale Wexler EM). `init`: a starting +/// guess for the hole (e.g. the previous frame's fill warped here); with it only the finest +/// scale is refined. +pub fn patch_fill(img: &Image, hole: &[bool], init: Option<&Image>, patch: usize, seed: u32) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + if !hole.iter().any(|b| *b) || w == 0 || h == 0 { + return img.clone(); + } + let patch = patch.max(3) | 1; + let fine = Level { w, h, img: img.data.clone(), hole: hole.to_vec() }; + if let Some(init) = init { + let mut l = fine; + for (i, v) in l.img.iter_mut().enumerate() { + if l.hole[i] { + *v = init.data[i]; + } + } + let mut nnf = vec![]; + pm_level(&mut l, &mut nnf, 2, 4, patch, seed); + return Image { width: img.width, height: img.height, data: l.img }; + } + // Pyramid: stop while the image is still a few patches across. + let mut levels = vec![fine]; + while levels.len() < 6 { + let last = levels.last().expect("level"); + if last.w.min(last.h) / 2 < patch * 4 || !last.hole.iter().any(|b| *b) { + break; + } + let d = downsample(last); + // Keep at least one fully known patch. + if !d.hole.iter().any(|b| !*b) { + break; + } + levels.push(d); + } + // Coarsest: membrane start. + let top = levels.len() - 1; + { + let l = &mut levels[top]; + let known: Vec = l.hole.iter().map(|b| !*b).collect(); + membrane_core(&mut l.img, &known, l.w, l.h); + } + let mut nnf: Vec> = vec![]; + for k in (0..=top).rev() { + if k < top { + // Upsample the coarser result into this level's hole and the NNF (×2). + let (coarse_img, cw, ch, coarse_nnf) = { + let c = &levels[k + 1]; + (c.img.clone(), c.w, c.h, nnf.clone()) + }; + let l = &mut levels[k]; + for y in 0..l.h { + for x in 0..l.w { + let i = y * l.w + x; + if l.hole[i] { + // Bilinear sample of the coarse image. + let fx = ((x as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (cw - 1) as f32); + let fy = ((y as f32 + 0.5) / 2.0 - 0.5).clamp(0.0, (ch - 1) as f32); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(cw - 1), (y0 + 1).min(ch - 1)); + let (tx, ty) = (fx - x0 as f32, fy - y0 as f32); + let g = |xx: usize, yy: usize| coarse_img[yy * cw + xx]; + let mut v = [0.0f32; 4]; + for c in 0..4 { + let top = g(x0, y0)[c] + (g(x1, y0)[c] - g(x0, y0)[c]) * tx; + let bot = g(x0, y1)[c] + (g(x1, y1)[c] - g(x0, y1)[c]) * tx; + v[c] = top + (bot - top) * ty; + } + l.img[i] = v; + } + } + } + nnf = (0..l.w * l.h) + .map(|i| { + let (x, y) = (i % l.w, i / l.w); + let ci = (y / 2).min(ch - 1) * cw + (x / 2).min(cw - 1); + coarse_nnf.get(ci).copied().flatten().map(|s| [s[0] * 2 + (x % 2) as i32, s[1] * 2 + (y % 2) as i32]) + }) + .collect(); + } + let em = if k == top { 8 } else { 4 }; + pm_level(&mut levels[k], &mut nnf, em, 5, patch, seed.wrapping_add(k as u32 * 101)); + } + let l = levels.swap_remove(0); + Image { width: img.width, height: img.height, data: l.img } +} + +// ------------------------------------------------------------------ flow-guided propagation + +/// Block flow from `a` to `b` with the tiles that touch either hole replaced by the membrane +/// interpolation of the surrounding motion (flow completion). +fn completed_flow(a: &Image, b: &Image, hole_a: &[bool], hole_b: &[bool]) -> Flow { + let mut f = block_flow(a, b, 8, 6); + if f.v.is_empty() { + return f; + } + let (w, bs) = (a.width as usize, f.block as usize); + let mut known = vec![true; f.v.len()]; + for (i, k) in known.iter_mut().enumerate() { + let (c, r) = (i % f.cols, i / f.cols); + // A tile is unreliable when any pixel within it (and a block's margin) is a hole. + let (x0, y0) = ((c * bs).saturating_sub(bs / 2), (r * bs).saturating_sub(bs / 2)); + let (x1, y1) = (((c + 1) * bs + bs / 2).min(w), ((r + 1) * bs + bs / 2).min(a.height as usize)); + 'tile: for y in y0..y1 { + for x in x0..x1 { + if hole_a[y * w + x] || hole_b[y * w + x] { + *k = false; + break 'tile; + } + } + } + } + if known.iter().all(|k| !*k) { + f.v.iter_mut().for_each(|v| *v = [0.0; 2]); + return f; + } + let mut d: Vec<[f32; 4]> = f.v.iter().map(|v| [v[0], v[1], 0.0, 0.0]).collect(); + membrane_core(&mut d, &known, f.cols, f.rows); + for (v, x) in f.v.iter_mut().zip(d) { + *v = [x[0], x[1]]; + } + f +} + +/// The input of a sequence fill. +pub struct FillInput { + /// Frames (premultiplied RGBA, all the same size). + pub frames: Vec, + /// Per frame, `true` where the pixel is missing. + pub holes: Vec>, + /// Complete images for some frames: (frame index, image). + pub references: Vec<(usize, Image)>, +} + +/// Trace pixel (x, y) of frame `t` through `flows` (`step` = +1 forward with `fwd`, −1 backward +/// with `bwd`) to the nearest frame where it is visible: (sampled colour, frames travelled). +fn trace( + t: usize, + x: f64, + y: f64, + step: i64, + fwd: &[Flow], + bwd: &[Flow], + known: &dyn Fn(usize, i64, i64) -> bool, + frames: &[&Image], + max: usize, +) -> Option<([f32; 4], usize)> { + let n = frames.len(); + let (w, h) = (frames[0].width as f64, frames[0].height as f64); + let (mut px, mut py) = (x, y); + let mut k = t as i64; + for d in 1..=max { + let v = if step > 0 { + if k as usize + 1 >= n { + return None; + } + fwd[k as usize].at(px, py) + } else { + if k == 0 { + return None; + } + bwd[k as usize - 1].at(px, py) + }; + px += v[0] as f64; + py += v[1] as f64; + k += step; + if px < 0.0 || py < 0.0 || px >= w || py >= h { + return None; + } + // Visible when the bilinear support is known. + let (x0, y0) = ((px - 0.5).floor() as i64, (py - 0.5).floor() as i64); + let ok = [(x0, y0), (x0 + 1, y0), (x0, y0 + 1), (x0 + 1, y0 + 1)] + .iter() + .all(|&(xx, yy)| known(k as usize, xx.clamp(0, w as i64 - 1), yy.clamp(0, h as i64 - 1))); + if ok { + return Some((frames[k as usize].sample_bilinear_clamped(px, py), d)); + } + } + None +} + +/// Fill a range of frames. `progress(done, total)` returns `false` to cancel (→ `None`). +pub fn fill_sequence(input: &FillInput, o: &FillOpts, progress: &mut dyn FnMut(usize, usize) -> bool) -> Option> { + let n = input.frames.len(); + if n == 0 { + return Some(vec![]); + } + let (w, h) = (input.frames[0].width as usize, input.frames[0].height as usize); + // Steps: flows (1) + propagation (1) + per-frame synthesis (n). + let total = n + 2; + if o.method == FillMethod::EdgeBlend { + let mut out = Vec::with_capacity(n); + for (i, (f, hl)) in input.frames.iter().zip(&input.holes).enumerate() { + out.push(opaque_fill(membrane_fill(f, hl), hl)); + if !progress(i + 1, n) { + return None; + } + } + return Some(out); + } + // Reference frames make their frame fully known. + let mut frames: Vec = input.frames.clone(); + let mut holes: Vec> = input.holes.clone(); + let mut is_ref = vec![false; n]; + for (k, img) in &input.references { + if *k < n && img.width as usize == w && img.height as usize == h { + frames[*k] = img.clone(); + holes[*k] = vec![false; w * h]; + is_ref[*k] = true; + } + } + // 1. Flows between membrane-filled neighbours, completed inside the holes. + let pre: Vec = frames.par_iter().zip(&holes).map(|(f, hl)| membrane_fill(f, hl)).collect(); + let pairs: Vec<(Flow, Flow)> = (0..n.saturating_sub(1)) + .into_par_iter() + .map(|i| (completed_flow(&pre[i], &pre[i + 1], &holes[i], &holes[i + 1]), completed_flow(&pre[i + 1], &pre[i], &holes[i + 1], &holes[i]))) + .collect(); + let (fwd, bwd): (Vec, Vec) = pairs.into_iter().unzip(); + if !progress(1, total) { + return None; + } + // 2. Propagation (and the lighting-correction border ring). + let refs: Vec<&Image> = frames.iter().collect(); + let known = |k: usize, x: i64, y: i64| !holes[k][y as usize * w + x as usize]; + let ring_r = 2usize; + let propagated: Vec<(Image, Vec)> = (0..n) + .into_par_iter() + .map(|t| { + let mut img = frames[t].clone(); + let mut left = holes[t].clone(); + if is_ref[t] || !holes[t].iter().any(|b| *b) { + return (img, left); + } + let sample = |i: usize| -> Option<[f32; 4]> { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let f = trace(t, x, y, 1, &fwd, &bwd, &known, &refs, n); + let b = trace(t, x, y, -1, &fwd, &bwd, &known, &refs, n); + match (f, b) { + (Some((a, da)), Some((c, dc))) => { + let (wa, wc) = (1.0 / da as f32, 1.0 / dc as f32); + Some(std::array::from_fn(|k| (a[k] * wa + c[k] * wc) / (wa + wc))) + } + (Some((a, _)), None) | (None, Some((a, _))) => Some(a), + _ => None, + } + }; + let filled: Vec<(usize, [f32; 4])> = (0..w * h).filter(|&i| holes[t][i]).filter_map(|i| sample(i).map(|v| (i, v))).collect(); + for (i, v) in &filled { + img.data[*i] = *v; + left[*i] = false; + } + // Lighting correction: membrane of the border mismatch, added to propagated pixels. + if o.lighting > 0.0 && !filled.is_empty() { + let grown = dilate(&holes[t], w, h, ring_r); + let mut diff = vec![[0.0f32; 4]; w * h]; + let mut dk = vec![true; w * h]; + for i in 0..w * h { + if holes[t][i] { + dk[i] = false; + } else if grown[i] + && let Some(v) = sample(i) + { + let a = frames[t].data[i]; + diff[i] = std::array::from_fn(|c| if c < 3 { a[c] - v[c] } else { 0.0 }); + } + } + membrane_core(&mut diff, &dk, w, h); + for (i, _) in &filled { + for c in 0..3 { + img.data[*i][c] += o.lighting * diff[*i][c]; + } + } + } + (img, left) + }) + .collect(); + if !progress(2, total) { + return None; + } + // 3. Synthesis of what no frame shows. + let mut out: Vec> = vec![None; n]; + let first = (0..n).find(|&t| propagated[t].1.iter().any(|b| *b)); + let Some(t0) = first else { + return Some(propagated.into_iter().zip(&holes).map(|((img, _), hl)| opaque_fill(img, hl)).collect()); + }; + // Frames before t0 are complete already. + for (t, slot) in out.iter_mut().enumerate().take(t0) { + *slot = Some(propagated[t].0.clone()); + } + let mut done = 2; + out[t0] = Some(patch_fill(&propagated[t0].0, &propagated[t0].1, None, o.patch, o.seed)); + done += 1; + if !progress(done, total) { + return None; + } + for t in t0 + 1..n { + let (img, left) = &propagated[t]; + if !left.iter().any(|b| *b) { + out[t] = Some(img.clone()); + } else { + // Warp the previous result here along the backward flow (t → t − 1). + let prev = out[t - 1].as_ref().expect("previous frame"); + let mut warped = img.clone(); + for i in 0..w * h { + if left[i] { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let v = bwd[t - 1].at(x, y); + warped.data[i] = prev.sample_bilinear_clamped(x + v[0] as f64, y + v[1] as f64); + } + } + out[t] = Some(match o.method { + FillMethod::Surface => warped, + _ => patch_fill(img, left, Some(&warped), o.patch, o.seed), + }); + } + done += 1; + if !progress(done, total) { + return None; + } + } + Some(out.into_iter().zip(&holes).map(|(img, hl)| opaque_fill(img.unwrap_or_default(), hl)).collect()) +} + +/// Filled pixels are opaque. +fn opaque_fill(mut img: Image, hole: &[bool]) -> Image { + for (p, hl) in img.data.iter_mut().zip(hole) { + if *hl && p[3] > 1e-6 { + let a = p[3]; + *p = [p[0] / a, p[1] / a, p[2] / a, 1.0]; + } + } + img +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash_noise; + + /// A textured background (periodic stripes + blocky noise), shifted by `dx`. + fn background(w: u32, h: u32, dx: f32) -> Image { + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let fx = x as f32 + dx; + let s = 0.5 + 0.3 * (fx * std::f32::consts::TAU / 10.0).sin(); + let n = hash_noise(((fx.floor() as i64).rem_euclid(20) / 4) as u32, (y % 20) / 4, 3); + img.set(x, y, [s, 0.3 + 0.4 * n, 0.6 - 0.3 * s * n, 1.0]); + } + } + img + } + + fn cut(img: &Image, hole: &[bool]) -> Image { + let mut o = img.clone(); + for (p, h) in o.data.iter_mut().zip(hole) { + if *h { + *p = [0.0; 4]; + } + } + o + } + + fn rect_hole(w: u32, h: u32, x0: u32, y0: u32, s: u32) -> Vec { + (0..w * h).map(|i| (x0..x0 + s).contains(&(i % w)) && (y0..y0 + s).contains(&(i / w))).collect() + } + + #[test] + fn membrane_matches_a_linear_ramp() { + let (w, h) = (40u32, 30u32); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let v = x as f32 / w as f32; + img.set(x, y, [v, v, v, 1.0]); + } + } + let hole = rect_hole(w, h, 10, 8, 14); + let f = membrane_fill(&cut(&img, &hole), &hole); + assert!(psnr(&f, &img, Some(&hole)) > 40.0, "{}", psnr(&f, &img, Some(&hole))); + } + + #[test] + fn patchmatch_beats_the_membrane_on_texture() { + let (w, h) = (96u32, 72u32); + let gt = background(w, h, 0.0); + let hole = rect_hole(w, h, 40, 28, 16); + let src = cut(>, &hole); + let mem = membrane_fill(&src, &hole); + let pm = patch_fill(&src, &hole, None, 7, 1); + let (pm_db, mem_db) = (psnr(&pm, >, Some(&hole)), psnr(&mem, >, Some(&hole))); + assert!(pm_db > mem_db + 3.0 && pm_db > 20.0, "PatchMatch {pm_db:.1} dB vs membrane {mem_db:.1} dB"); + // Known pixels are untouched. + for i in 0..hole.len() { + if !hole[i] { + assert_eq!(pm.data[i], gt.data[i]); + } + } + } + + /// A moving object removed from a (static or panning) textured background. + fn sequence(n: usize, pan: f32) -> (FillInput, Vec) { + let (w, h) = (96u32, 64u32); + let mut frames = vec![]; + let mut holes = vec![]; + let mut gts = vec![]; + for t in 0..n { + let gt = background(w, h, t as f32 * pan); + let hole = rect_hole(w, h, 10 + 8 * t as u32, 22, 18); + frames.push(cut(>, &hole)); + holes.push(hole); + gts.push(gt); + } + (FillInput { frames, holes, references: vec![] }, gts) + } + + #[test] + fn object_fill_recovers_background_and_is_stable() { + for pan in [0.0, 1.0] { + let (input, gts) = sequence(8, pan); + let out = fill_sequence(&input, &FillOpts::default(), &mut |_, _| true).expect("fill"); + for t in 0..out.len() { + let db = psnr(&out[t], >s[t], Some(&input.holes[t])); + assert!(db > 26.0, "frame {t} pan {pan}: {db:.1} dB"); + assert_eq!(out[t].data[0][3], 1.0); + } + if pan == 0.0 { + // Temporal consistency: on a static background the filled pixels agree between + // frames. + for t in 1..out.len() { + let both: Vec = input.holes[t].iter().zip(&input.holes[t - 1]).map(|(a, b)| *a || *b).collect(); + let db = psnr(&out[t], &out[t - 1], Some(&both)); + assert!(db > 30.0, "frames {}–{t} differ: {db:.1} dB", t - 1); + } + } + } + } + + #[test] + fn surface_and_edge_blend_and_references() { + // A hole that never moves: nothing to propagate, so synthesis must carry it. + let (w, h) = (80u32, 60u32); + let gt = background(w, h, 0.0); + let hole = rect_hole(w, h, 30, 20, 14); + let input = FillInput { frames: vec![cut(>, &hole); 4], holes: vec![hole.clone(); 4], references: vec![] }; + let surf = fill_sequence(&input, &FillOpts { method: FillMethod::Surface, ..Default::default() }, &mut |_, _| true).expect("fill"); + for t in 1..4 { + // Surface carries one fill through: identical frames. + assert!(psnr(&surf[t], &surf[0], Some(&hole)) > 40.0); + } + let edge = fill_sequence(&input, &FillOpts { method: FillMethod::EdgeBlend, ..Default::default() }, &mut |_, _| true).expect("fill"); + assert!(psnr(&edge[0], &membrane_fill(&input.frames[0], &hole), Some(&hole)) > 60.0); + // A clean reference frame recovers the background exactly. + let mut with_ref = FillInput { frames: input.frames.clone(), holes: input.holes.clone(), references: vec![(2, gt.clone())] }; + let out = fill_sequence(&with_ref, &FillOpts::default(), &mut |_, _| true).expect("fill"); + for (t, f) in out.iter().enumerate() { + assert!(psnr(f, >, Some(&hole)) > 35.0, "frame {t}: {:.1}", psnr(f, >, Some(&hole))); + } + // Cancel. + with_ref.references.clear(); + assert!(fill_sequence(&with_ref, &FillOpts::default(), &mut |d, _| d < 2).is_none()); + } + + #[test] + fn lighting_correction_absorbs_a_brightness_change() { + // The background brightens over time; propagated pixels come from darker frames. + let (mut input, mut gts) = sequence(6, 0.0); + for t in 0..6 { + let k = 1.0 + 0.06 * t as f32; + for img in [&mut input.frames[t], &mut gts[t]] { + for p in &mut img.data { + for c in 0..3 { + p[c] *= k; + } + } + } + } + let plain = fill_sequence(&input, &FillOpts::default(), &mut |_, _| true).expect("fill"); + let lit = fill_sequence(&input, &FillOpts { lighting: 1.0, ..Default::default() }, &mut |_, _| true).expect("fill"); + let score = |o: &[Image]| (0..6).map(|t| psnr(&o[t], >s[t], Some(&input.holes[t]))).sum::(); + assert!(score(&lit) > score(&plain), "lit {} vs plain {}", score(&lit), score(&plain)); + } +} diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index 4f244199..f3e64695 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -5,7 +5,10 @@ pub mod blur; pub mod channels3d; +pub mod cuts; pub mod flow; +pub mod inpaint; +pub mod scopes; pub mod warp; use effectcraft_color::{BlendMode, blend_pixel}; diff --git a/crates/raster/src/scopes.rs b/crates/raster/src/scopes.rs new file mode 100644 index 00000000..befb26ef --- /dev/null +++ b/crates/raster/src/scopes.rs @@ -0,0 +1,552 @@ +//! Video scopes (the Lumetri Scopes panel): waveform (RGB, Luma, YC), vectorscope (YUV, HLS), +//! histogram and parade (RGB, YUV), computed from a frame as density plots. +//! +//! The maths is the textbook broadcast definition: luma `Y = Kr·R + Kg·G + Kb·B` with the +//! ITU-R BT.601 / BT.709 / BT.2020 coefficients, colour differences `Cb = (B − Y) / (2(1 − Kb))` +//! and `Cr = (R − Y) / (2(1 − Kr))` (each in −0.5…0.5). Values are read straight (un-premultiplied) +//! from the frame. Every scope is a [`Scope`]: a `width × height` grid of three density planes +//! (one per trace colour) normalised so the busiest cell is 1, ready to be tinted and drawn. +//! +//! Scales: **8-bit** maps 0…1 to the plot (0…255 graticule); **float** maps −0.25…1.25 so +//! super-white and sub-black values stay visible. **Clamp** limits the signal to 0…1 first. + +use rayon::prelude::*; + +use crate::Image; + +/// Luma / colour-difference coefficients. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ColorStandard { + Rec601, + #[default] + Rec709, + Rec2020, +} + +impl ColorStandard { + pub const ALL: [ColorStandard; 3] = [ColorStandard::Rec601, ColorStandard::Rec709, ColorStandard::Rec2020]; + /// (Kr, Kb). + pub fn coefficients(self) -> (f32, f32) { + match self { + ColorStandard::Rec601 => (0.299, 0.114), + ColorStandard::Rec709 => (0.2126, 0.0722), + ColorStandard::Rec2020 => (0.2627, 0.0593), + } + } + pub fn name(self) -> &'static str { + match self { + ColorStandard::Rec601 => "rec601", + ColorStandard::Rec709 => "rec709", + ColorStandard::Rec2020 => "rec2020", + } + } + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '.', '-', '_'], "").as_str() { + "rec601" | "601" | "bt601" => Some(ColorStandard::Rec601), + "rec709" | "709" | "bt709" => Some(ColorStandard::Rec709), + "rec2020" | "2020" | "bt2020" => Some(ColorStandard::Rec2020), + _ => None, + } + } + /// (Y, Cb, Cr) of a straight RGB colour. + pub fn ycbcr(self, rgb: [f32; 3]) -> [f32; 3] { + let (kr, kb) = self.coefficients(); + let y = kr * rgb[0] + (1.0 - kr - kb) * rgb[1] + kb * rgb[2]; + [y, (rgb[2] - y) / (2.0 * (1.0 - kb)), (rgb[0] - y) / (2.0 * (1.0 - kr))] + } +} + +/// Which scope. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScopeKind { + WaveformRgb, + WaveformLuma, + /// Luma (plane 0) and chroma amplitude `√(Cb² + Cr²)` (plane 2). + WaveformYc, + /// Cb across, Cr up. + VectorscopeYuv, + /// Hue around the circle, saturation outwards (HLS). + VectorscopeHls, + /// R, G, B histograms (bins across, counts up). + Histogram, + ParadeRgb, + ParadeYuv, +} + +impl ScopeKind { + pub const ALL: [ScopeKind; 8] = [ + ScopeKind::WaveformRgb, + ScopeKind::WaveformLuma, + ScopeKind::WaveformYc, + ScopeKind::VectorscopeYuv, + ScopeKind::VectorscopeHls, + ScopeKind::Histogram, + ScopeKind::ParadeRgb, + ScopeKind::ParadeYuv, + ]; + pub fn name(self) -> &'static str { + match self { + ScopeKind::WaveformRgb => "waveformRgb", + ScopeKind::WaveformLuma => "waveformLuma", + ScopeKind::WaveformYc => "waveformYc", + ScopeKind::VectorscopeYuv => "vectorscopeYuv", + ScopeKind::VectorscopeHls => "vectorscopeHls", + ScopeKind::Histogram => "histogram", + ScopeKind::ParadeRgb => "paradeRgb", + ScopeKind::ParadeYuv => "paradeYuv", + } + } + pub fn label(self) -> &'static str { + match self { + ScopeKind::WaveformRgb => "Waveform (RGB)", + ScopeKind::WaveformLuma => "Waveform (Luma)", + ScopeKind::WaveformYc => "Waveform (YC)", + ScopeKind::VectorscopeYuv => "Vectorscope YUV", + ScopeKind::VectorscopeHls => "Vectorscope HLS", + ScopeKind::Histogram => "Histogram", + ScopeKind::ParadeRgb => "Parade (RGB)", + ScopeKind::ParadeYuv => "Parade (YUV)", + } + } + pub fn from_name(s: &str) -> Option { + let n = s.to_ascii_lowercase().replace([' ', '(', ')', '_', '-'], ""); + Self::ALL.iter().copied().find(|k| k.name().to_ascii_lowercase() == n || k.label().to_ascii_lowercase().replace([' ', '(', ')'], "") == n) + } + /// Vectorscopes are square. + pub fn is_vectorscope(self) -> bool { + matches!(self, ScopeKind::VectorscopeYuv | ScopeKind::VectorscopeHls) + } +} + +/// Scope options. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ScopeOpts { + pub standard: ColorStandard, + /// Float scale (−0.25…1.25) instead of 8-bit (0…1). + pub float: bool, + /// Clamp the signal to 0…1 before plotting. + pub clamp: bool, + /// Plot size. + pub width: u32, + pub height: u32, +} + +impl Default for ScopeOpts { + fn default() -> Self { + ScopeOpts { standard: ColorStandard::Rec709, float: false, clamp: true, width: 256, height: 256 } + } +} + +impl ScopeOpts { + /// Signal range shown on the value axis. + pub fn range(&self) -> (f32, f32) { + if self.float { (-0.25, 1.25) } else { (0.0, 1.0) } + } + /// Plot row (0 = top) of a value, or `None` when off-scale. + fn row(&self, v: f32) -> Option { + let (lo, hi) = self.range(); + let f = (v - lo) / (hi - lo); + if !(0.0..=1.0).contains(&f) { + return None; + } + let r = ((1.0 - f) * (self.height - 1) as f32).round() as usize; + Some(r.min(self.height as usize - 1)) + } +} + +/// A computed scope: three density planes (`planes[c][y * width + x]`, 0…1). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Scope { + pub width: u32, + pub height: u32, + pub planes: [Vec; 3], + /// Raw counts before normalisation (same layout), for statistics and tests. + pub counts: [Vec; 3], +} + +impl Scope { + fn new(w: u32, h: u32) -> Scope { + let n = w as usize * h as usize; + Scope { width: w, height: h, planes: [vec![0.0; n], vec![0.0; n], vec![0.0; n]], counts: [vec![0; n], vec![0; n], vec![0; n]] } + } + fn merge(mut self, o: Scope) -> Scope { + for c in 0..3 { + for (a, b) in self.counts[c].iter_mut().zip(&o.counts[c]) { + *a += *b; + } + } + self + } + fn normalise(&mut self) { + // Square-root density so faint traces stay visible next to dense ones. + let max = self.counts.iter().flat_map(|p| p.iter()).copied().max().unwrap_or(0).max(1) as f32; + for c in 0..3 { + self.planes[c] = self.counts[c].iter().map(|&n| (n as f32 / max).sqrt()).collect(); + } + } + /// Total count of plane `c`. + pub fn total(&self, c: usize) -> u64 { + self.counts[c].iter().map(|&n| n as u64).sum() + } + /// The cell of plane `c` with the most samples, as (x, y). + pub fn peak(&self, c: usize) -> (u32, u32) { + let (i, _) = self.counts[c].iter().enumerate().max_by_key(|(_, n)| **n).unwrap_or((0, &0)); + (i as u32 % self.width.max(1), i as u32 / self.width.max(1)) + } + /// Straight RGBA8 picture: plane 0 red, 1 green, 2 blue (`mono`: all planes white). + pub fn to_rgba8(&self, mono: bool) -> Vec { + let n = self.width as usize * self.height as usize; + let mut out = vec![0u8; n * 4]; + for i in 0..n { + let (r, g, b) = (self.planes[0][i], self.planes[1][i], self.planes[2][i]); + let (r, g, b) = if mono { + let m = r.max(g).max(b); + (m, m, m) + } else { + (r, g, b) + }; + let a = r.max(g).max(b); + out[i * 4] = (r.min(1.0) * 255.0) as u8; + out[i * 4 + 1] = (g.min(1.0) * 255.0) as u8; + out[i * 4 + 2] = (b.min(1.0) * 255.0) as u8; + out[i * 4 + 3] = (a.min(1.0) * 255.0) as u8; + } + out + } +} + +/// Straight RGB of pixel `i` (transparent pixels read as black), clamped when asked. +#[inline] +fn rgb(img: &Image, i: usize, clamp: bool) -> [f32; 3] { + let p = img.data[i]; + let mut c = if p[3] > 1e-6 { [p[0] / p[3], p[1] / p[3], p[2] / p[3]] } else { [0.0; 3] }; + // Composite over black, as a monitor shows the frame. + let a = p[3].clamp(0.0, 1.0); + for v in &mut c { + *v *= a; + if clamp { + *v = v.clamp(0.0, 1.0); + } + } + c +} + +/// Compute a scope of `img`. +pub fn compute(img: &Image, kind: ScopeKind, o: &ScopeOpts) -> Scope { + let (w, h) = (o.width.max(8), o.height.max(8)); + let o = ScopeOpts { width: w, height: h, ..*o }; + if img.is_empty() { + return Scope::new(w, h); + } + let iw = img.width as usize; + let rows: Vec = (0..img.height as usize).collect(); + let mut s = rows + .par_chunks(16) + .map(|chunk| { + let mut s = Scope::new(w, h); + for &y in chunk { + for x in 0..iw { + plot(&mut s, kind, &o, rgb(img, y * iw + x, o.clamp), x as f32 / iw as f32); + } + } + s + }) + .reduce(|| Scope::new(w, h), Scope::merge); + s.normalise(); + s +} + +#[inline] +fn bump(s: &mut Scope, c: usize, x: usize, y: usize) { + let i = y * s.width as usize + x.min(s.width as usize - 1); + s.counts[c][i] += 1; +} + +/// Add one pixel (straight RGB, at horizontal position `fx` 0…1 in the frame). +fn plot(s: &mut Scope, kind: ScopeKind, o: &ScopeOpts, c: [f32; 3], fx: f32) { + let w = s.width as usize; + let col = |fx: f32, x0: usize, span: usize| x0 + ((fx * span as f32) as usize).min(span - 1); + match kind { + ScopeKind::WaveformRgb => { + let x = col(fx, 0, w); + for (ch, v) in c.iter().enumerate() { + if let Some(y) = o.row(*v) { + bump(s, ch, x, y); + } + } + } + ScopeKind::WaveformLuma => { + let x = col(fx, 0, w); + let yv = o.standard.ycbcr(c)[0]; + if let Some(y) = o.row(yv) { + for ch in 0..3 { + bump(s, ch, x, y); + } + } + } + ScopeKind::WaveformYc => { + let x = col(fx, 0, w); + let [yv, cb, cr] = o.standard.ycbcr(c); + if let Some(y) = o.row(yv) { + bump(s, 0, x, y); + bump(s, 1, x, y); + } + if let Some(y) = o.row((cb * cb + cr * cr).sqrt()) { + bump(s, 2, x, y); + } + } + ScopeKind::ParadeRgb | ScopeKind::ParadeYuv => { + let third = (w / 3).max(1); + let vals = if kind == ScopeKind::ParadeRgb { + c + } else { + let [y, cb, cr] = o.standard.ycbcr(c); + // Colour differences centred on mid-scale. + [y, cb + 0.5, cr + 0.5] + }; + for (ch, v) in vals.iter().enumerate() { + let x = col(fx, ch * third, third); + if let Some(y) = o.row(*v) { + // RGB parade: tinted per channel; YUV: Y white, Cb blue, Cr red. + match (kind, ch) { + (ScopeKind::ParadeYuv, 0) => { + for p in 0..3 { + bump(s, p, x, y); + } + } + (ScopeKind::ParadeYuv, 1) => bump(s, 2, x, y), + (ScopeKind::ParadeYuv, _) => bump(s, 0, x, y), + _ => bump(s, ch, x, y), + } + } + } + } + ScopeKind::VectorscopeYuv => { + let [_, cb, cr] = o.standard.ycbcr(c); + if let Some((x, y)) = vec_cell(s, cb, cr) { + for ch in 0..3 { + bump(s, ch, x, y); + } + } + } + ScopeKind::VectorscopeHls => { + let (hue, _, sat) = hls(c); + let a = hue.to_radians(); + if let Some((x, y)) = vec_cell(s, sat * 0.5 * a.cos(), sat * 0.5 * a.sin()) { + for ch in 0..3 { + bump(s, ch, x, y); + } + } + } + ScopeKind::Histogram => { + let (lo, hi) = o.range(); + for (ch, v) in c.iter().enumerate() { + let f = (v - lo) / (hi - lo); + if (0.0..=1.0).contains(&f) { + let x = ((f * (w - 1) as f32).round() as usize).min(w - 1); + // Histograms keep their counts in row 0; the bars are drawn by `histogram_bars`. + bump(s, ch, x, 0); + } + } + } + } +} + +/// Vectorscope cell of (u, v) in −0.5…0.5 (the disc fills the plot; +v up). +fn vec_cell(s: &Scope, u: f32, v: f32) -> Option<(usize, usize)> { + let (w, h) = (s.width as f32, s.height as f32); + let x = ((u + 0.5) * (w - 1.0)).round(); + let y = ((0.5 - v) * (h - 1.0)).round(); + if x < 0.0 || y < 0.0 || x >= w || y >= h { + return None; + } + Some((x as usize, y as usize)) +} + +/// (hue degrees, lightness, saturation) of straight RGB (HLS). +pub fn hls(c: [f32; 3]) -> (f32, f32, f32) { + let (r, g, b) = (c[0], c[1], c[2]); + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let l = (max + min) * 0.5; + let d = max - min; + if d < 1e-6 { + return (0.0, l, 0.0); + } + let s = if l > 0.5 { d / (2.0 - max - min).max(1e-6) } else { d / (max + min).max(1e-6) }; + let h = if max == r { + ((g - b) / d).rem_euclid(6.0) + } else if max == g { + (b - r) / d + 2.0 + } else { + (r - g) / d + 4.0 + }; + (h * 60.0, l, s.min(1.0)) +} + +/// Histogram bars: per channel, `bins` counts across the value range (counts in a +/// [`ScopeKind::Histogram`] scope's first row). +pub fn histogram_bars(s: &Scope) -> [Vec; 3] { + let w = s.width as usize; + [s.counts[0][..w].to_vec(), s.counts[1][..w].to_vec(), s.counts[2][..w].to_vec()] +} + +/// Where a vectorscope target (75 % colour bars' primaries and secondaries) sits, as (u, v) in +/// −0.5…0.5, for the graticule. +pub fn vector_targets(standard: ColorStandard, hls_mode: bool) -> Vec<(&'static str, [f32; 2])> { + let cols: [(&str, [f32; 3]); 6] = [ + ("R", [0.75, 0.0, 0.0]), + ("Mg", [0.75, 0.0, 0.75]), + ("B", [0.0, 0.0, 0.75]), + ("Cy", [0.0, 0.75, 0.75]), + ("G", [0.0, 0.75, 0.0]), + ("Yl", [0.75, 0.75, 0.0]), + ]; + cols.iter() + .map(|(n, c)| { + if hls_mode { + let (h, _, s) = hls(*c); + let a = h.to_radians(); + (*n, [s * 0.5 * a.cos(), s * 0.5 * a.sin()]) + } else { + let [_, cb, cr] = standard.ycbcr(*c); + (*n, [cb, cr]) + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn solid(w: u32, h: u32, c: [f32; 3]) -> Image { + Image::filled(w, h, [c[0], c[1], c[2], 1.0]) + } + + #[test] + fn waveform_of_flat_grey_is_one_line() { + let img = solid(64, 32, [0.5, 0.5, 0.5]); + let o = ScopeOpts { width: 64, height: 101, ..Default::default() }; + let s = compute(&img, ScopeKind::WaveformLuma, &o); + // 0.5 sits on row 50 of 0..=100; every sample lands there. + assert_eq!(s.total(0), 64 * 32); + for x in 0..64 { + assert_eq!(s.counts[0][50 * 64 + x], 32); + } + } + + #[test] + fn waveform_rgb_ramps_follow_columns() { + // Horizontal ramp: column x has value x / (w - 1). + let w = 101; + let mut img = Image::new(w, 4); + for y in 0..4 { + for x in 0..w { + let v = x as f32 / (w - 1) as f32; + img.set(x, y, [v, 1.0 - v, 0.0, 1.0]); + } + } + let o = ScopeOpts { width: w, height: 101, ..Default::default() }; + let s = compute(&img, ScopeKind::WaveformRgb, &o); + for x in 0..w as usize { + // Red: row 100 - x; green: row x. + assert_eq!(s.counts[0][(100 - x) * w as usize + x], 4, "red column {x}"); + assert_eq!(s.counts[1][x * w as usize + x], 4, "green column {x}"); + } + } + + #[test] + fn luma_coefficients_per_standard() { + let red = [1.0, 0.0, 0.0]; + assert!((ColorStandard::Rec601.ycbcr(red)[0] - 0.299).abs() < 1e-6); + assert!((ColorStandard::Rec709.ycbcr(red)[0] - 0.2126).abs() < 1e-6); + assert!((ColorStandard::Rec2020.ycbcr(red)[0] - 0.2627).abs() < 1e-6); + // Pure red has Cr = +0.5 in every standard, Cb negative. + for st in ColorStandard::ALL { + let [_, cb, cr] = st.ycbcr(red); + assert!((cr - 0.5).abs() < 1e-6); + assert!(cb < 0.0); + // White and grey have no chroma. + let [_, cb, cr] = st.ycbcr([0.6, 0.6, 0.6]); + assert!(cb.abs() < 1e-6 && cr.abs() < 1e-6); + } + } + + #[test] + fn vectorscope_places_red_on_its_target() { + let img = solid(16, 16, [1.0, 0.0, 0.0]); + let o = ScopeOpts { width: 101, height: 101, ..Default::default() }; + let s = compute(&img, ScopeKind::VectorscopeYuv, &o); + let [_, cb, cr] = ColorStandard::Rec709.ycbcr([1.0, 0.0, 0.0]); + let (x, y) = s.peak(0); + assert_eq!(x, ((cb + 0.5) * 100.0).round() as u32); + assert_eq!(y, ((0.5 - cr) * 100.0).round() as u32); + // Grey sits in the centre. + let g = compute(&solid(8, 8, [0.3, 0.3, 0.3]), ScopeKind::VectorscopeYuv, &o); + assert_eq!(g.peak(0), (50, 50)); + // HLS: fully saturated green at 120° on the rim. + let h = compute(&solid(8, 8, [0.0, 1.0, 0.0]), ScopeKind::VectorscopeHls, &o); + let a = 120f32.to_radians(); + assert_eq!(h.peak(0), (((0.5 * a.cos() + 0.5) * 100.0).round() as u32, ((0.5 - 0.5 * a.sin()) * 100.0).round() as u32)); + } + + #[test] + fn histogram_counts_and_clamp() { + // Half black, half white. + let mut img = Image::new(10, 10); + for y in 0..10 { + for x in 0..10 { + let v = if x < 5 { 0.0 } else { 1.0 }; + img.set(x, y, [v, v, v, 1.0]); + } + } + let o = ScopeOpts { width: 256, height: 64, ..Default::default() }; + let s = compute(&img, ScopeKind::Histogram, &o); + let bars = histogram_bars(&s); + for b in &bars { + assert_eq!(b[0], 50); + assert_eq!(b[255], 50); + assert_eq!(b.iter().sum::(), 100); + } + // Super-white: off scale in 8-bit unclamped, on the float scale, pinned to 1 clamped. + let hot = solid(4, 4, [1.2, 1.2, 1.2]); + let unclamped = ScopeOpts { clamp: false, ..o }; + assert_eq!(compute(&hot, ScopeKind::Histogram, &unclamped).total(0), 0); + let fl = ScopeOpts { clamp: false, float: true, ..o }; + let f = histogram_bars(&compute(&hot, ScopeKind::Histogram, &fl)); + let expect = (((1.2 + 0.25) / 1.5) * 255.0f32).round() as usize; + assert_eq!(f[0][expect], 16); + let c = histogram_bars(&compute(&hot, ScopeKind::Histogram, &o)); + assert_eq!(c[0][255], 16); + } + + #[test] + fn parade_splits_channels_into_thirds() { + let img = solid(30, 3, [1.0, 0.5, 0.0]); + let o = ScopeOpts { width: 90, height: 101, ..Default::default() }; + let s = compute(&img, ScopeKind::ParadeRgb, &o); + // Red trace in the first third at the top, green mid-height in the second, blue at the + // bottom of the third. + let (rx, ry) = s.peak(0); + let (gx, gy) = s.peak(1); + let (bx, by) = s.peak(2); + assert!(rx < 30 && ry == 0); + assert!((30..60).contains(&gx) && gy == 50); + assert!((60..90).contains(&bx) && by == 100); + let yuv = compute(&solid(30, 3, [0.5, 0.5, 0.5]), ScopeKind::ParadeYuv, &o); + // Grey: Y = 0.5 and both colour differences at mid-scale. + assert_eq!(yuv.peak(1).1, 50); + assert_eq!(yuv.peak(2).1, 50); + } + + #[test] + fn yc_waveform_has_chroma_trace() { + let img = solid(8, 8, [1.0, 0.0, 0.0]); + let o = ScopeOpts { width: 8, height: 101, ..Default::default() }; + let s = compute(&img, ScopeKind::WaveformYc, &o); + let [y, cb, cr] = ColorStandard::Rec709.ycbcr([1.0, 0.0, 0.0]); + assert_eq!(s.peak(0).1, ((1.0 - y) * 100.0).round() as u32); + assert_eq!(s.peak(2).1, ((1.0 - (cb * cb + cr * cr).sqrt()) * 100.0).round() as u32); + } +} From 878b47b5edabb21298e9be451bddf71550b7deeb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:23:01 -0400 Subject: [PATCH 04/36] M12.7: adjustment layers on the GPU (effect stacks resident on the device, CPU fallback per non-GPU effect) Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/context.rs | 22 ++++++ crates/gpu/src/effects.rs | 42 +++++++++++ crates/gpu/src/lib.rs | 2 + crates/gpu/src/ops.rs | 11 +++ crates/gpu/src/shaders/kernels.wgsl | 16 +++++ crates/gpu/src/tests_adjust.rs | 105 ++++++++++++++++++++++++++++ crates/gpu/src/walk.rs | 38 +++++++++- crates/render/src/lib.rs | 85 ++++++++++++++++++---- 8 files changed, 306 insertions(+), 15 deletions(-) create mode 100644 crates/gpu/src/tests_adjust.rs diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index d549149d..66bee99d 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -26,6 +26,7 @@ const ENTRIES: &[&str] = &[ "shadow_make", "shadow_combine", "pointwise", + "adjust_mix", ]; /// Entry points that also bind group 1 (four read-only storage buffers; see @@ -582,6 +583,27 @@ impl<'g> Enc<'g> { self.bytes(bytes) } + /// Copy an image into a storage buffer (RGBA f32 rows) for kernels that read a third image + /// through `data`: (buffer, row length in pixels). + pub fn to_buffer(&mut self, img: &GpuImage) -> (wgpu::Buffer, u32) { + let row = (img.width * 16).div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; + let buf = self.g.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("image rows"), + size: row as u64 * img.height as u64, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + self.encoder().copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture: &img.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { + buffer: &buf, + layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(img.height) }, + }, + wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 }, + ); + (buf, row / 16) + } + /// A read-only storage buffer holding `bytes` (padded to 16 bytes). pub fn bytes(&self, mut bytes: Vec) -> wgpu::Buffer { while bytes.len() < 16 || bytes.len() % 4 != 0 { diff --git a/crates/gpu/src/effects.rs b/crates/gpu/src/effects.rs index fdc317b3..36b692f1 100644 --- a/crates/gpu/src/effects.rs +++ b/crates/gpu/src/effects.rs @@ -62,6 +62,48 @@ pub(crate) fn run_chain(e: &mut Enc, chain: &[FxStep], buf: &Buf, levels: Option Some(Buf { img, offset: b.offset, scale: b.scale }) } +/// An effect stack running on a GPU-resident image (adjustment layers): GPU chains stay on +/// the device; CPU effects read the image back, run, and upload the result. +pub(crate) struct GpuFx<'e, 'g> { + e: &'e mut Enc<'g>, + b: Option, + levels: Option, +} + +impl<'e, 'g> GpuFx<'e, 'g> { + pub fn new(e: &'e mut Enc<'g>, img: GpuImage, offset: [f64; 2], scale: f64, levels: Option) -> Self { + GpuFx { e, b: Some(GBuf { img, offset, scale }), levels } + } + + /// The result: (image, offset, scale); `None` when a step failed. + pub fn finish(self) -> Option<(GpuImage, [f64; 2], f64)> { + self.b.map(|b| (b.img, b.offset, b.scale)) + } +} + +impl effectcraft_render::FxTarget for GpuFx<'_, '_> { + fn gpu(&mut self, steps: &[FxStep]) -> bool { + let Some(b) = &self.b else { return false }; + let mut cur = GBuf { img: b.img.clone(), offset: b.offset, scale: b.scale }; + for step in steps { + let Some(next) = apply(self.e, step.spec.id, &step.ctx, cur) else { return false }; + cur = next; + if let Some(l) = self.levels { + cur.img = ops::quantize(self.e, &cur.img, l); + } + } + self.b = Some(cur); + true + } + + fn cpu(&mut self, f: &mut dyn FnMut(Buf) -> Buf) { + let Some(b) = self.b.take() else { return }; + let Some(img) = self.e.download(&b.img) else { return }; + let out = f(Buf { img, offset: b.offset, scale: b.scale }); + self.b = self.e.g.upload_image(&out.img).map(|img| GBuf { img, offset: out.offset, scale: out.scale }); + } +} + fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { match id { "ec.blur.gaussian" => gaussian(e, ctx, b), diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 7a2634b3..a7c9cdc4 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -143,4 +143,6 @@ mod tests; #[cfg(test)] mod tests_3d; #[cfg(test)] +mod tests_adjust; +#[cfg(test)] mod tests_adv3d; diff --git a/crates/gpu/src/ops.rs b/crates/gpu/src/ops.rs index 775e86c1..cd743fbf 100644 --- a/crates/gpu/src/ops.rs +++ b/crates/gpu/src/ops.rs @@ -134,6 +134,17 @@ pub fn channel_mix(e: &mut Enc, canvas: &GpuImage, tmp: &GpuImage, channels: [bo out } +/// Adjustment layer finish: move `canvas` toward `adjusted` by `matte`'s alpha × `opacity`. +pub fn adjust_mix(e: &mut Enc, canvas: &GpuImage, adjusted: &GpuImage, matte: &GpuImage, opacity: f32) -> GpuImage { + let (rows, stride) = e.to_buffer(matte); + let mut p = Params::default(); + p.u[0][0] = stride; + p.f[0][0] = opacity; + let out = e.image(canvas.width, canvas.height); + e.pixels("adjust_mix", &p, canvas, Some(adjusted), &out, Some(&rows)); + out +} + /// Clamp and quantise to `levels` steps (8/16 bpc). pub fn quantize(e: &mut Enc, img: &GpuImage, levels: f32) -> GpuImage { let mut p = Params::default(); diff --git a/crates/gpu/src/shaders/kernels.wgsl b/crates/gpu/src/shaders/kernels.wgsl index c94076a0..443abcf9 100644 --- a/crates/gpu/src/shaders/kernels.wgsl +++ b/crates/gpu/src/shaders/kernels.wgsl @@ -828,3 +828,19 @@ fn pointwise(@builtin(global_invocation_id) gid: vec3) { } textureStore(out, p, px); } + +// Adjustment layer finish (Renderer::draw_adjustment): canvas `src` moves toward the adjusted +// comp `aux` by the footprint's alpha × opacity. The footprint is `data` (RGBA f32 rows of +// u[0].x pixels); f[0].x = opacity. +@compute @workgroup_size(16, 16) +fn adjust_mix(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let c = textureLoad(src, p, 0); + let a = textureLoad(aux, p, 0); + let k = data[(gid.y * P.u[0].x + gid.x) * 4u + 3u] * P.f[0].x; + textureStore(out, p, c + (a - c) * k); +} diff --git a/crates/gpu/src/tests_adjust.rs b/crates/gpu/src/tests_adjust.rs new file mode 100644 index 00000000..ae7f88da --- /dev/null +++ b/crates/gpu/src/tests_adjust.rs @@ -0,0 +1,105 @@ +//! Adjustment layers on the GPU vs the CPU: GPU-only stacks (resident on the device), mixed +//! stacks (CPU effects in between read back and upload), masks, opacity, colour management. + +use effectcraft_keyframe::{ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{BitDepth, ColorSpace, MaskMode}; +use effectcraft_render::{Backend, FxStep, FxTarget, RenderOpts, Renderer}; +use effectcraft_time::Tick; + +use crate::tests::{Pattern, Scene, c, check, compare_at, gpu, n, opts, set}; + +fn adjustment_scene(depth: BitDepth, stack: &[(&str, Vec<(&str, Value)>)], masked: bool) -> Scene { + let mut s = Scene::new(depth); + let bg = s.footage(97, 61); + s.push(bg); + let mut l = s.footage(60, 40); + set(&mut l, "transform/rotation", Value::Scalar(15.0)); + s.push(l); + let mut adj = s.solid([1.0, 1.0, 1.0], 80, 50); + adj.switches.adjustment = true; + set(&mut adj, "transform/opacity", Value::Scalar(75.0)); + set(&mut adj, "transform/rotation", Value::Scalar(-6.0)); + for (id, vals) in stack { + s.effect(&mut adj, id, vals); + } + if masked { + let mut next = s.p.next_id; + let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::ellipse([40.0, 25.0], 60.0, 40.0), MaskMode::Add, [255, 255, 0]); + s.p.next_id = next; + adj.props.sub_mut("masks").unwrap().children.push(m.into()); + } + s.push(adj); + let top = s.solid([0.2, 0.6, 0.9], 20, 20); + s.push(top); + s +} + +fn gpu_stack() -> Vec<(&'static str, Vec<(&'static str, Value)>)> { + vec![ + ("ec.blur.gaussian", vec![("blurriness", n(6.0))]), + ("ec.color.levels", vec![("inBlack", n(0.1)), ("gamma", n(1.4))]), + ("ec.color.tint", vec![("black", c(0.1, 0.0, 0.2)), ("amount", n(60.0))]), + ] +} + +#[test] +fn gpu_adjustment_chains_match_the_cpu() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { + for masked in [false, true] { + let s = adjustment_scene(depth, &gpu_stack(), masked); + check(&format!("gpu adjustment {depth:?} masked {masked}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + check(&format!("gpu adjustment {depth:?} masked {masked} half"), compare_at(&s, RenderOpts { scale: 0.5, ..opts() }, Tick::ZERO), 0.0); + } + } +} + +#[test] +fn mixed_adjustment_chains_fall_back_per_effect() { + let mut stack = gpu_stack(); + stack.insert(1, ("ec.stylize.posterize", vec![])); + stack.push(("ec.stylize.glow", vec![("threshold", n(30.0))])); + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let s = adjustment_scene(depth, &stack, true); + check(&format!("mixed adjustment {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } + // Colour-managed: effects run in the working space, compositing in linear light. + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = adjustment_scene(depth, &stack, false); + s.p.settings.working_space = Some(ColorSpace::Rec709); + s.p.settings.blend_linear = true; + check(&format!("colour-managed adjustment {depth:?}"), compare_at(&s, opts(), Tick::ZERO), 0.0); + } +} + +/// Counts where each run of the stack goes. +#[derive(Default)] +struct Probe { + gpu: usize, + cpu: usize, +} + +impl FxTarget for Probe { + fn gpu(&mut self, steps: &[FxStep]) -> bool { + self.gpu += steps.len(); + true + } + fn cpu(&mut self, _f: &mut dyn FnMut(effectcraft_effects::Buf) -> effectcraft_effects::Buf) { + self.cpu += 1; + } +} + +#[test] +fn adjustment_stacks_split_into_gpu_runs_and_cpu_steps() { + let Some(g) = gpu() else { return }; + let mut stack = gpu_stack(); + stack.insert(1, ("ec.stylize.posterize", vec![])); + let s = adjustment_scene(BitDepth::Bpc32, &stack, false); + let mut r = Renderer::new(&s.p, &Pattern, RenderOpts { backend: Backend::Gpu, ..opts() }); + r.accel = Some(g); + let ctx = r.eval_ctx(s.cid, Tick::ZERO).unwrap(); + let adj = ctx.comp.layers.iter().find(|l| l.switches.adjustment).unwrap(); + let mut p = Probe::default(); + r.run_effects_on(&ctx, adj, true, &mut p); + assert_eq!((p.gpu, p.cpu), (3, 1)); +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index a0b51f1f..cfec880d 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -124,9 +124,14 @@ fn draw_collapsed<'a>(e: &mut Enc, r: &Renderer<'a>, ctx: &EvalCtx<'a>, layer: & } fn draw_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut GpuImage) -> Option<()> { - if layer.switches.adjustment || layer.switches.quality == Quality::Wireframe { - // Adjustment layers run their effects on the comp below (CPU, with GPU effects where - // available); wireframes draw outlines. + if layer.switches.adjustment { + if draw_adjustment(e, r, ctx, layer, canvas).is_some() { + return Some(()); + } + return on_cpu(e, canvas, |img| r.draw_layer_cpu(ctx, layer, img)); + } + if layer.switches.quality == Quality::Wireframe { + // Wireframes draw outlines on the CPU. return on_cpu(e, canvas, |img| r.draw_layer_cpu(ctx, layer, img)); } let opacity = r.layer_opacity(ctx, layer); @@ -162,6 +167,33 @@ fn draw_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: & Some(()) } +/// Adjustment layer (`Renderer::draw_adjustment`): the effect stack runs on the comp below, +/// resident on the GPU (non-GPU effects read back, run on the CPU and upload), and the result +/// replaces the canvas inside the layer's footprint × opacity. `None` = draw it on the CPU. +fn draw_adjustment(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut GpuImage) -> Option<()> { + let opacity = r.layer_opacity(ctx, layer); + let Some(foot) = r.adjustment_footprint(ctx, layer) else { return Some(()) }; + let (w, h) = (canvas.width, canvas.height); + let matte = place(e, r, ctx, layer, &Arc::new(foot), &e.image(w, h), BlendMode::Normal, 1.0)?; + let pipe = r.pipe(); + // Effects run in the working space, not the linear blending space. + let below = match pipe.from_blend() { + Some(c) => ops::convert(e, canvas, &c), + None => canvas.clone(), + }; + let mut fx = crate::effects::GpuFx::new(e, below, [0.0; 2], r.opts.scale, pipe.levels); + r.run_effects_on(ctx, layer, true, &mut fx); + let (mut adjusted, _, _) = fx.finish()?; + if adjusted.width != w || adjusted.height != h { + return Some(()); + } + if let Some(c) = pipe.to_blend() { + adjusted = ops::convert(e, &adjusted, &c); + } + *canvas = ops::adjust_mix(e, canvas, &adjusted, &matte, opacity); + Some(()) +} + fn composite_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, buf: &Arc, canvas: &mut GpuImage, opacity: f32) -> Option<()> { let matte = r.track_matte(ctx, layer).is_some(); if !matte && !layer.preserve_transparency { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 5ae4ebc5..ffb55936 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -242,6 +242,41 @@ pub trait Accelerator: Send + Sync { } } +/// Where an effect stack runs (see [`Renderer::run_effects_on`]): a CPU buffer, or an image +/// resident on an accelerator. +pub trait FxTarget { + /// Run a chain of accelerator-supported effects (quantising after each to the bit depth). + /// `false` = not handled: the effects then run one by one through [`FxTarget::cpu`]. + fn gpu(&mut self, steps: &[FxStep]) -> bool; + /// Run one CPU effect: `f` maps the buffer. + fn cpu(&mut self, f: &mut dyn FnMut(Buf) -> Buf); +} + +/// The CPU effect target (GPU chains go through [`Accelerator::effects`]: upload, readback). +struct CpuFx<'a> { + accel: Option<&'a dyn Accelerator>, + levels: Option, + buf: Option, +} + +impl FxTarget for CpuFx<'_> { + fn gpu(&mut self, steps: &[FxStep]) -> bool { + let (Some(a), Some(b)) = (self.accel, &self.buf) else { return false }; + match a.effects(steps, b, self.levels) { + Some(out) => { + self.buf = Some(out); + true + } + None => false, + } + } + fn cpu(&mut self, f: &mut dyn FnMut(Buf) -> Buf) { + if let Some(b) = self.buf.take() { + self.buf = Some(f(b)); + } + } +} + #[derive(Clone, Copy, Debug)] pub struct RenderOpts { /// Output scale relative to comp pixels (1 = Full, 0.5 = Half…). @@ -592,19 +627,28 @@ impl<'a> Renderer<'a> { } /// Run the first `limit` effects of the layer's stack (by position; disabled ones count). - fn apply_effects_timed( + fn apply_effects_timed(&self, ctx: &EvalCtx, layer: &Layer, buf: Buf, adjustment: bool, limit: usize, timing: Option<&mut Vec<(String, f64)>>) -> Buf { + let mut t = CpuFx { accel: self.active_accel(), levels: self.pipe.levels, buf: Some(buf) }; + self.run_effect_stack(ctx, layer, adjustment, limit, timing, &mut t); + t.buf.unwrap_or_else(|| Buf { img: Image::new(1, 1), offset: [0.0; 2], scale: self.opts.scale }) + } + + /// Run the layer's video effect stack on `target`: runs of GPU-capable effects go to + /// [`FxTarget::gpu`] (when an accelerator is active and supports them), the rest to + /// [`FxTarget::cpu`] one by one (quantised to the bit depth after each). + fn run_effect_stack( &self, ctx: &EvalCtx, layer: &Layer, - mut buf: Buf, adjustment: bool, limit: usize, mut timing: Option<&mut Vec<(String, f64)>>, - ) -> Buf { + target: &mut dyn FxTarget, + ) { if !layer.switches.effects { - return buf; + return; } - let Some(fx) = layer.effects() else { return buf }; + let Some(fx) = layer.effects() else { return }; let lt = layer.layer_time(ctx.time); let size = source_size(self.project, layer); let layer_size = if size.0 == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; @@ -635,7 +679,7 @@ impl<'a> Renderer<'a> { let mut k = 0; while k < stack.len() { let (i, g, spec) = stack[k]; - // A run of GPU effects goes to the accelerator in one upload / readback. + // A run of GPU effects goes to the accelerator in one go. if let Some(a) = accel && spec.gpu && a.supports_effect(spec.id) @@ -651,8 +695,7 @@ impl<'a> Renderer<'a> { }) .collect(); let t0 = web_time::Instant::now(); - if let Some(out) = a.effects(&steps, &buf, self.pipe.levels) { - buf = out; + if target.gpu(&steps) { if let Some(v) = timing.as_deref_mut() { let ms = t0.elapsed().as_secs_f64() * 1e3 / run.len() as f64; v.extend(run.iter().map(|(_, _, s)| (format!("{} (gpu)", s.id), ms))); @@ -667,14 +710,16 @@ impl<'a> Renderer<'a> { let env = EffectEnv { effect_index: i, ..env }; let ectx = EffectCtx { params: ¶ms, time: lt.seconds(), layer_size, seed: g.uid as u32, adjustment, env }; let t0 = web_time::Instant::now(); - buf = effectcraft_effects::apply(spec, &ectx, buf); - // 8/16 bpc effects write integer pixels. - self.pipe.quantize(&mut buf.img); + target.cpu(&mut |buf| { + let mut buf = effectcraft_effects::apply(spec, &ectx, buf); + // 8/16 bpc effects write integer pixels. + self.pipe.quantize(&mut buf.img); + buf + }); if let Some(v) = timing.as_deref_mut() { v.push((spec.id.to_string(), t0.elapsed().as_secs_f64() * 1e3)); } } - buf } /// Source pixels of a layer in layer space (before masks/effects). @@ -1389,6 +1434,22 @@ impl<'a> Renderer<'a> { three_d::compose::gpu_run(self, ctx, run, out) } + /// Run the layer's effect stack on `target` (an accelerator's resident image): GPU-capable + /// runs through [`FxTarget::gpu`], the others through [`FxTarget::cpu`], with the CPU's + /// parameters and environment. `adjustment`: the stack runs on the comp below (adjustment + /// layers). + pub fn run_effects_on(&self, ctx: &EvalCtx, layer: &Layer, adjustment: bool, target: &mut dyn FxTarget) { + self.run_effect_stack(ctx, layer, adjustment, usize::MAX, None, target); + } + + /// An adjustment layer's footprint (its source with masks applied, layer space): where + /// its effects show. `None` = the layer changes nothing. + pub fn adjustment_footprint(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + let mut foot = self.source(ctx, layer)?; + let _ = masks::apply(ctx, layer, &mut foot); + Some(foot) + } + /// Draw one 2D layer into `canvas` on the CPU (any kind: adjustment, wireframe, styled…). pub fn draw_layer_cpu(&self, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image) { self.draw_layer(ctx, layer, canvas, false); From 9a6389db86a0109fddff225533b900141d77e6b0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:25:12 -0400 Subject: [PATCH 05/36] M12.7: GPU effect family modules (colour, distort, generate) scaffolding; clippy fixes Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/classic3d.rs | 2 +- crates/gpu/src/context.rs | 17 ++++++++++++++--- crates/gpu/src/effects.rs | 21 ++++++++++++--------- crates/gpu/src/fx_color.rs | 18 ++++++++++++++++++ crates/gpu/src/fx_distort.rs | 18 ++++++++++++++++++ crates/gpu/src/fx_generate.rs | 18 ++++++++++++++++++ crates/gpu/src/lib.rs | 3 +++ crates/gpu/src/ops.rs | 2 +- crates/gpu/src/shaders/fx_color.wgsl | 2 ++ crates/gpu/src/shaders/fx_distort.wgsl | 2 ++ crates/gpu/src/shaders/fx_generate.wgsl | 2 ++ crates/gpu/src/tests.rs | 4 ++-- 12 files changed, 93 insertions(+), 16 deletions(-) create mode 100644 crates/gpu/src/fx_color.rs create mode 100644 crates/gpu/src/fx_distort.rs create mode 100644 crates/gpu/src/fx_generate.rs create mode 100644 crates/gpu/src/shaders/fx_color.wgsl create mode 100644 crates/gpu/src/shaders/fx_distort.wgsl create mode 100644 crates/gpu/src/shaders/fx_generate.wgsl diff --git a/crates/gpu/src/classic3d.rs b/crates/gpu/src/classic3d.rs index 9e73edb3..0c72dd60 100644 --- a/crates/gpu/src/classic3d.rs +++ b/crates/gpu/src/classic3d.rs @@ -188,7 +188,7 @@ pub(crate) fn draw_run(e: &mut Enc, run: &Run3d, canvas: &GpuImage) -> Option e.g.device.limits().max_storage_buffer_binding_size as u64 { + if (mattes.len() * 4) as u64 > e.g.device.limits().max_storage_buffer_binding_size { return None; } let mut lights = Words::default(); diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index 66bee99d..e7cda93e 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -141,7 +141,15 @@ impl GpuContext { } } let name = format!("{} ({:?})", info.name, info.backend); - let src = [include_str!("shaders/common.wgsl"), include_str!("shaders/kernels.wgsl"), include_str!("shaders/classic3d.wgsl")].concat(); + let src = [ + include_str!("shaders/common.wgsl"), + include_str!("shaders/kernels.wgsl"), + include_str!("shaders/classic3d.wgsl"), + include_str!("shaders/fx_color.wgsl"), + include_str!("shaders/fx_distort.wgsl"), + include_str!("shaders/fx_generate.wgsl"), + ] + .concat(); let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("effectcraft kernels"), source: wgpu::ShaderSource::Wgsl(src.into()) }); let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { @@ -190,6 +198,9 @@ impl GpuContext { }); let pipelines = ENTRIES .iter() + .chain(crate::fx_color::KERNELS) + .chain(crate::fx_distort::KERNELS) + .chain(crate::fx_generate::KERNELS) .map(|e| (e, &layout)) .chain(EXT_ENTRIES.iter().map(|e| (e, &layout_ext))) .map(|(e, layout)| { @@ -585,7 +596,7 @@ impl<'g> Enc<'g> { /// Copy an image into a storage buffer (RGBA f32 rows) for kernels that read a third image /// through `data`: (buffer, row length in pixels). - pub fn to_buffer(&mut self, img: &GpuImage) -> (wgpu::Buffer, u32) { + pub fn image_rows(&mut self, img: &GpuImage) -> (wgpu::Buffer, u32) { let row = (img.width * 16).div_ceil(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT) * wgpu::COPY_BYTES_PER_ROW_ALIGNMENT; let buf = self.g.device.create_buffer(&wgpu::BufferDescriptor { label: Some("image rows"), @@ -606,7 +617,7 @@ impl<'g> Enc<'g> { /// A read-only storage buffer holding `bytes` (padded to 16 bytes). pub fn bytes(&self, mut bytes: Vec) -> wgpu::Buffer { - while bytes.len() < 16 || bytes.len() % 4 != 0 { + while bytes.len() < 16 || !bytes.len().is_multiple_of(4) { bytes.push(0); } self.g.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("data"), contents: &bytes, usage: wgpu::BufferUsages::STORAGE }) diff --git a/crates/gpu/src/effects.rs b/crates/gpu/src/effects.rs index 36b692f1..2ef75c48 100644 --- a/crates/gpu/src/effects.rs +++ b/crates/gpu/src/effects.rs @@ -16,19 +16,19 @@ pub fn supports(id: &str) -> bool { } /// A layer buffer on the GPU (see [`Buf`]). -struct GBuf { - img: GpuImage, - offset: [f64; 2], - scale: f64, +pub(crate) struct GBuf { + pub img: GpuImage, + pub offset: [f64; 2], + pub scale: f64, } impl GBuf { - fn to_px(&self, p: [f64; 2]) -> (f64, f64) { + pub fn to_px(&self, p: [f64; 2]) -> (f64, f64) { (p[0] * self.scale + self.offset[0], p[1] * self.scale + self.offset[1]) } /// Buf::pad: grow by `pad` transparent pixels on every side. - fn pad(&mut self, e: &mut Enc, pad: u32) -> Option<()> { + pub fn pad(&mut self, e: &mut Enc, pad: u32) -> Option<()> { if pad == 0 { return Some(()); } @@ -114,6 +114,9 @@ fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { "ec.distort.transform" => transform(e, ctx, b), "ec.color.curves" => curves(e, ctx, b), _ if id.starts_with("ec.control.") => Some(b), + _ if crate::fx_color::IDS.contains(&id) => crate::fx_color::apply(e, id, ctx, b), + _ if crate::fx_distort::IDS.contains(&id) => crate::fx_distort::apply(e, id, ctx, b), + _ if crate::fx_generate::IDS.contains(&id) => crate::fx_generate::apply(e, id, ctx, b), _ => pointwise(e, id, ctx, b), } } @@ -121,7 +124,7 @@ fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { // ---------------------------------------------------------------- blurs /// raster::blur::box_radii. -fn box_radii(sigma: f64, n: usize) -> Vec { +pub(crate) fn box_radii(sigma: f64, n: usize) -> Vec { if sigma <= 0.0 { return vec![0; n]; } @@ -137,7 +140,7 @@ fn box_radii(sigma: f64, n: usize) -> Vec { } /// Horizontal passes with radii `rx`, then vertical passes with radii `ry` (raster::blur). -fn box_passes(e: &mut Enc, img: &GpuImage, rx: &[usize], ry: &[usize], repeat: bool) -> GpuImage { +pub(crate) fn box_passes(e: &mut Enc, img: &GpuImage, rx: &[usize], ry: &[usize], repeat: bool) -> GpuImage { let mut cur = img.clone(); let passes = rx.iter().map(|&r| (false, r)).chain(ry.iter().map(|&r| (true, r))).filter(|p| p.1 > 0); for (vertical, r) in passes { @@ -153,7 +156,7 @@ fn box_passes(e: &mut Enc, img: &GpuImage, rx: &[usize], ry: &[usize], repeat: b } /// raster::gaussian_blur (3 box passes per axis). -fn gaussian_blur(e: &mut Enc, img: &GpuImage, sx: f64, sy: f64, repeat: bool) -> GpuImage { +pub(crate) fn gaussian_blur(e: &mut Enc, img: &GpuImage, sx: f64, sy: f64, repeat: bool) -> GpuImage { let rx = if sx > 0.05 { box_radii(sx, 3) } else { vec![] }; let ry = if sy > 0.05 { box_radii(sy, 3) } else { vec![] }; box_passes(e, img, &rx, &ry, repeat) diff --git a/crates/gpu/src/fx_color.rs b/crates/gpu/src/fx_color.rs new file mode 100644 index 00000000..8205f042 --- /dev/null +++ b/crates/gpu/src/fx_color.rs @@ -0,0 +1,18 @@ +//! GPU effects, colour family (kernels in `shaders/fx_color.wgsl`): the CPU effects' exact +//! steps with the pixel loops as compute kernels. + +use effectcraft_effects::EffectCtx; + +use crate::context::Enc; +use crate::effects::GBuf; + +/// Compute entry points in `fx_color.wgsl`. +pub(crate) const KERNELS: &[&str] = &[]; + +/// Effect ids implemented here. +pub(crate) const IDS: &[&str] = &[]; + +/// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. +pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { + None +} diff --git a/crates/gpu/src/fx_distort.rs b/crates/gpu/src/fx_distort.rs new file mode 100644 index 00000000..fe92ad91 --- /dev/null +++ b/crates/gpu/src/fx_distort.rs @@ -0,0 +1,18 @@ +//! GPU effects, distortion and stylize family (kernels in `shaders/fx_distort.wgsl`): the CPU effects' exact +//! steps with the pixel loops as compute kernels. + +use effectcraft_effects::EffectCtx; + +use crate::context::Enc; +use crate::effects::GBuf; + +/// Compute entry points in `fx_distort.wgsl`. +pub(crate) const KERNELS: &[&str] = &[]; + +/// Effect ids implemented here. +pub(crate) const IDS: &[&str] = &[]; + +/// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. +pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { + None +} diff --git a/crates/gpu/src/fx_generate.rs b/crates/gpu/src/fx_generate.rs new file mode 100644 index 00000000..61c18def --- /dev/null +++ b/crates/gpu/src/fx_generate.rs @@ -0,0 +1,18 @@ +//! GPU effects, blur, transition and generate family (kernels in `shaders/fx_generate.wgsl`): the CPU effects' exact +//! steps with the pixel loops as compute kernels. + +use effectcraft_effects::EffectCtx; + +use crate::context::Enc; +use crate::effects::GBuf; + +/// Compute entry points in `fx_generate.wgsl`. +pub(crate) const KERNELS: &[&str] = &[]; + +/// Effect ids implemented here. +pub(crate) const IDS: &[&str] = &[]; + +/// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. +pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { + None +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index a7c9cdc4..9ebce209 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -25,6 +25,9 @@ mod adv3d; mod classic3d; mod context; mod effects; +mod fx_color; +mod fx_distort; +mod fx_generate; mod ops; mod walk; diff --git a/crates/gpu/src/ops.rs b/crates/gpu/src/ops.rs index cd743fbf..e7a9c406 100644 --- a/crates/gpu/src/ops.rs +++ b/crates/gpu/src/ops.rs @@ -136,7 +136,7 @@ pub fn channel_mix(e: &mut Enc, canvas: &GpuImage, tmp: &GpuImage, channels: [bo /// Adjustment layer finish: move `canvas` toward `adjusted` by `matte`'s alpha × `opacity`. pub fn adjust_mix(e: &mut Enc, canvas: &GpuImage, adjusted: &GpuImage, matte: &GpuImage, opacity: f32) -> GpuImage { - let (rows, stride) = e.to_buffer(matte); + let (rows, stride) = e.image_rows(matte); let mut p = Params::default(); p.u[0][0] = stride; p.f[0][0] = opacity; diff --git a/crates/gpu/src/shaders/fx_color.wgsl b/crates/gpu/src/shaders/fx_color.wgsl new file mode 100644 index 00000000..3e42688d --- /dev/null +++ b/crates/gpu/src/shaders/fx_color.wgsl @@ -0,0 +1,2 @@ +// GPU effects (color family): see src/fx_color.rs. Each entry point mirrors the CPU effect in +// effectcraft-effects operation for operation; parameter layouts are documented per entry. diff --git a/crates/gpu/src/shaders/fx_distort.wgsl b/crates/gpu/src/shaders/fx_distort.wgsl new file mode 100644 index 00000000..b3ce3463 --- /dev/null +++ b/crates/gpu/src/shaders/fx_distort.wgsl @@ -0,0 +1,2 @@ +// GPU effects (distort family): see src/fx_distort.rs. Each entry point mirrors the CPU effect in +// effectcraft-effects operation for operation; parameter layouts are documented per entry. diff --git a/crates/gpu/src/shaders/fx_generate.wgsl b/crates/gpu/src/shaders/fx_generate.wgsl new file mode 100644 index 00000000..bc67b93a --- /dev/null +++ b/crates/gpu/src/shaders/fx_generate.wgsl @@ -0,0 +1,2 @@ +// GPU effects (generate family): see src/fx_generate.rs. Each entry point mirrors the CPU effect in +// effectcraft-effects operation for operation; parameter layouts are documented per entry. diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 137fdec3..f795b58d 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -379,7 +379,7 @@ fn layer_styles_collapsed_precomps_and_cpu_fallbacks() { /// The effect alone on a buffer: GPU kernels vs the CPU effect (32-bit float), including the /// buffer geometry (padding, offset). -fn effect_direct(id: &str, vals: &[(&str, Value)], adjustment: bool) { +pub(crate) fn effect_direct(id: &str, vals: &[(&str, Value)], adjustment: bool) { let Some(g) = gpu() else { return }; let spec = effectcraft_effects::find(id).unwrap(); let size = [70.0, 44.0]; @@ -401,7 +401,7 @@ fn effect_direct(id: &str, vals: &[(&str, Value)], adjustment: bool) { } /// One footage layer with `fx` applied, composited over a background, at 8 and 32 bpc. -fn effect_case(id: &str, vals: &[(&str, Value)]) { +pub(crate) fn effect_case(id: &str, vals: &[(&str, Value)]) { effect_direct(id, vals, false); effect_direct(id, vals, true); for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { From 9994f5d3092d30e7ad068d1e85b1048e9e32f9fb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:25:56 -0400 Subject: [PATCH 06/36] M7.7: viewer and colour completion View > Switch 3D View > Default, Split with New Locked Viewer, Use Display Color Management (sRGB/P3 display in Settings > Previews), Simulate Output (Rec. 709/2020, P3, sRGB, Linear, NTSC, PAL, Mac 1.8, Custom; Preserve RGB). Project Settings: Color Engine (OCIO built-in config with ACEScg/ACES2065-1 working spaces), HDR clip/compand/ tone map, output space incl. Rec. 2100 PQ/HLG (SMPTE ST 2084, BT.2100, BT.2408 reference white; GPU curves too), Time Display Feet + Frames (35mm/16mm) display and entry. OCIO effects read custom .ocio configs (YAML subset; Matrix/File/Exponent/Log/ LogAffine/CDL/Range/Group transforms, roles, displays/views). Co-Authored-By: Claude Opus 5.5 --- crates/color/src/space.rs | 255 +++++- crates/effects/src/lib.rs | 1 + crates/effects/src/ocio.rs | 62 +- crates/effects/src/ocio_config.rs | 928 ++++++++++++++++++++++ crates/engine/src/commands/file.rs | 47 +- crates/engine/src/commands/mod.rs | 2 +- crates/engine/src/commands/stubs.rs | 5 - crates/engine/src/commands/three_d.rs | 6 +- crates/engine/src/commands/time.rs | 34 +- crates/engine/src/commands/viewer_cmds.rs | 126 ++- crates/engine/src/lib.rs | 5 + crates/engine/src/menus.rs | 14 + crates/engine/src/prefs.rs | 3 + crates/engine/src/tests_color_view.rs | 223 ++++++ crates/engine/src/viewer.rs | 268 +++++++ crates/gpu/src/ops.rs | 12 +- crates/gpu/src/shaders/common.wgsl | 47 +- crates/gpu/src/walk.rs | 8 + crates/project/src/lib.rs | 69 ++ crates/render/src/color.rs | 74 +- crates/render/src/lib.rs | 6 + crates/render/src/three_d/camera.rs | 9 +- crates/time/src/lib.rs | 74 ++ crates/ui-egui/src/panels/forms.rs | 60 +- crates/ui-egui/src/panels/mod.rs | 12 +- crates/ui-egui/src/panels/viewer.rs | 65 ++ crates/ui-egui/src/panels/viewer_tools.rs | 21 +- docs/parity.md | 9 +- docs/preferences.md | 1 + 29 files changed, 2362 insertions(+), 84 deletions(-) create mode 100644 crates/effects/src/ocio_config.rs create mode 100644 crates/engine/src/tests_color_view.rs diff --git a/crates/color/src/space.rs b/crates/color/src/space.rs index 2bc93513..ee7590ec 100644 --- a/crates/color/src/space.rs +++ b/crates/color/src/space.rs @@ -1,5 +1,5 @@ -//! RGB colour spaces for colour management: the project working space, footage interpretation -//! and the display. +//! RGB colour spaces for colour management: the project working space, footage interpretation, +//! output and the display. //! //! Each space is defined by its published primaries and white point and its transfer function: //! @@ -7,15 +7,23 @@ //! * **Rec. 709** — ITU-R BT.709-6 primaries, D65, the BT.1886 display EOTF (gamma 2.4). //! * **Rec. 2020** — ITU-R BT.2020-2 primaries, D65, BT.1886 (gamma 2.4). //! * **Display P3** — SMPTE EG 432-1 (DCI-P3) primaries with D65 and the sRGB curve. +//! * **ACEScg** — AP1 primaries (Academy S-2014-004), the ACES white (x 0.32168, y 0.33767), +//! linear. +//! * **ACES2065-1** — AP0 primaries (SMPTE ST 2065-1), the ACES white, linear. +//! * **Rec. 2100 PQ** — BT.2020 primaries with the SMPTE ST 2084 / ITU-R BT.2100 perceptual +//! quantizer; linear 1.0 is the HDR reference white of ITU-R BT.2408 (203 cd/m²). +//! * **Rec. 2100 HLG** — BT.2020 primaries with the ITU-R BT.2100 hybrid log-gamma OETF; +//! linear 1.0 is the BT.2408 reference white (75% HLG signal). //! //! RGB → XYZ matrices are derived from the primaries (the standard construction from chromaticity -//! coordinates: columns of xyY → XYZ scaled so that RGB (1, 1, 1) maps to the white point), so +//! coordinates: columns of xyY → XYZ scaled so that RGB (1, 1, 1) maps to the white point), and +//! spaces whose white is not D65 (ACES) are adapted to D65 with the Bradford transform, so //! conversions between any two spaces are exact up to `f64` rounding. Transfer functions extend //! to over-range and negative values (32 bpc) by mirroring around zero. use serde::{Deserialize, Serialize}; -/// An RGB colour space (all D65). +/// An RGB colour space. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColorSpace { /// sRGB IEC 61966-2-1. @@ -26,13 +34,117 @@ pub enum ColorSpace { Rec2020, /// Display P3. DisplayP3, + /// ACEScg (AP1, linear). + AcesCg, + /// ACES2065-1 (AP0, linear). + Aces2065, + /// Rec. 2100 PQ (BT.2020 primaries, SMPTE ST 2084). + Rec2100Pq, + /// Rec. 2100 HLG (BT.2020 primaries, hybrid log-gamma). + Rec2100Hlg, } /// CIE xy chromaticity of D65. const D65: [f64; 2] = [0.3127, 0.3290]; +/// CIE xy chromaticity of the ACES white point. +pub const ACES_WHITE: [f64; 2] = [0.32168, 0.33767]; +const AP0: [[f64; 2]; 3] = [[0.7347, 0.2653], [0.0, 1.0], [0.0001, -0.0770]]; +const AP1: [[f64; 2]; 3] = [[0.713, 0.293], [0.165, 0.830], [0.128, 0.044]]; +const BT2020: [[f64; 2]; 3] = [[0.708, 0.292], [0.170, 0.797], [0.131, 0.046]]; + +/// SMPTE ST 2084 constants. +const PQ_M1: f64 = 2610.0 / 16384.0; +const PQ_M2: f64 = 2523.0 / 4096.0 * 128.0; +const PQ_C1: f64 = 3424.0 / 4096.0; +const PQ_C2: f64 = 2413.0 / 4096.0 * 32.0; +const PQ_C3: f64 = 2392.0 / 4096.0 * 32.0; +/// Linear 1.0 in cd/m² (ITU-R BT.2408 HDR reference white). +pub const PQ_REFERENCE_WHITE: f64 = 203.0; +/// ITU-R BT.2100 HLG constants. +const HLG_A: f64 = 0.178_832_77; +const HLG_B: f64 = 0.284_668_92; +const HLG_C: f64 = 0.559_910_73; +/// Scene light (E, 0..1) of linear 1.0: the BT.2408 reference white at a 75% HLG signal. +pub const HLG_REFERENCE: f64 = 0.264_962_56; + +/// SMPTE ST 2084 inverse EOTF: absolute luminance / 10 000 cd/m² → signal. +pub fn pq_encode(y: f64) -> f64 { + let y = y.max(0.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * y) / (1.0 + PQ_C3 * y)).powf(PQ_M2) +} + +/// SMPTE ST 2084 EOTF: signal → absolute luminance / 10 000 cd/m². +pub fn pq_decode(e: f64) -> f64 { + let p = e.clamp(0.0, 1.0).powf(1.0 / PQ_M2); + ((p - PQ_C1).max(0.0) / (PQ_C2 - PQ_C3 * p)).powf(1.0 / PQ_M1) +} + +/// ITU-R BT.2100 HLG OETF: scene light E (0..1) → signal. +pub fn hlg_encode(e: f64) -> f64 { + let e = e.max(0.0); + if e <= 1.0 / 12.0 { (3.0 * e).sqrt() } else { HLG_A * (12.0 * e - HLG_B).ln() + HLG_C } +} + +/// ITU-R BT.2100 HLG inverse OETF: signal → scene light E. +pub fn hlg_decode(v: f64) -> f64 { + let v = v.max(0.0); + if v <= 0.5 { v * v / 3.0 } else { (((v - HLG_C) / HLG_A).exp() + HLG_B) / 12.0 } +} + +/// Transfer curves (see [`ColorSpace::curve`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Curve { + Linear, + Srgb, + /// BT.1886 gamma 2.4. + Gamma24, + Pq, + Hlg, +} + +impl Curve { + /// Encoded value → linear light. + pub fn decode(self, v: f32) -> f32 { + let a = v.abs(); + let l = match self { + Curve::Linear => a, + Curve::Srgb => crate::srgb_to_linear(a), + Curve::Gamma24 => a.powf(2.4), + Curve::Pq => (pq_decode(a as f64) * 10_000.0 / PQ_REFERENCE_WHITE) as f32, + Curve::Hlg => (hlg_decode(a as f64) / HLG_REFERENCE) as f32, + }; + l.copysign(v) + } + + /// Linear light → encoded value. + pub fn encode(self, v: f32) -> f32 { + let a = v.abs(); + let e = match self { + Curve::Linear => a, + Curve::Srgb => crate::linear_to_srgb(a), + Curve::Gamma24 => a.powf(1.0 / 2.4), + Curve::Pq => pq_encode(a as f64 * PQ_REFERENCE_WHITE / 10_000.0) as f32, + Curve::Hlg => hlg_encode(a as f64 * HLG_REFERENCE) as f32, + }; + e.copysign(v) + } +} impl ColorSpace { - pub const ALL: [ColorSpace; 4] = [ColorSpace::Srgb, ColorSpace::Rec709, ColorSpace::Rec2020, ColorSpace::DisplayP3]; + /// Every space (footage interpretation, output). + pub const ALL: [ColorSpace; 8] = [ + ColorSpace::Srgb, + ColorSpace::Rec709, + ColorSpace::Rec2020, + ColorSpace::DisplayP3, + ColorSpace::AcesCg, + ColorSpace::Aces2065, + ColorSpace::Rec2100Pq, + ColorSpace::Rec2100Hlg, + ]; + /// Spaces a project can work in (Project Settings ▸ Color ▸ Working Space). + pub const WORKING: [ColorSpace; 6] = + [ColorSpace::Srgb, ColorSpace::Rec709, ColorSpace::Rec2020, ColorSpace::DisplayP3, ColorSpace::AcesCg, ColorSpace::Aces2065]; /// Display name (as in Project Settings ▸ Color ▸ Working Space). pub fn label(self) -> &'static str { @@ -41,16 +153,25 @@ impl ColorSpace { ColorSpace::Rec709 => "HDTV (Rec. 709)", ColorSpace::Rec2020 => "Rec. 2020", ColorSpace::DisplayP3 => "Display P3", + ColorSpace::AcesCg => "ACEScg", + ColorSpace::Aces2065 => "ACES2065-1", + ColorSpace::Rec2100Pq => "Rec. 2100 PQ", + ColorSpace::Rec2100Hlg => "Rec. 2100 HLG", } } - /// Short identifier used by commands and files (`srgb`, `rec709`, `rec2020`, `p3`). + /// Short identifier used by commands and files (`srgb`, `rec709`, `rec2020`, `p3`, `acescg`, + /// `aces2065`, `rec2100pq`, `rec2100hlg`). pub fn id(self) -> &'static str { match self { ColorSpace::Srgb => "srgb", ColorSpace::Rec709 => "rec709", ColorSpace::Rec2020 => "rec2020", ColorSpace::DisplayP3 => "p3", + ColorSpace::AcesCg => "acescg", + ColorSpace::Aces2065 => "aces2065", + ColorSpace::Rec2100Pq => "rec2100pq", + ColorSpace::Rec2100Hlg => "rec2100hlg", } } @@ -62,6 +183,10 @@ impl ColorSpace { "rec709" | "bt709" | "hdtvrec709" | "hdtv" | "709" => Some(ColorSpace::Rec709), "rec2020" | "bt2020" | "2020" | "uhdtv" => Some(ColorSpace::Rec2020), "p3" | "displayp3" | "p3d65" | "dcip3d65" => Some(ColorSpace::DisplayP3), + "acescg" | "ap1" => Some(ColorSpace::AcesCg), + "aces2065" | "aces20651" | "aces" | "ap0" => Some(ColorSpace::Aces2065), + "rec2100pq" | "bt2100pq" | "pq" | "st2084" | "hdr10" => Some(ColorSpace::Rec2100Pq), + "rec2100hlg" | "bt2100hlg" | "hlg" => Some(ColorSpace::Rec2100Hlg), _ => None, } } @@ -70,37 +195,70 @@ impl ColorSpace { pub fn primaries(self) -> [[f64; 2]; 3] { match self { ColorSpace::Srgb | ColorSpace::Rec709 => [[0.640, 0.330], [0.300, 0.600], [0.150, 0.060]], - ColorSpace::Rec2020 => [[0.708, 0.292], [0.170, 0.797], [0.131, 0.046]], + ColorSpace::Rec2020 | ColorSpace::Rec2100Pq | ColorSpace::Rec2100Hlg => BT2020, ColorSpace::DisplayP3 => [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]], + ColorSpace::AcesCg => AP1, + ColorSpace::Aces2065 => AP0, } } - /// Linear RGB → CIE XYZ (Y of white = 1). + /// White point (CIE xy). + pub fn white(self) -> [f64; 2] { + match self { + ColorSpace::AcesCg | ColorSpace::Aces2065 => ACES_WHITE, + _ => D65, + } + } + + /// The transfer curve. + pub fn curve(self) -> Curve { + match self { + ColorSpace::Srgb | ColorSpace::DisplayP3 => Curve::Srgb, + ColorSpace::Rec709 | ColorSpace::Rec2020 => Curve::Gamma24, + ColorSpace::AcesCg | ColorSpace::Aces2065 => Curve::Linear, + ColorSpace::Rec2100Pq => Curve::Pq, + ColorSpace::Rec2100Hlg => Curve::Hlg, + } + } + + /// The space is linear light by definition (ACES). + pub fn is_linear(self) -> bool { + self.curve() == Curve::Linear + } + + /// A high-dynamic-range encoding (Rec. 2100 PQ / HLG). + pub fn is_hdr(self) -> bool { + matches!(self.curve(), Curve::Pq | Curve::Hlg) + } + + /// Linear RGB → CIE XYZ D65 (Y of white = 1; non-D65 spaces Bradford-adapted). pub fn to_xyz(self) -> [[f64; 3]; 3] { - rgb_to_xyz(self.primaries(), D65) + let w = self.white(); + let m = rgb_to_xyz(self.primaries(), w); + if w == D65 { m } else { mul(&bradford(w, D65), &m) } } /// Encoded value → linear light. pub fn decode(self, v: f32) -> f32 { - let a = v.abs(); - let l = match self { - ColorSpace::Srgb | ColorSpace::DisplayP3 => crate::srgb_to_linear(a), - ColorSpace::Rec709 | ColorSpace::Rec2020 => a.powf(2.4), - }; - l.copysign(v) + self.curve().decode(v) } /// Linear light → encoded value. pub fn encode(self, v: f32) -> f32 { - let a = v.abs(); - let e = match self { - ColorSpace::Srgb | ColorSpace::DisplayP3 => crate::linear_to_srgb(a), - ColorSpace::Rec709 | ColorSpace::Rec2020 => a.powf(1.0 / 2.4), - }; - e.copysign(v) + self.curve().encode(v) } } +/// Bradford chromatic adaptation from white `src` to white `dst` (CIE xy). +pub fn bradford(src: [f64; 2], dst: [f64; 2]) -> [[f64; 3]; 3] { + const MB: [[f64; 3]; 3] = [[0.8951, 0.2664, -0.1614], [-0.7502, 1.7135, 0.0367], [0.0389, -0.0685, 1.0296]]; + let xyz = |c: [f64; 2]| [c[0] / c[1], 1.0, (1.0 - c[0] - c[1]) / c[1]]; + let s = mul_vec(&MB, xyz(src)); + let d = mul_vec(&MB, xyz(dst)); + let diag = [[d[0] / s[0], 0.0, 0.0], [0.0, d[1] / s[1], 0.0], [0.0, 0.0, d[2] / s[2]]]; + mul(&invert(&MB), &mul(&diag, &MB)) +} + /// The standard RGB → XYZ matrix from primaries and white point. pub fn rgb_to_xyz(p: [[f64; 2]; 3], white: [f64; 2]) -> [[f64; 3]; 3] { let xyz = |c: [f64; 2]| [c[0] / c[1], 1.0, (1.0 - c[0] - c[1]) / c[1]]; @@ -160,7 +318,7 @@ impl Conversion { /// the conversion is the identity. pub fn new(from: ColorSpace, from_linear: bool, to: ColorSpace, to_linear: bool) -> Option { let same_primaries = from.primaries() == to.primaries(); - let same_curve = from_linear == to_linear && (from_linear || curve_eq(from, to)); + let same_curve = from_linear == to_linear && (from_linear || from.curve() == to.curve()); if same_primaries && same_curve { return None; } @@ -201,11 +359,6 @@ impl Conversion { } } -fn curve_eq(a: ColorSpace, b: ColorSpace) -> bool { - let srgb = |s: ColorSpace| matches!(s, ColorSpace::Srgb | ColorSpace::DisplayP3); - srgb(a) == srgb(b) -} - #[cfg(test)] mod tests { use super::*; @@ -235,11 +388,17 @@ mod tests { let back = Conversion::new(b, bl, a, al); assert_eq!(fwd.is_none(), back.is_none()); for c in samples { + // HDR signals are 0..1. + if a.is_hdr() && !al && c.iter().any(|v| *v > 1.0) { + continue; + } let mid = fwd.map(|f| f.apply(c)).unwrap_or(c); let o = back.map(|f| f.apply(mid)).unwrap_or(mid); for i in 0..3 { - // Pure power curves amplify f32 rounding near zero. - assert!((o[i] - c[i]).abs() < 2e-3, "{a:?}{al}→{b:?}{bl}: {c:?} → {mid:?} → {o:?}"); + // Pure power curves amplify f32 rounding near zero; PQ signals near 1.0 + // stand for ~50× reference white. + let tol = if a.is_hdr() || b.is_hdr() { 1e-2 } else { 2e-3 }; + assert!((o[i] - c[i]).abs() < tol, "{a:?}{al}→{b:?}{bl}: {c:?} → {mid:?} → {o:?}"); } } } @@ -270,4 +429,42 @@ mod tests { assert_eq!(ColorSpace::parse("sRGB IEC61966-2.1"), Some(ColorSpace::Srgb)); assert_eq!(ColorSpace::parse("Display P3"), Some(ColorSpace::DisplayP3)); } + #[test] + fn aces_and_rec2100_spaces() { + // SMPTE ST 2065-1 AP0 → XYZ (ACES white, unadapted). + let m = rgb_to_xyz(ColorSpace::Aces2065.primaries(), ACES_WHITE); + let want = [[0.952_552_4, 0.0, 0.000_093_68], [0.343_966_4, 0.728_166_1, -0.072_132_5], [0.0, 0.0, 1.008_825_2]]; + for i in 0..3 { + for j in 0..3 { + assert!((m[i][j] - want[i][j]).abs() < 1e-5, "{i}{j}: {}", m[i][j]); + } + } + // ACEScg white → sRGB white; ACEScg red sits outside sRGB (negative green/blue). + let c = Conversion::new(ColorSpace::AcesCg, true, ColorSpace::Srgb, true).unwrap(); + let w = c.apply([1.0, 1.0, 1.0]); + assert!(w.iter().all(|v| (v - 1.0).abs() < 1e-3), "{w:?}"); + let r = c.apply([1.0, 0.0, 0.0]); + assert!((r[0] - 1.705).abs() < 3e-3 && r[1] < 0.0 && r[2] < 0.0, "{r:?}"); + assert!(ColorSpace::AcesCg.is_linear() && !ColorSpace::Srgb.is_linear()); + // PQ: the BT.2408 reference white (1.0 = 203 cd/m²) encodes to ≈ 0.58; 10 000 cd/m² to 1. + let pq = ColorSpace::Rec2100Pq; + assert!((pq.encode(1.0) - 0.5807).abs() < 1e-3, "{}", pq.encode(1.0)); + assert!((pq.encode(10_000.0 / 203.0) - 1.0).abs() < 1e-4); + assert!((pq_encode(100.0 / 10_000.0) - 0.508).abs() < 1e-3); + // HLG: reference white at 75%; the OETF is continuous at 1/12. + let hlg = ColorSpace::Rec2100Hlg; + assert!((hlg.encode(1.0) - 0.75).abs() < 1e-5, "{}", hlg.encode(1.0)); + assert!((hlg_encode(1.0 / 12.0) - 0.5).abs() < 1e-9 && (hlg_encode(1.0) - 1.0).abs() < 1e-6); + for v in [0.0f32, 0.01, 0.18, 1.0, 3.0] { + assert!((pq.decode(pq.encode(v)) - v).abs() < 1e-3 * v.max(1.0), "pq {v}"); + assert!((hlg.decode(hlg.encode(v)) - v).abs() < 1e-3 * v.max(1.0), "hlg {v}"); + } + assert!(pq.is_hdr() && hlg.is_hdr() && !ColorSpace::Rec2020.is_hdr()); + // Same primaries as Rec. 2020: only the curve changes. + assert!(Conversion::new(ColorSpace::Rec2020, false, ColorSpace::Rec2100Pq, false).unwrap().matrix.is_none()); + for cs in ColorSpace::ALL { + assert_eq!(ColorSpace::parse(cs.id()), Some(cs)); + assert_eq!(ColorSpace::parse(cs.label()), Some(cs), "{}", cs.label()); + } + } } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 39912324..42fb9f3c 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -34,6 +34,7 @@ mod noise; mod noise2; mod obsolete; mod ocio; +pub mod ocio_config; pub mod paint; mod perspective; mod perspective2; diff --git a/crates/effects/src/ocio.rs b/crates/effects/src/ocio.rs index d868a8f9..b3ffe363 100644 --- a/crates/effects/src/ocio.rs +++ b/crates/effects/src/ocio.rs @@ -559,6 +559,10 @@ pub const DISPLAYS: &[(&str, [[f64; 2]; 3], Tf)] = pub const VIEWS: &[&str] = &["Standard", "Tone Mapped", "Raw"]; +/// Configuration popup: the built-in config, or a custom `.ocio` file (Config File: a path or +/// the config's text; spaces chosen by name, see [`crate::ocio_config`]). +pub const CONFIGS: &[&str] = &["Built-in (EffectCraft minimal)", "Custom (.ocio file)"]; + /// Built-in looks: CDLs applied in ACEScct. pub fn looks() -> Vec<(&'static str, Cdl)> { let grey = 0.413_588_402_492_442_1; // ACEScct of 0.18 @@ -647,7 +651,35 @@ fn space_at(i: u32) -> &'static Space { &SPACES[(i as usize).min(SPACES.len() - 1)] } +/// The custom config chosen by an effect (Configuration ▸ Custom with a readable file). +fn custom_config(ctx: &EffectCtx) -> Option> { + if ctx.params.e("config") != 1 { + return None; + } + crate::ocio_config::load_config(ctx.params.s("configFile")) +} + +/// A space of a custom config by its typed name, else by the built-in popup's name. +fn custom_space<'c>(cfg: &'c crate::ocio_config::Config, name: &str, popup: u32) -> Option<&'c crate::ocio_config::ConfigSpace> { + cfg.space(name).or_else(|| cfg.space(space_at(popup).name)) +} + fn ocio_cst(ctx: &EffectCtx, mut b: Buf) -> Buf { + // Custom configuration that can't be read: pixels pass through. + if ctx.params.e("config") == 1 && custom_config(ctx).is_none() { + return b; + } + if let Some(cfg) = custom_config(ctx) { + let (Some(a), Some(z)) = ( + custom_space(&cfg, ctx.params.s("sourceName"), ctx.params.e("source")), + custom_space(&cfg, ctx.params.s("destinationName"), ctx.params.e("destination")), + ) else { + return b; + }; + let (a, z) = if ctx.params.e("direction") == 1 { (z, a) } else { (a, z) }; + map_px(&mut b, |c| cfg.convert(a, z, c.map(|v| v as f64)).map(|v| v as f32)); + return b; + } let (a, z) = (space_at(ctx.params.e("source")), space_at(ctx.params.e("destination"))); let (a, z) = if ctx.params.e("direction") == 1 { (z, a) } else { (a, z) }; if a.name == z.name { @@ -683,6 +715,21 @@ fn ocio_cdl(ctx: &EffectCtx, mut b: Buf) -> Buf { } fn ocio_display(ctx: &EffectCtx, mut b: Buf) -> Buf { + // Custom configuration that can't be read: pixels pass through. + if ctx.params.e("config") == 1 && custom_config(ctx).is_none() { + return b; + } + if let Some(cfg) = custom_config(ctx) { + // The display's view names a colour space of the config. + let display = DISPLAYS.get(ctx.params.e("display") as usize).map_or("", |d| d.0); + let view = VIEWS.get(ctx.params.e("view") as usize).copied().unwrap_or(""); + let dn = if ctx.params.s("displayName").is_empty() { display } else { ctx.params.s("displayName") }; + let vn = if ctx.params.s("viewName").is_empty() { view } else { ctx.params.s("viewName") }; + let (Some(a), Some(z)) = (custom_space(&cfg, ctx.params.s("sourceName"), ctx.params.e("source")), cfg.view_space(dn, vn)) else { return b }; + let (a, z) = if ctx.params.e("direction") == 1 { (z, a) } else { (a, z) }; + map_px(&mut b, |c| cfg.convert(a, z, c.map(|v| v as f64)).map(|v| v as f32)); + return b; + } let input = space_at(ctx.params.e("source")); let f = display_xform(input, ctx.params.e("display") as usize, ctx.params.e("view") as usize, ctx.params.e("direction") == 1); map_px(&mut b, f); @@ -758,6 +805,10 @@ fn profile(ctx: &EffectCtx, i: u32) -> (Space, bool) { ColorSpace::Rec709 => sp("Rec.709", REC709, D65, Tf::Gamma(2.4)), ColorSpace::Rec2020 => sp("Rec.2020", REC2020, D65, Tf::Gamma(2.4)), ColorSpace::DisplayP3 => sp("Display P3", P3, D65, Tf::Srgb), + ColorSpace::AcesCg => *space_by_name("ACEScg").expect("built-in"), + ColorSpace::Aces2065 => *space_by_name("ACES2065-1").expect("built-in"), + // HDR encodings are output spaces, never a working space. + ColorSpace::Rec2100Pq | ColorSpace::Rec2100Hlg => sp("Rec.2020", REC2020, D65, Tf::Gamma(2.4)), }; match i { 0 => (ws(ctx.env.working_space.unwrap_or(ColorSpace::Srgb)), ctx.env.working_linear), @@ -919,9 +970,12 @@ pub fn specs() -> Vec { "OCIO Color Space Transform", cc, vec![ - p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + p("config", "Configuration", Value::Enum(0), popup(CONFIGS)), + p("configFile", "Config File (.ocio)", Value::Str(String::new()), ParamUi::Text), space_param("source", "Source", lin709), + p("sourceName", "Source (config name)", Value::Str(String::new()), ParamUi::Text), space_param("destination", "Destination", srgb), + p("destinationName", "Destination (config name)", Value::Str(String::new()), ParamUi::Text), direction(), ], ocio_cst, @@ -931,10 +985,14 @@ pub fn specs() -> Vec { "OCIO Display Transform", cc, vec![ - p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + p("config", "Configuration", Value::Enum(0), popup(CONFIGS)), + p("configFile", "Config File (.ocio)", Value::Str(String::new()), ParamUi::Text), space_param("source", "Source", lin709), + p("sourceName", "Source (config name)", Value::Str(String::new()), ParamUi::Text), p("display", "Display", Value::Enum(0), popup(&DISPLAYS.iter().map(|d| d.0).collect::>())), + p("displayName", "Display (config name)", Value::Str(String::new()), ParamUi::Text), p("view", "View", Value::Enum(0), popup(VIEWS)), + p("viewName", "View (config name)", Value::Str(String::new()), ParamUi::Text), direction(), ], ocio_display, diff --git a/crates/effects/src/ocio_config.rs b/crates/effects/src/ocio_config.rs new file mode 100644 index 00000000..2ed7cee8 --- /dev/null +++ b/crates/effects/src/ocio_config.rs @@ -0,0 +1,928 @@ +//! Custom OpenColorIO configuration files (`.ocio`) for the OCIO effects, read from the +//! published configuration format (a YAML document), not from OpenColorIO's source. +//! +//! Supported subset: +//! +//! * `search_path` (a string with `:`-separated entries or a list), resolved against the +//! config's folder, for `FileTransform` sources; +//! * `roles` (any role name can stand for its colour space); +//! * `colorspaces`: `name`, `aliases`, `isdata`, and the transforms to / from the reference +//! space — `to_reference` / `from_reference` (OCIO v1) or `to_scene_reference` / +//! `from_scene_reference` (v2); a space with only one direction is inverted for the other; +//! * `displays`: display → views (`! {name, colorspace}`); +//! * transforms: `MatrixTransform` (`matrix` 4×4 row-major, `offset`), `FileTransform` (the LUT +//! and CDL files [`super::ocio::parse_file`] reads; `interpolation`, `ccc_id`), +//! `ExponentTransform` (`value`), `LogTransform` (`base`), `LogAffineTransform` (`base`, +//! `logSideSlope`, `logSideOffset`, `linSideSlope`, `linSideOffset`), `CDLTransform` (`slope`, +//! `offset`, `power`, `sat`), `RangeTransform` (`min_in_value`, `max_in_value`, +//! `min_out_value`, `max_out_value`) and `GroupTransform` (`children`), each with +//! `direction: inverse`. Other transforms are reported by [`Config::unsupported`] and pass +//! colours through. +//! +//! The YAML reader covers what configs use: block mappings and sequences, flow `{}` / `[]` +//! collections (also spanning lines), `!` tags, quoted scalars, `|` / `>` block scalars +//! and comments. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use super::ocio::{Cdl, FileXform, load_file}; + +// ---------------------------------------------------------------- YAML subset + +/// A parsed YAML node. +#[derive(Clone, Debug, PartialEq)] +pub enum Yaml { + Str(String), + Seq(Vec), + Map(Vec<(String, Yaml)>), + /// A `!` node. + Tagged(String, Box), + Null, +} + +impl Yaml { + pub fn get(&self, k: &str) -> Option<&Yaml> { + match self { + Yaml::Map(m) => m.iter().find(|(key, _)| key == k).map(|(_, v)| v), + Yaml::Tagged(_, v) => v.get(k), + _ => None, + } + } + pub fn str(&self) -> Option<&str> { + match self { + Yaml::Str(s) => Some(s), + Yaml::Tagged(_, v) => v.str(), + _ => None, + } + } + pub fn seq(&self) -> &[Yaml] { + match self { + Yaml::Seq(v) => v, + Yaml::Tagged(_, v) => v.seq(), + _ => &[], + } + } + pub fn tag(&self) -> Option<&str> { + match self { + Yaml::Tagged(t, _) => Some(t), + _ => None, + } + } + fn f(&self) -> Option { + self.str()?.trim().parse().ok() + } + fn nums(&self) -> Vec { + match self { + Yaml::Seq(v) => v.iter().filter_map(Yaml::f).collect(), + Yaml::Str(_) => self.f().into_iter().collect(), + Yaml::Tagged(_, v) => v.nums(), + _ => vec![], + } + } +} + +/// Strip a trailing comment (a `#` outside quotes, at the start or after whitespace). +fn strip_comment(line: &str) -> &str { + let (mut sq, mut dq) = (false, false); + let b = line.as_bytes(); + for (i, &c) in b.iter().enumerate() { + match c { + b'\'' if !dq => sq = !sq, + b'"' if !sq => dq = !dq, + b'#' if !sq && !dq && (i == 0 || b[i - 1].is_ascii_whitespace()) => return &line[..i], + _ => {} + } + } + line +} + +fn unquote(s: &str) -> String { + let t = s.trim(); + if t.len() >= 2 && ((t.starts_with('"') && t.ends_with('"')) || (t.starts_with('\'') && t.ends_with('\''))) { + return t[1..t.len() - 1].replace("\\\"", "\"").replace("''", "'"); + } + t.to_string() +} + +/// Split `key: value` at the first `:` followed by a space or the end (outside quotes/brackets). +fn split_key(s: &str) -> Option<(String, &str)> { + let b = s.as_bytes(); + let (mut sq, mut dq, mut depth) = (false, false, 0i32); + for (i, &c) in b.iter().enumerate() { + match c { + b'\'' if !dq => sq = !sq, + b'"' if !sq => dq = !dq, + b'{' | b'[' if !sq && !dq => depth += 1, + b'}' | b']' if !sq && !dq => depth -= 1, + b':' if !sq && !dq && depth == 0 && (i + 1 == b.len() || b[i + 1] == b' ' || b[i + 1] == b'\t') => { + let k = s[..i].trim(); + if k.is_empty() || k.starts_with('!') || k.starts_with('{') || k.starts_with('[') { + return None; + } + return Some((unquote(k), s[i + 1..].trim())); + } + _ => {} + } + } + None +} + +/// Bracket balance of a flow fragment (outside quotes). +fn balance(s: &str) -> i32 { + let (mut sq, mut dq, mut d) = (false, false, 0); + for c in s.chars() { + match c { + '\'' if !dq => sq = !sq, + '"' if !sq => dq = !dq, + '{' | '[' if !sq && !dq => d += 1, + '}' | ']' if !sq && !dq => d -= 1, + _ => {} + } + } + d +} + +struct Flow<'a> { + s: &'a [u8], + i: usize, +} + +impl Flow<'_> { + fn ws(&mut self) { + while self.i < self.s.len() && (self.s[self.i] as char).is_whitespace() { + self.i += 1; + } + } + fn peek(&self) -> Option { + self.s.get(self.i).copied() + } + fn value(&mut self) -> Yaml { + self.ws(); + match self.peek() { + Some(b'{') => { + self.i += 1; + let mut m = vec![]; + loop { + self.ws(); + match self.peek() { + None => break, + Some(b'}') => { + self.i += 1; + break; + } + Some(b',') => { + self.i += 1; + continue; + } + _ => {} + } + let k = self.scalar(true); + self.ws(); + if self.peek() == Some(b':') { + self.i += 1; + } + let v = self.value(); + m.push((unquote(&k), v)); + } + Yaml::Map(m) + } + Some(b'[') => { + self.i += 1; + let mut v = vec![]; + loop { + self.ws(); + match self.peek() { + None => break, + Some(b']') => { + self.i += 1; + break; + } + Some(b',') => { + self.i += 1; + continue; + } + _ => v.push(self.value()), + } + } + Yaml::Seq(v) + } + Some(b'!') => { + let start = self.i; + while self.i < self.s.len() && !(self.s[self.i] as char).is_whitespace() && self.s[self.i] != b'{' && self.s[self.i] != b'[' { + self.i += 1; + } + let tag = tag_name(std::str::from_utf8(&self.s[start..self.i]).unwrap_or("")); + self.ws(); + let inner = match self.peek() { + Some(b'{') | Some(b'[') => self.value(), + _ => Yaml::Null, + }; + Yaml::Tagged(tag, Box::new(inner)) + } + _ => { + let s = self.scalar(false); + if s.is_empty() { Yaml::Null } else { Yaml::Str(unquote(&s)) } + } + } + } + /// A plain or quoted scalar, ending at `,` `}` `]` (and `:` + space for keys). + fn scalar(&mut self, key: bool) -> String { + self.ws(); + let start = self.i; + if let Some(q @ (b'"' | b'\'')) = self.peek() { + self.i += 1; + while self.i < self.s.len() && self.s[self.i] != q { + self.i += 1; + } + self.i = (self.i + 1).min(self.s.len()); + return String::from_utf8_lossy(&self.s[start..self.i]).into_owned(); + } + while self.i < self.s.len() { + let c = self.s[self.i]; + if c == b',' || c == b'}' || c == b']' { + break; + } + if key && c == b':' { + break; + } + self.i += 1; + } + String::from_utf8_lossy(&self.s[start..self.i]).trim().to_string() + } +} + +fn tag_name(t: &str) -> String { + t.trim().trim_start_matches('!').trim_start_matches('<').trim_end_matches('>').to_string() +} + +fn flow(s: &str) -> Yaml { + Flow { s: s.as_bytes(), i: 0 }.value() +} + +struct Lines { + /// (indent, content without comment), blank lines removed. + v: Vec<(usize, String)>, +} + +impl Lines { + fn new(text: &str) -> Lines { + let v = text + .lines() + .filter(|l| !l.trim_start().starts_with('%') && l.trim() != "---" && l.trim() != "...") + .map(|l| { + let c = strip_comment(l).trim_end(); + (c.len() - c.trim_start().len(), c.trim_start().to_string()) + }) + .filter(|(_, c)| !c.is_empty()) + .collect(); + Lines { v } + } + + /// The value of a `key:` / `- ` remainder `rest` whose children are indented more than + /// `indent` (from line `*i`). + fn value(&self, rest: &str, indent: usize, i: &mut usize) -> Yaml { + let rest = rest.trim(); + if rest.is_empty() { + return match self.v.get(*i) { + // A sequence may sit at the key's own indent. + Some((ind, c)) if *ind > indent || (*ind == indent && (c == "-" || c.starts_with("- "))) => self.block(*ind, i), + _ => Yaml::Null, + }; + } + if rest == "|" || rest == ">" || rest.starts_with("|-") || rest.starts_with(">-") || rest.starts_with("|+") || rest.starts_with(">+") { + let mut out = vec![]; + while let Some((ind, c)) = self.v.get(*i) { + if *ind <= indent { + break; + } + out.push(c.clone()); + *i += 1; + } + return Yaml::Str(out.join(if rest.starts_with('>') { " " } else { "\n" })); + } + if rest.starts_with('!') && !rest.contains('{') && !rest.contains('[') { + let tag = tag_name(rest); + let inner = match self.v.get(*i) { + Some((ind, _)) if *ind > indent => self.block(*ind, i), + _ => Yaml::Null, + }; + return Yaml::Tagged(tag, Box::new(inner)); + } + if rest.starts_with('{') || rest.starts_with('[') || (rest.starts_with('!') && (rest.contains('{') || rest.contains('['))) { + // Flow collections may continue on the next lines. + let mut text = rest.to_string(); + while balance(&text) > 0 && *i < self.v.len() { + text.push(' '); + text.push_str(&self.v[*i].1); + *i += 1; + } + return flow(&text); + } + Yaml::Str(unquote(rest)) + } + + /// A block node whose lines start at `indent`. + fn block(&self, indent: usize, i: &mut usize) -> Yaml { + let Some((_, first)) = self.v.get(*i) else { return Yaml::Null }; + if first == "-" || first.starts_with("- ") { + let mut items = vec![]; + while let Some((ind, c)) = self.v.get(*i) { + if *ind != indent || !(c == "-" || c.starts_with("- ")) { + break; + } + *i += 1; + let rest = c[1..].trim_start().to_string(); + // `- key: value` opens a mapping whose keys are indented to the key. + if split_key(&rest).is_some() && !rest.starts_with('{') && !rest.starts_with('!') { + let inner = indent + 1 + (c.len() - 1 - c[1..].trim_start().len()); + items.push(self.map_from(Some(rest), inner.max(indent + 2), i)); + } else { + items.push(self.value(&rest, indent, i)); + } + } + return Yaml::Seq(items); + } + if split_key(first).is_some() { + return self.map_from(None, indent, i); + } + *i += 1; + Yaml::Str(unquote(first)) + } + + /// A block mapping at `indent`; `first` is an entry already taken from a `- ` line. + fn map_from(&self, first: Option, indent: usize, i: &mut usize) -> Yaml { + let mut m = vec![]; + if let Some(f) = first + && let Some((k, rest)) = split_key(&f) + { + let v = self.value(rest, indent, i); + m.push((k, v)); + } + while let Some((ind, c)) = self.v.get(*i) { + if *ind != indent { + if *ind < indent { + break; + } + // Over-indented stray line: skip. + *i += 1; + continue; + } + let Some((k, rest)) = split_key(c) else { break }; + *i += 1; + let v = self.value(rest, indent, i); + m.push((k, v)); + } + Yaml::Map(m) + } +} + +/// Parse a YAML document (the subset described in the module docs). +pub fn parse_yaml(text: &str) -> Yaml { + let l = Lines::new(text); + let mut i = 0; + let indent = l.v.first().map_or(0, |(ind, _)| *ind); + l.block(indent, &mut i) +} + +// ---------------------------------------------------------------- transforms + +/// A colour transform of a config. +#[derive(Clone, Debug)] +pub enum Xf { + /// 4×4 row-major matrix (RGB of the first three rows/columns) plus offset. + Matrix { + m: [f64; 16], + offset: [f64; 4], + }, + File { + src: String, + xf: Option>, + interp: u32, + ccc: String, + }, + Exponent([f64; 3]), + Log { + base: f64, + }, + LogAffine { + base: f64, + log_slope: [f64; 3], + log_offset: [f64; 3], + lin_slope: [f64; 3], + lin_offset: [f64; 3], + }, + Cdl(Cdl), + Range { + min_in: f64, + max_in: f64, + min_out: f64, + max_out: f64, + }, + Group(Vec), + Inverse(Box), + /// An unsupported transform (by tag): identity. + Unsupported(String), +} + +fn vec3_of(y: Option<&Yaml>, d: f64) -> [f64; 3] { + let v = y.map(Yaml::nums).unwrap_or_default(); + match v.len() { + 0 => [d; 3], + 1 | 2 => [v[0]; 3], + _ => [v[0], v[1], v[2]], + } +} + +fn mat_apply(m: &[f64; 16], off: &[f64; 4], c: [f64; 3]) -> [f64; 3] { + [0, 1, 2].map(|r| m[r * 4] * c[0] + m[r * 4 + 1] * c[1] + m[r * 4 + 2] * c[2] + off[r]) +} + +fn mat3_inverse(m: &[f64; 16]) -> Option<[[f64; 3]; 3]> { + let a = [[m[0], m[1], m[2]], [m[4], m[5], m[6]], [m[8], m[9], m[10]]]; + let det = a[0][0] * (a[1][1] * a[2][2] - a[1][2] * a[2][1]) - a[0][1] * (a[1][0] * a[2][2] - a[1][2] * a[2][0]) + + a[0][2] * (a[1][0] * a[2][1] - a[1][1] * a[2][0]); + (det.abs() > 1e-12).then(|| effectcraft_color::space::invert(&a)) +} + +fn log_base(v: f64, base: f64) -> f64 { + v.max(f64::MIN_POSITIVE).ln() / base.ln() +} + +impl Xf { + fn from_yaml(y: &Yaml, dir: &Path, search: &[String]) -> Xf { + let tag = y.tag().unwrap_or(""); + let body = match y { + Yaml::Tagged(_, b) => b.as_ref(), + other => other, + }; + let inverse = body.get("direction").and_then(Yaml::str).is_some_and(|d| d.eq_ignore_ascii_case("inverse")); + let xf = match tag { + "MatrixTransform" => { + let v = body.get("matrix").map(Yaml::nums).unwrap_or_default(); + let mut m = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0]; + if v.len() == 16 { + m.copy_from_slice(&v); + } else if v.len() == 9 { + for r in 0..3 { + for c in 0..3 { + m[r * 4 + c] = v[r * 3 + c]; + } + } + } + let o = body.get("offset").map(Yaml::nums).unwrap_or_default(); + let mut offset = [0.0; 4]; + for (k, v) in o.iter().take(4).enumerate() { + offset[k] = *v; + } + Xf::Matrix { m, offset } + } + "FileTransform" => { + let src = body.get("src").and_then(Yaml::str).unwrap_or("").to_string(); + let interp = match body.get("interpolation").and_then(Yaml::str).unwrap_or("linear").to_ascii_lowercase().as_str() { + "nearest" => 0, + "tetrahedral" | "best" => 2, + _ => 1, + }; + let ccc = body.get("cccid").or_else(|| body.get("ccc_id")).and_then(Yaml::str).unwrap_or("").to_string(); + let xf = resolve(&src, dir, search).and_then(|p| load_file(&p.to_string_lossy())); + Xf::File { src, xf, interp, ccc } + } + "ExponentTransform" => Xf::Exponent(vec3_of(body.get("value"), 1.0)), + "LogTransform" => Xf::Log { base: body.get("base").and_then(Yaml::f).unwrap_or(2.0) }, + "LogAffineTransform" => Xf::LogAffine { + base: body.get("base").and_then(Yaml::f).unwrap_or(2.0), + log_slope: vec3_of(body.get("logSideSlope"), 1.0), + log_offset: vec3_of(body.get("logSideOffset"), 0.0), + lin_slope: vec3_of(body.get("linSideSlope"), 1.0), + lin_offset: vec3_of(body.get("linSideOffset"), 0.0), + }, + "CDLTransform" => Xf::Cdl(Cdl { + slope: vec3_of(body.get("slope"), 1.0), + offset: vec3_of(body.get("offset"), 0.0), + power: vec3_of(body.get("power"), 1.0), + sat: body.get("sat").and_then(Yaml::f).unwrap_or(1.0), + }), + "RangeTransform" => Xf::Range { + min_in: body.get("min_in_value").and_then(Yaml::f).unwrap_or(0.0), + max_in: body.get("max_in_value").and_then(Yaml::f).unwrap_or(1.0), + min_out: body.get("min_out_value").and_then(Yaml::f).unwrap_or(0.0), + max_out: body.get("max_out_value").and_then(Yaml::f).unwrap_or(1.0), + }, + "GroupTransform" => Xf::Group(body.get("children").map(|c| c.seq().iter().map(|t| Xf::from_yaml(t, dir, search)).collect()).unwrap_or_default()), + other => Xf::Unsupported(other.to_string()), + }; + if inverse { Xf::Inverse(Box::new(xf)) } else { xf } + } + + /// Apply forward (or inverse) to linear-or-encoded RGB. + pub fn apply(&self, c: [f64; 3], inverse: bool) -> [f64; 3] { + match self { + Xf::Inverse(x) => x.apply(c, !inverse), + Xf::Matrix { m, offset } => { + if !inverse { + return mat_apply(m, offset, c); + } + match mat3_inverse(m) { + Some(inv) => { + let d = [c[0] - offset[0], c[1] - offset[1], c[2] - offset[2]]; + effectcraft_color::space::mul_vec(&inv, d) + } + None => c, + } + } + Xf::File { xf, interp, ccc, .. } => match xf { + Some(x) => x.apply(c.map(|v| v as f32), *interp, inverse, ccc).map(|v| v as f64), + None => c, + }, + Xf::Exponent(e) => [0, 1, 2].map(|k| { + let p = if inverse { 1.0 / e[k].max(1e-9) } else { e[k] }; + c[k].max(0.0).powf(p) + }), + Xf::Log { base } => { + if inverse { + c.map(|v| base.powf(v)) + } else { + c.map(|v| log_base(v, *base)) + } + } + Xf::LogAffine { base, log_slope, log_offset, lin_slope, lin_offset } => [0, 1, 2].map(|k| { + if inverse { + (base.powf((c[k] - log_offset[k]) / log_slope[k]) - lin_offset[k]) / lin_slope[k] + } else { + log_slope[k] * log_base(lin_slope[k] * c[k] + lin_offset[k], *base) + log_offset[k] + } + }), + Xf::Cdl(cdl) => { + if inverse { + cdl.invert(c, true) + } else { + cdl.apply(c, true) + } + } + Xf::Range { min_in, max_in, min_out, max_out } => { + let (a0, a1, b0, b1) = if inverse { (*min_out, *max_out, *min_in, *max_in) } else { (*min_in, *max_in, *min_out, *max_out) }; + let k = if (a1 - a0).abs() > 1e-12 { (b1 - b0) / (a1 - a0) } else { 0.0 }; + c.map(|v| b0 + (v - a0) * k) + } + Xf::Group(list) => { + let mut v = c; + if inverse { + for x in list.iter().rev() { + v = x.apply(v, true); + } + } else { + for x in list { + v = x.apply(v, false); + } + } + v + } + Xf::Unsupported(_) => c, + } + } + + fn unsupported(&self, out: &mut Vec) { + match self { + Xf::Unsupported(t) => out.push(t.clone()), + Xf::Inverse(x) => x.unsupported(out), + Xf::Group(v) => v.iter().for_each(|x| x.unsupported(out)), + Xf::File { src, xf: None, .. } => out.push(format!("FileTransform (missing `{src}`)")), + _ => {} + } + } +} + +/// Find a FileTransform source on the search path (relative to the config's folder). +fn resolve(src: &str, dir: &Path, search: &[String]) -> Option { + let p = Path::new(src); + if p.is_absolute() { + return p.exists().then(|| p.to_path_buf()); + } + let mut dirs: Vec = search.iter().map(|s| dir.join(s)).collect(); + dirs.push(dir.to_path_buf()); + dirs.into_iter().map(|d| d.join(src)).find(|c| c.exists()) +} + +// ---------------------------------------------------------------- the config + +/// A colour space of a config. +#[derive(Clone, Debug)] +pub struct ConfigSpace { + pub name: String, + pub aliases: Vec, + pub family: String, + pub is_data: bool, + pub to_ref: Option, + pub from_ref: Option, +} + +impl ConfigSpace { + fn to_reference(&self, c: [f64; 3]) -> [f64; 3] { + match (&self.to_ref, &self.from_ref) { + (Some(x), _) => x.apply(c, false), + (None, Some(x)) => x.apply(c, true), + _ => c, + } + } + fn reference_to(&self, c: [f64; 3]) -> [f64; 3] { + match (&self.from_ref, &self.to_ref) { + (Some(x), _) => x.apply(c, false), + (None, Some(x)) => x.apply(c, true), + _ => c, + } + } +} + +/// A parsed `.ocio` configuration. +#[derive(Clone, Debug, Default)] +pub struct Config { + pub name: String, + pub search_path: Vec, + pub roles: Vec<(String, String)>, + pub spaces: Vec, + /// Display → [(view, colour space)]. + pub displays: Vec<(String, Vec<(String, String)>)>, +} + +impl Config { + /// Parse config text; FileTransform sources resolve against `dir`. + pub fn parse(text: &str, dir: &Path) -> Result { + let y = parse_yaml(text); + if !matches!(y, Yaml::Map(_)) { + return Err("not an OCIO config (expected a YAML mapping)".into()); + } + let search_path: Vec = match y.get("search_path") { + Some(Yaml::Seq(v)) => v.iter().filter_map(|s| s.str().map(str::to_string)).collect(), + Some(s) => s.str().map(|s| s.split(':').map(|p| p.trim().to_string()).filter(|p| !p.is_empty()).collect()).unwrap_or_default(), + None => vec![], + }; + let roles = match y.get("roles") { + Some(Yaml::Map(m)) => m.iter().filter_map(|(k, v)| v.str().map(|s| (k.clone(), s.to_string()))).collect(), + _ => vec![], + }; + let mut spaces = vec![]; + for list in ["colorspaces", "display_colorspaces"] { + for cs in y.get(list).map(Yaml::seq).unwrap_or(&[]) { + let Some(name) = cs.get("name").and_then(Yaml::str) else { continue }; + let xf = |keys: &[&str]| keys.iter().find_map(|k| cs.get(k)).filter(|v| !matches!(v, Yaml::Null)).map(|v| Xf::from_yaml(v, dir, &search_path)); + spaces.push(ConfigSpace { + name: name.to_string(), + aliases: cs.get("aliases").map(|a| a.seq().iter().filter_map(|s| s.str().map(str::to_string)).collect()).unwrap_or_default(), + family: cs.get("family").and_then(Yaml::str).unwrap_or("").to_string(), + is_data: cs.get("isdata").and_then(Yaml::str).is_some_and(|v| v == "true"), + to_ref: xf(&["to_reference", "to_scene_reference", "to_display_reference"]), + from_ref: xf(&["from_reference", "from_scene_reference", "from_display_reference"]), + }); + } + } + if spaces.is_empty() { + return Err("the config has no colorspaces".into()); + } + let displays = match y.get("displays") { + Some(Yaml::Map(m)) => m + .iter() + .map(|(d, views)| { + let v = views.seq().iter().filter_map(|v| Some((v.get("name")?.str()?.to_string(), v.get("colorspace")?.str()?.to_string()))).collect(); + (d.clone(), v) + }) + .collect(), + _ => vec![], + }; + Ok(Config { name: y.get("name").and_then(Yaml::str).unwrap_or("").to_string(), search_path, roles, spaces, displays }) + } + + pub fn space_names(&self) -> Vec<&str> { + self.spaces.iter().map(|s| s.name.as_str()).collect() + } + + /// A colour space by name, alias or role (case-insensitive). + pub fn space(&self, name: &str) -> Option<&ConfigSpace> { + let n = name.trim(); + let by = |n: &str| self.spaces.iter().find(|s| s.name.eq_ignore_ascii_case(n) || s.aliases.iter().any(|a| a.eq_ignore_ascii_case(n))); + by(n).or_else(|| self.roles.iter().find(|(r, _)| r.eq_ignore_ascii_case(n)).and_then(|(_, s)| by(s))) + } + + /// The colour space of a display's view. + pub fn view_space(&self, display: &str, view: &str) -> Option<&ConfigSpace> { + let (_, views) = self.displays.iter().find(|(d, _)| d.eq_ignore_ascii_case(display.trim())).or(self.displays.first())?; + let (_, cs) = views.iter().find(|(v, _)| v.eq_ignore_ascii_case(view.trim())).or(views.first())?; + self.space(cs) + } + + /// Convert one colour from space `a` to space `b` (through the reference space; data + /// spaces pass through). + pub fn convert(&self, a: &ConfigSpace, b: &ConfigSpace, c: [f64; 3]) -> [f64; 3] { + if a.is_data || b.is_data || a.name == b.name { + return c; + } + b.reference_to(a.to_reference(c)) + } + + /// Transforms this module can't apply (by tag), for warnings. + pub fn unsupported(&self) -> Vec { + let mut out = vec![]; + for s in &self.spaces { + for x in [&s.to_ref, &s.from_ref].into_iter().flatten() { + x.unsupported(&mut out); + } + } + out.sort(); + out.dedup(); + out + } +} + +type ConfigCache = Mutex>>>; + +/// Load (and cache) a config from a `.ocio` path or the config text itself. +pub fn load_config(src: &str) -> Option> { + let t = src.trim(); + if t.is_empty() { + return None; + } + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Ok(m) = cache.lock() + && let Some(v) = m.get(src) + { + return v.clone(); + } + let parsed = if src.contains('\n') { + Config::parse(src, Path::new(".")) + } else { + let p = Path::new(t); + std::fs::read_to_string(p).map_err(|e| e.to_string()).and_then(|text| Config::parse(&text, p.parent().unwrap_or(Path::new(".")))) + } + .ok() + .map(Arc::new); + if let Ok(mut m) = cache.lock() { + m.insert(src.to_string(), parsed.clone()); + } + parsed +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONFIG: &str = r#" +ocio_profile_version: 2 + +name: Test Config # a comment +search_path: "luts:other" +roles: + scene_linear: lin + color_timing: "log2" + +displays: + sRGB: + - ! {name: Standard, colorspace: gamma22} + - ! {name: Raw, colorspace: raw} + +colorspaces: + - ! + name: lin + family: "" + description: | + The reference. + Linear. + isdata: false + + - ! + name: gamma22 + aliases: [g22, "gamma 2.2"] + to_scene_reference: ! {value: [2.2, 2.2, 2.2, 1]} + + - ! + name: half + from_reference: ! {matrix: [0.5, 0, 0, 0, + 0, 0.5, 0, 0, + 0, 0, 0.5, 0, + 0, 0, 0, 1], offset: [0.1, 0.1, 0.1, 0]} + + - ! + name: log2 + from_reference: ! + children: + - ! {matrix: [2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1]} + - ! {base: 2} + + - ! + name: affine + to_reference: ! {base: 10, logSideSlope: 0.5, logSideOffset: 0.2, linSideSlope: 2, linSideOffset: 0.01, direction: inverse} + + - ! + name: lut + to_reference: ! {src: double.cube, interpolation: linear} + + - ! + name: raw + isdata: true + + - ! + name: weird + to_reference: ! {exposure: 1} +"#; + + fn close(a: [f64; 3], b: [f64; 3], tol: f64) -> bool { + (0..3).all(|i| (a[i] - b[i]).abs() < tol) + } + + #[test] + fn yaml_subset() { + let y = parse_yaml("a: 1\nb:\n - x\n - {k: [1, 2], s: 'q: r'}\nc: ! {v: 3}\nd:\n- 4\n- 5\n"); + assert_eq!(y.get("a").and_then(Yaml::str), Some("1")); + assert_eq!(y.get("b").unwrap().seq().len(), 2); + assert_eq!(y.get("b").unwrap().seq()[1].get("k").unwrap().nums(), vec![1.0, 2.0]); + assert_eq!(y.get("b").unwrap().seq()[1].get("s").and_then(Yaml::str), Some("q: r")); + assert_eq!(y.get("c").unwrap().tag(), Some("T")); + assert_eq!(y.get("c").unwrap().get("v").and_then(Yaml::str), Some("3")); + assert_eq!(y.get("d").unwrap().seq().len(), 2, "a sequence at the key's indent"); + } + + #[test] + fn parses_spaces_roles_displays_and_converts() { + let dir = std::env::temp_dir().join(format!("ec-ocio-{}", std::process::id())); + std::fs::create_dir_all(dir.join("luts")).unwrap(); + // A 1D LUT that doubles (0..1 → 0..2). + std::fs::write(dir.join("luts/double.cube"), "LUT_1D_SIZE 2\nDOMAIN_MIN 0 0 0\nDOMAIN_MAX 1 1 1\n0 0 0\n2 2 2\n").unwrap(); + let c = Config::parse(CONFIG, &dir).unwrap(); + assert_eq!(c.name, "Test Config"); + assert_eq!(c.search_path, vec!["luts", "other"]); + assert_eq!(c.space_names(), vec!["lin", "gamma22", "half", "log2", "affine", "lut", "raw", "weird"]); + // Roles and aliases resolve. + assert_eq!(c.space("scene_linear").unwrap().name, "lin"); + assert_eq!(c.space("G22").unwrap().name, "gamma22"); + assert_eq!(c.space("gamma 2.2").unwrap().name, "gamma22"); + assert_eq!(c.view_space("sRGB", "Standard").unwrap().name, "gamma22"); + let sp = |n: &str| c.space(n).unwrap(); + // gamma22 → lin: decode with the exponent. + let g = c.convert(sp("gamma22"), sp("lin"), [0.5, 0.25, 1.0]); + assert!(close(g, [0.5f64.powf(2.2), 0.25f64.powf(2.2), 1.0], 1e-9), "{g:?}"); + // lin → gamma22: the inverse (only to_reference given). + let back = c.convert(sp("lin"), sp("gamma22"), g); + assert!(close(back, [0.5, 0.25, 1.0], 1e-9), "{back:?}"); + // lin → half: matrix and offset; and back through the inverse matrix. + let h = c.convert(sp("lin"), sp("half"), [0.4, 0.6, 1.0]); + assert!(close(h, [0.3, 0.4, 0.6], 1e-9), "{h:?}"); + assert!(close(c.convert(sp("half"), sp("lin"), h), [0.4, 0.6, 1.0], 1e-9)); + // lin → log2: a group (×2 then log2): 0.5 → 0; 2 → 2. + let l = c.convert(sp("lin"), sp("log2"), [0.5, 2.0, 4.0]); + assert!(close(l, [0.0, 2.0, 3.0], 1e-9), "{l:?}"); + assert!(close(c.convert(sp("log2"), sp("lin"), l), [0.5, 2.0, 4.0], 1e-9)); + // LogAffine (inverse direction as to_reference) round-trips. + let a = c.convert(sp("lin"), sp("affine"), [0.18, 0.5, 1.0]); + let expect = [0.18f64, 0.5, 1.0].map(|v| 0.5 * (2.0 * v + 0.01).log10() + 0.2); + assert!(close(a, expect, 1e-9), "{a:?} vs {expect:?}"); + assert!(close(c.convert(sp("affine"), sp("lin"), a), [0.18, 0.5, 1.0], 1e-9)); + // FileTransform found on the search path. + let f = c.convert(sp("lut"), sp("lin"), [0.25, 0.5, 0.0]); + assert!(close(f, [0.5, 1.0, 0.0], 1e-6), "{f:?}"); + // Data spaces pass through; unsupported transforms are reported. + assert_eq!(c.convert(sp("raw"), sp("gamma22"), [0.3, 0.3, 0.3]), [0.3, 0.3, 0.3]); + assert_eq!(c.unsupported(), vec!["ExposureContrastTransform".to_string()]); + // Loading from a file path (cached), and errors. + std::fs::write(dir.join("config.ocio"), CONFIG).unwrap(); + let loaded = load_config(&dir.join("config.ocio").to_string_lossy()).unwrap(); + assert_eq!(loaded.spaces.len(), 8); + assert!(load_config("/no/such/config.ocio").is_none()); + assert!(Config::parse("just: text\n", &dir).is_err()); + } + /// The OCIO Color Space and Display Transform effects with Configuration ▸ Custom. + #[test] + fn effects_use_a_custom_config() { + use effectcraft_keyframe::Value; + let cfg = "ocio_profile_version: 1\nroles:\n default: lin\ndisplays:\n Monitor:\n - ! {name: Video, colorspace: g2}\ncolorspaces:\n - !\n name: lin\n - !\n name: g2\n to_reference: ! {value: [2, 2, 2, 1]}\n"; + let img = crate::Image::filled(2, 2, [0.5, 0.25, 1.0, 1.0]); + let fx = |id: &str, extra: &[(&str, Value)]| { + let mut v = vec![("config", Value::Enum(1)), ("configFile", Value::Str(cfg.into()))]; + v.extend(extra.iter().cloned()); + crate::run_fx(id, &v, img.clone(), 0.0, crate::EffectEnv::default()).img.data[0] + }; + let o = fx("ec.color.ociocolorspace", &[("sourceName", Value::Str("g2".into())), ("destinationName", Value::Str("default".into()))]); + assert!((o[0] - 0.25).abs() < 1e-6 && (o[1] - 0.0625).abs() < 1e-6 && (o[2] - 1.0).abs() < 1e-6, "{o:?}"); + // Inverse direction. + let o = fx( + "ec.color.ociocolorspace", + &[("sourceName", Value::Str("g2".into())), ("destinationName", Value::Str("lin".into())), ("direction", Value::Enum(1))], + ); + assert!((o[0] - 0.5f32.sqrt()).abs() < 1e-6, "{o:?}"); + // Display ▸ view → its colour space. + let o = fx( + "ec.color.ociodisplay", + &[("sourceName", Value::Str("lin".into())), ("displayName", Value::Str("Monitor".into())), ("viewName", Value::Str("Video".into()))], + ); + assert!((o[0] - 0.5f32.sqrt()).abs() < 1e-6, "{o:?}"); + // An unreadable config leaves pixels alone. + let o = crate::run_fx( + "ec.color.ociocolorspace", + &[("config", Value::Enum(1)), ("configFile", Value::Str("/missing.ocio".into()))], + img.clone(), + 0.0, + crate::EffectEnv::default(), + ); + assert_eq!(o.img.data, img.data); + } +} diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 48285a33..58e89598 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -258,15 +258,52 @@ fn project_settings(s: &mut Session, p: &Value) -> Result { if let Some(l) = p.get("blendLinear").or_else(|| p.get("blendColorsUsing1Gamma")).and_then(Value::as_bool) { proj.settings.blend_linear = l; } + use effectcraft_project::{ColorEngine, ColorSpace, HdrMode}; + let cmd = "file.projectSettings"; + if let Some(e) = str_p(p, "colorEngine") { + let engine = match e.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "adobe" | "adobemanaged" | "builtin" => ColorEngine::Adobe, + "ocio" | "ociomanaged" => ColorEngine::Ocio, + _ => return Err(bad(cmd, format!("colorEngine: adobe|ocio, not `{e}`"))), + }; + if engine == ColorEngine::Ocio && proj.settings.color_engine != ColorEngine::Ocio { + // The OCIO built-in config works in ACES and renders the display through its + // tone-mapped view. + if !proj.settings.working_space.is_some_and(|w| w.is_linear()) { + proj.settings.working_space = Some(ColorSpace::AcesCg); + } + if proj.settings.hdr == HdrMode::Clip { + proj.settings.hdr = HdrMode::ToneMap; + } + } + proj.settings.color_engine = engine; + } if let Some(w) = p.get("workingSpace") { + let ids = ColorSpace::WORKING.iter().map(|c| c.id()).collect::>().join("|"); proj.settings.working_space = match w.as_str() { None | Some("none" | "None" | "") => None, Some(n) => Some( - effectcraft_project::ColorSpace::parse(n) - .ok_or_else(|| bad("file.projectSettings", format!("workingSpace: none|srgb|rec709|rec2020|p3, not `{n}`")))?, + ColorSpace::parse(n).filter(|c| ColorSpace::WORKING.contains(c)).ok_or_else(|| bad(cmd, format!("workingSpace: none|{ids}, not `{n}`")))?, ), }; } + if proj.settings.color_engine == ColorEngine::Ocio && !proj.settings.working_space.is_some_and(|w| w.is_linear()) { + return Err(bad(cmd, "the OCIO built-in config's working spaces are acescg and aces2065")); + } + if let Some(h) = str_p(p, "hdr") { + proj.settings.hdr = match h.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "clip" | "off" | "none" => HdrMode::Clip, + "compand" => HdrMode::Compand, + "tonemap" | "tonemapped" => HdrMode::ToneMap, + _ => return Err(bad(cmd, format!("hdr: clip|compand|toneMap, not `{h}`"))), + }; + } + if let Some(o) = p.get("outputSpace") { + proj.settings.output_space = match o.as_str() { + None | Some("none" | "None" | "" | "default") => None, + Some(n) => Some(ColorSpace::parse(n).ok_or_else(|| bad(cmd, format!("outputSpace: srgb|rec709|rec2020|p3|rec2100pq|rec2100hlg, not `{n}`")))?), + }; + } if let Some(r) = p.get("renderer").or_else(|| p.get("gpuAcceleration")) { proj.settings.gpu_acceleration = match r { Value::Bool(b) => *b, @@ -279,8 +316,8 @@ fn project_settings(s: &mut Session, p: &Value) -> Result { }; } if let Some(t) = str_p(p, "timeDisplay") { - proj.settings.time_display = - if t.eq_ignore_ascii_case("frames") { effectcraft_project::TimeDisplayStyle::Frames } else { effectcraft_project::TimeDisplayStyle::Timecode }; + proj.settings.time_display = effectcraft_project::TimeDisplayStyle::parse(t) + .ok_or_else(|| bad("file.projectSettings", format!("timeDisplay: timecode|frames|feet35|feet16, not `{t}`")))?; } Ok(()) })?; @@ -353,7 +390,7 @@ pub fn specs() -> Vec { "Project Settings...", ["File"], Some("Cmd+Alt+Shift+K"), - "{bitDepth?: 8|16|32, workingSpace?: none|srgb|rec709|rec2020|p3, linearize?, blendLinear?, renderer?: gpu|software, timeDisplay?: timecode|frames}", + "{bitDepth?: 8|16|32, colorEngine?: adobe|ocio, workingSpace?: none|srgb|rec709|rec2020|p3|acescg|aces2065, linearize?, blendLinear?, hdr?: clip|compand|toneMap, outputSpace?: srgb|rec709|rec2020|p3|rec2100pq|rec2100hlg, renderer?: gpu|software, timeDisplay?: timecode|frames|feet35|feet16}", always, project_settings ), diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4d2adea4..63909f63 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -44,7 +44,7 @@ mod styles; mod text_anim; pub mod text_edit; mod three_d; -mod time; +pub mod time; mod track; mod view; pub mod viewer_cmds; diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index c2683e60..24fea4e7 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -29,11 +29,6 @@ pub fn specs() -> Vec { stub!("camera.linkFocusToPoi", "Link Focus Distance to Point of Interest", ["Layer", "Camera"], "{}"), stub!("camera.linkFocusToLayer", "Link Focus Distance to Layer", ["Layer", "Camera"], "{}"), stub!("camera.setFocusToLayer", "Set Focus Distance to Layer", ["Layer", "Camera"], "{}"), - stub!("view.3d.default", "Default", ["View", "Switch 3D View"], "{}"), - stub!("view.splitLockedViewer", "Split with New Locked Viewer", ["View"], "{}"), - // Color management. - stub!("view.displayColorManagement", "Use Display Color Management", ["View"], "{}"), - stub!("view.simulateOutput", "Simulate Output", [], "{profile}"), // Keyframes / text / tracking. stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), diff --git a/crates/engine/src/commands/three_d.rs b/crates/engine/src/commands/three_d.rs index 55060d17..c9ce4f4d 100644 --- a/crates/engine/src/commands/three_d.rs +++ b/crates/engine/src/commands/three_d.rs @@ -297,7 +297,7 @@ fn views(s: &mut Session, cid: ItemId) -> &mut Views3D { } fn view_p(p: &Value, cmd: &str) -> Result { - let v = str_p(p, "view").ok_or_else(|| bad(cmd, "missing `view` (activeCamera|front|left|top|back|right|bottom|custom1|custom2|custom3)"))?; + let v = str_p(p, "view").ok_or_else(|| bad(cmd, "missing `view` (activeCamera|default|front|left|top|back|right|bottom|custom1|custom2|custom3)"))?; View3D::from_id(v).ok_or_else(|| bad(cmd, format!("unknown view `{v}`"))) } @@ -334,6 +334,7 @@ view_cmd!(view_bottom, View3D::Bottom); view_cmd!(view_custom1, View3D::Custom1); view_cmd!(view_custom2, View3D::Custom2); view_cmd!(view_custom3, View3D::Custom3); +view_cmd!(view_default, View3D::Default); fn last_view(s: &mut Session, p: &Value) -> Result { let cid = comp_id(s, p)?; @@ -697,11 +698,12 @@ pub fn specs() -> Vec { "Switch 3D View", [], None, - "{view: activeCamera|front|left|top|back|right|bottom|custom1|custom2|custom3, comp?}", + "{view: activeCamera|default|front|left|top|back|right|bottom|custom1|custom2|custom3, comp?}", has_comp, set_3d_view ), cmd!("view.3d.activeCamera", "Active Camera", ["View", "Switch 3D View"], Some("F12"), "{comp?}", has_comp, view_active), + cmd!("view.3d.default", "Default", ["View", "Switch 3D View"], None, "{comp?}", has_comp, view_default), cmd!("view.3d.front", "Front", ["View", "Switch 3D View"], Some("F10"), "{comp?}", has_comp, view_front), cmd!("view.3d.left", "Left", ["View", "Switch 3D View"], None, "{comp?}", has_comp, view_left), cmd!("view.3d.top", "Top", ["View", "Switch 3D View"], None, "{comp?}", has_comp, view_top), diff --git a/crates/engine/src/commands/time.rs b/crates/engine/src/commands/time.rs index d2971b71..0b6d19bc 100644 --- a/crates/engine/src/commands/time.rs +++ b/crates/engine/src/commands/time.rs @@ -1,5 +1,6 @@ //! Time navigation (CTI) commands. +use effectcraft_project::{Comp, TimeDisplayStyle}; use effectcraft_time::Tick; use serde_json::{Value, json}; @@ -9,7 +10,36 @@ use crate::{EngineError, Result, Session, cmd}; fn report(s: &Session) -> Value { let t = s.time(); let frame = s.active_comp().map(|c| c.frame_rate.frame_at(t)).unwrap_or(0); - json!({"time": t.seconds(), "frame": frame}) + let display = s.active_comp().map(|c| display_time(s, c, t)); + json!({"time": t.seconds(), "frame": frame, "display": display}) +} + +/// A comp time as the project's Time Display Style shows it: timecode (`0:00:02:15`, `;` for +/// drop-frame), frames (`00063`, from Start Numbering Frames At) or Feet + Frames (`0003+15`). +pub fn display_time(s: &Session, comp: &Comp, t: Tick) -> String { + let fr = comp.frame_rate; + let st = &s.project.settings; + match st.time_display { + TimeDisplayStyle::Frames => format!("{:05}", fr.frame_at(t) + st.frame_start), + TimeDisplayStyle::Feet35 | TimeDisplayStyle::Feet16 => { + effectcraft_time::format_feet_frames(fr.frame_at(t + comp.display_start) + st.frame_start, st.time_display.frames_per_foot().unwrap_or(16)) + } + TimeDisplayStyle::Timecode => effectcraft_time::format_timecode_ae(fr.frame_at(t + comp.display_start), fr, false), + } +} + +/// Parse a time typed in the project's Time Display Style (relative `+n`/`-n` from frame +/// `current`) into a comp frame. +pub fn parse_display_time(s: &Session, comp: &Comp, text: &str, current: i64) -> std::result::Result { + let fr = comp.frame_rate; + let st = &s.project.settings; + match st.time_display.frames_per_foot() { + Some(pf) => { + let start = fr.frame_at(comp.display_start) + st.frame_start; + effectcraft_time::parse_feet_frames(text, pf, current + start).map(|f| f - start).map_err(|e| e.0) + } + None => effectcraft_time::parse_timecode(text, fr, fr.supports_drop_frame(), current).map_err(|e| e.0), + } } fn set(s: &mut Session, p: &Value) -> Result { @@ -18,7 +48,7 @@ fn set(s: &mut Session, p: &Value) -> Result { comp.frame_rate.tick_of(f) } else if let Some(tc) = str_p(p, "timecode") { let cur = comp.frame_rate.frame_at(s.time()); - let f = effectcraft_time::parse_timecode(tc, comp.frame_rate, comp.frame_rate.supports_drop_frame(), cur).map_err(|e| super::bad("time.set", e.0))?; + let f = parse_display_time(s, comp, tc, cur).map_err(|e| super::bad("time.set", e))?; comp.frame_rate.tick_of(f) } else { Tick::from_seconds_f64(f_p(p, "time").ok_or_else(|| super::bad("time.set", "need `time`, `frame` or `timecode`"))?) diff --git a/crates/engine/src/commands/viewer_cmds.rs b/crates/engine/src/commands/viewer_cmds.rs index c0161510..9bc2d99c 100644 --- a/crates/engine/src/commands/viewer_cmds.rs +++ b/crates/engine/src/commands/viewer_cmds.rs @@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use super::{CommandSpec, always, b_p, bad, comp_id, f_p, has_comp, merge_p, str_p}; -use crate::viewer::{Channel, Snapshot}; -use crate::{EngineError, Result, Session, VertexRef, cmd}; +use crate::viewer::{Channel, SimProfile, Simulation, Snapshot}; +use crate::{EngineError, Result, Session, VertexRef, cmd, query}; /// Composition ▸ Preview ▸ Fast Previews modes. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -61,8 +61,8 @@ impl FastPreviews { } } -/// Viewer display options (Show Channel, exposure, snapshot, Fast Previews). -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +/// Viewer display options (Show Channel, exposure, snapshot, Fast Previews, display colour). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct ViewOptions { pub channel: Channel, @@ -73,6 +73,36 @@ pub struct ViewOptions { /// Show Snapshot (F5 held): the viewer shows the snapshot instead of the frame. pub show_snapshot: bool, pub fast_previews: FastPreviews, + /// View ▸ Use Display Color Management (with a working space): the frame is converted + /// from the working space to the display; off shows the working space's numbers. + pub display_color_management: bool, + /// View ▸ Simulate Output. + pub simulation: Simulation, + /// The profile View ▸ Simulate Output ▸ Custom... uses. + pub custom_simulation: Simulation, +} + +impl Default for ViewOptions { + fn default() -> Self { + ViewOptions { + channel: Channel::default(), + colorized: false, + exposure: 0.0, + show_snapshot: false, + fast_previews: FastPreviews::default(), + display_color_management: true, + simulation: Simulation::default(), + custom_simulation: Simulation { profile: SimProfile::Linear, preserve_rgb: true }, + } + } +} + +/// View ▸ Split with New Locked Viewer: a second Composition viewer that keeps showing one +/// comp and 3D view whatever comp is active. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct LockedViewer { + pub comp: effectcraft_project::ItemId, + pub view: effectcraft_render::three_d::View3D, } fn snapping(s: &mut Session, p: &Value) -> Result { @@ -281,9 +311,97 @@ fn shape_layer_or_none(s: &Session) -> std::result::Result<(), String> { has_comp(s) } +// ---------------------------------------------------------------- display colour, locked viewer + +/// View ▸ Use Display Color Management (toggle, or `value`). +fn display_cm(s: &mut Session, p: &Value) -> Result { + let v = b_p(p, "value").unwrap_or(!s.state.viewer.display_color_management); + s.state.viewer.display_color_management = v; + s.events.push(crate::Event::ProjectChanged { revision: s.revision }); + Ok(json!({"displayColorManagement": v, "workingSpace": s.project.settings.working_space.map(|w| w.id()), "display": s.prefs.previews.display_profile})) +} + +/// View ▸ Simulate Output ▸ (profile). `custom` uses (and with `space` sets) the Custom profile. +fn simulate_output(s: &mut Session, p: &Value) -> Result { + let cmd = "view.simulateOutput"; + let name = str_p(p, "profile").unwrap_or("none"); + let ids = SimProfile::ALL.iter().map(|p| p.id()).collect::>().join("|"); + let sim = if name.eq_ignore_ascii_case("custom") { + let mut c = s.state.viewer.custom_simulation; + if let Some(sp) = str_p(p, "space") { + c.profile = SimProfile::parse(sp).ok_or_else(|| bad(cmd, format!("space: {ids}")))?; + } + if let Some(v) = b_p(p, "preserveRgb") { + c.preserve_rgb = v; + } + s.state.viewer.custom_simulation = c; + c + } else { + let profile = SimProfile::parse(name).ok_or_else(|| bad(cmd, format!("profile: {ids}|custom")))?; + Simulation { profile, preserve_rgb: b_p(p, "preserveRgb").unwrap_or(false) } + }; + s.state.viewer.simulation = sim; + s.events.push(crate::Event::ProjectChanged { revision: s.revision }); + Ok(json!({"profile": sim.profile.id(), "label": sim.profile.label(), "preserveRgb": sim.preserve_rgb})) +} + +/// View ▸ Split with New Locked Viewer: lock a second viewer to the comp (default the active +/// one) and its current 3D view (or `view`). +fn split_locked(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let view = match str_p(p, "view") { + Some(v) => effectcraft_render::three_d::View3D::from_id(v).ok_or_else(|| bad("view.splitLockedViewer", format!("unknown view `{v}`")))?, + None => s.state.views3d.get(&cid).map(|v| v.current).unwrap_or_default(), + }; + s.state.locked_viewer = Some(LockedViewer { comp: cid, view }); + s.events.push(crate::Event::ProjectChanged { revision: s.revision }); + Ok(json!({"comp": cid.0, "view": view.id()})) +} + +fn close_locked(s: &mut Session, _: &Value) -> Result { + let was = s.state.locked_viewer.take().is_some(); + s.events.push(crate::Event::ProjectChanged { revision: s.revision }); + Ok(json!({"closed": was})) +} + +fn has_locked(s: &Session) -> std::result::Result<(), String> { + if s.state.locked_viewer.is_some() { Ok(()) } else { Err("no locked viewer".into()) } +} + +/// The viewer's colour settings and what they do. +fn display_color_state(s: &mut Session, _: &Value) -> Result { + let v = &s.state.viewer; + Ok(json!({ + "displayColorManagement": v.display_color_management, + "simulation": {"profile": v.simulation.profile.id(), "preserveRgb": v.simulation.preserve_rgb}, + "display": s.prefs.previews.display_profile, + "workingSpace": s.project.settings.working_space.map(|w| w.id()), + "active": crate::viewer::DisplayColor::of(s).is_some(), + "lockedViewer": s.state.locked_viewer.map(|l| json!({"comp": l.comp.0, "view": l.view.id()})), + })) +} + pub fn specs() -> Vec { vec![ cmd!("view.snapping", "Snapping", ["View"], None, "{value?}", always, snapping), + cmd!("view.displayColorManagement", "Use Display Color Management", ["View"], None, "{value?}", always, display_cm), + cmd!( + "view.simulateOutput", + "Simulate Output", + [], + None, + "{profile: none|rec709|ntsc|pal|mac18|srgb|rec2020|p3|linear|custom, preserveRgb?, space? (custom)}", + always, + simulate_output + ), + cmd!("view.splitLockedViewer", "Split with New Locked Viewer", ["View"], None, "{comp?, view?: 3D view id}", has_comp, split_locked), + cmd!("view.closeLockedViewer", "Close Locked Viewer", [], None, "{}", has_locked, close_locked), + query!( + "view.displayColor", + "Viewer Color State", + "{} → display colour management, output simulation, display profile, locked viewer", + display_color_state + ), cmd!("view.channel", "Show Channel", [], None, "{channel: rgb|red|green|blue|alpha|rgbStraight, colorized?, toggle?}", always, channel), cmd!("view.exposure", "Adjust Exposure", [], None, "{stops? | delta?}", always, exposure), cmd!("view.resetExposure", "Reset Exposure", [], None, "{}", always, reset_exposure), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c9d931d7..3fc74b1e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -215,6 +215,9 @@ pub struct EditorState { /// Essential Graphics can expose). #[serde(default)] pub essential_solo: bool, + /// View ▸ Split with New Locked Viewer: the second viewer's comp and 3D view. + #[serde(default)] + pub locked_viewer: Option, } fn one_view() -> u8 { @@ -651,6 +654,8 @@ mod tests_anim_tools; #[cfg(test)] mod tests_camera_track; #[cfg(test)] +mod tests_color_view; +#[cfg(test)] mod tests_disk_cache; #[cfg(test)] mod tests_effects; diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 66d19894..b1041aaf 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -168,6 +168,17 @@ pub fn checked(s: &Session, command: &str, params: &Value) -> Option { "view.layout" => Some(params.get("views").and_then(Value::as_u64) == Some(s.state.view_layout.max(1) as u64)), "view.shareViewOptions" => Some(s.state.share_view_options), "view.snapping" => Some(s.state.snapping), + "view.displayColorManagement" => Some(s.state.viewer.display_color_management), + "view.simulateOutput" => { + let want = pstr("profile")?; + let sim = s.state.viewer.simulation; + Some(if want == "custom" { + sim.profile != crate::viewer::SimProfile::None && sim == s.state.viewer.custom_simulation + } else { + crate::viewer::SimProfile::parse(want) == Some(sim.profile) + && (sim != s.state.viewer.custom_simulation || sim.profile == crate::viewer::SimProfile::None) + }) + } "layer.mask.motionBlur" | "layer.mask.featherFalloff" | "path.rotoBezier" => { use effectcraft_project::{FeatherFalloff, GroupKind, MaskMotionBlur}; let GroupKind::Mask { motion_blur, feather_falloff, roto_bezier, .. } = layer?.masks()?.groups().next()?.kind else { return None }; @@ -853,6 +864,9 @@ View SDTV PAL | view.simulateOutput {"profile":"pal"} Legacy Macintosh RGB (Gamma 1.8) | view.simulateOutput {"profile":"mac18"} Internet Standard RGB (sRGB) | view.simulateOutput {"profile":"srgb"} + UHDTV (Rec. 2020) | view.simulateOutput {"profile":"rec2020"} + Display P3 | view.simulateOutput {"profile":"p3"} + Linear (1.0 Gamma) | view.simulateOutput {"profile":"linear"} Custom... | view.simulateOutput {"profile":"custom"} --- Show Rulers | view.rulers diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index 5a2afab8..2d744226 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -104,6 +104,8 @@ page!(Previews { /// `faster` or `moreAccurate`. zoom_quality: String = "moreAccurate".into(), show_gpu_info: bool = false, + /// The monitor's colour space for View ▸ Use Display Color Management: `srgb` or `p3`. + display_profile: String = "srgb".into(), }); page!(Appearance { @@ -728,6 +730,7 @@ pub fn pages() -> Vec { s("previews.cacheFramesWhenIdle", "Cache Frames When Idle", B, true), s("previews.fastPreviews", "Fast Previews (Draft 3D, Faster Effects)", B, true), s("previews.zoomQuality", "Viewer Zoom Quality", Kind::Choice(&[("Faster", "faster"), ("More Accurate", "moreAccurate")]), false), + s("previews.displayProfile", "Display Color Space", Kind::Choice(&[("sRGB", "srgb"), ("Display P3", "p3")]), true), Button { label: "GPU Information...", command: "app.gpuInfo", params: "{}" }, ], }, diff --git a/crates/engine/src/tests_color_view.rs b/crates/engine/src/tests_color_view.rs new file mode 100644 index 00000000..316d2e5a --- /dev/null +++ b/crates/engine/src/tests_color_view.rs @@ -0,0 +1,223 @@ +//! Viewer and colour completion (M7.7): View ▸ Switch 3D View ▸ Default, Split with New Locked +//! Viewer, Use Display Color Management, Simulate Output; Project Settings ▸ Color Engine, +//! ACES working spaces, HDR compand / tone mapping, Rec. 2100 output, Feet + Frames. + +use effectcraft_project::{ColorEngine, ColorSpace, HdrMode, Project, TimeDisplayStyle}; +use effectcraft_render::RenderOpts; +use effectcraft_render::three_d::View3D; +use effectcraft_time::Tick; +use serde_json::json; + +use crate::Session; +use crate::viewer::{DisplayColor, SimProfile, Simulation}; + +fn session() -> Session { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "C", "width": 64, "height": 48, "frameRate": 24, "duration": 10})).unwrap(); + s +} + +#[test] +fn default_3d_view_ignores_camera_layers() { + let mut s = session(); + let cid = s.active_comp_id().unwrap(); + s.execute("layer.newCamera", json!({"position": [32, 24, -300], "poi": [32, 24, 0]})).unwrap(); + assert_eq!(s.execute("view.3d.default", json!({})).unwrap()["view"], "default"); + let vs = s.state.views3d.get(&cid).unwrap().clone(); + assert_eq!(vs.current, View3D::Default); + // The comp's default camera: at −zoom looking at the centre, not the camera layer. + let cam = s.view_camera(cid).unwrap(); + let zoom = effectcraft_geom::default_camera_zoom(64.0); + assert!((cam.eye.z + zoom).abs() < 1e-9 && (cam.eye.x - 32.0).abs() < 1e-9 && !cam.ortho, "{:?}", cam.eye); + // Orbitable like a custom view; Reset 3D View restores it; Last view goes back. + s.execute("camera.orbit", json!({"yaw": 30})).unwrap(); + assert!((s.view_camera(cid).unwrap().eye.x - 32.0).abs() > 1.0); + s.execute("view.reset3DView", json!({})).unwrap(); + assert!((s.view_camera(cid).unwrap().eye.x - 32.0).abs() < 1e-9); + s.execute("view.3d.last", json!({})).unwrap(); + assert_eq!(s.state.views3d[&cid].current, View3D::ActiveCamera); + assert_eq!(s.execute("view.set3DView", json!({"view": "Default"})).unwrap()["view"], "default"); +} + +#[test] +fn locked_viewer_keeps_its_comp() { + let mut s = session(); + let a = s.active_comp_id().unwrap(); + s.execute("view.3d.top", json!({})).unwrap(); + assert!(!s.is_enabled("view.closeLockedViewer")); + let r = s.execute("view.splitLockedViewer", json!({})).unwrap(); + assert_eq!(r, json!({"comp": a.0, "view": "top"})); + // Another comp becomes active; the locked viewer stays on the first. + s.execute("comp.new", json!({"name": "B"})).unwrap(); + assert_ne!(s.active_comp_id(), Some(a)); + assert_eq!(s.state.locked_viewer.unwrap().comp, a); + let st = s.execute("view.displayColor", json!({})).unwrap(); + assert_eq!(st["lockedViewer"]["comp"], a.0); + // Serde of the editor state; explicit comp and view. + let back: crate::EditorState = serde_json::from_value(serde_json::to_value(&s.state).unwrap()).unwrap(); + assert_eq!(back.locked_viewer, s.state.locked_viewer); + s.execute("view.splitLockedViewer", json!({"comp": a.0, "view": "custom2"})).unwrap(); + assert_eq!(s.state.locked_viewer.unwrap().view, View3D::Custom2); + assert!(s.execute("view.splitLockedViewer", json!({"view": "sideways"})).is_err()); + assert_eq!(s.execute("view.closeLockedViewer", json!({})).unwrap()["closed"], true); + assert!(s.state.locked_viewer.is_none()); +} + +fn px(r: u8, g: u8, b: u8) -> Vec<[u8; 4]> { + vec![[r, g, b, 255]] +} + +#[test] +fn display_color_management_and_simulation() { + let mut s = session(); + // Unmanaged projects: nothing to convert. + assert!(DisplayColor::of(&s).is_none()); + s.execute("file.projectSettings", json!({"workingSpace": "rec2020"})).unwrap(); + // Managed, sRGB display, no simulation: the frame is already display-referred. + assert!(DisplayColor::of(&s).is_none()); + assert_eq!(crate::menus::checked(&s, "view.displayColorManagement", &json!({})), Some(true)); + // A Display P3 monitor: pure sRGB red becomes less saturated P3 numbers. + s.execute("prefs.set", json!({"key": "previews.displayProfile", "value": "p3"})).unwrap(); + let mut p = px(255, 0, 0); + DisplayColor::of(&s).unwrap().apply(&mut p); + assert!((p[0][0] as i32 - 234).abs() <= 2 && (p[0][1] as i32 - 51).abs() <= 3 && (p[0][2] as i32 - 35).abs() <= 3, "{p:?}"); + // Display colour management off: the working space's (Rec. 2020) numbers go to the screen. + s.execute("view.displayColorManagement", json!({})).unwrap(); + assert_eq!(crate::menus::checked(&s, "view.displayColorManagement", &json!({})), Some(false)); + let mut q = px(255, 0, 0); + DisplayColor::of(&s).unwrap().apply(&mut q); + assert!(q[0][0] < 230 && q[0][1] > 60, "raw Rec. 2020 numbers: {q:?}"); + s.execute("view.displayColorManagement", json!({"value": true})).unwrap(); + s.execute("prefs.set", json!({"key": "previews.displayProfile", "value": "srgb"})).unwrap(); + // Simulate Output ▸ Linear with Preserve RGB: linear numbers shown as-is (darker mid-grey). + s.execute("view.simulateOutput", json!({"profile": "linear", "preserveRgb": true})).unwrap(); + let mut g = px(128, 128, 128); + DisplayColor::of(&s).unwrap().apply(&mut g); + assert!((g[0][0] as i32 - 55).abs() <= 2, "{g:?}"); + // Without Preserve RGB the round trip through 8-bit linear bands the shadows. + s.execute("view.simulateOutput", json!({"profile": "linear"})).unwrap(); + let dc = DisplayColor::of(&s).unwrap(); + let levels: std::collections::BTreeSet = (0..32u8) + .map(|v| { + let mut p = px(v, v, v); + dc.apply(&mut p); + p[0][0] + }) + .collect(); + assert!(levels.len() < 20, "banding: {levels:?}"); + assert_eq!(crate::menus::checked(&s, "view.simulateOutput", &json!({"profile": "linear"})), Some(true)); + // Rec. 709 simulation keeps in-gamut colours; legacy Mac gamma 1.8 with Preserve RGB darkens. + s.execute("view.simulateOutput", json!({"profile": "rec709"})).unwrap(); + let mut c = px(200, 100, 50); + DisplayColor::of(&s).unwrap().apply(&mut c); + assert!((c[0][0] as i32 - 200).abs() <= 2 && (c[0][1] as i32 - 100).abs() <= 2, "{c:?}"); + s.execute("view.simulateOutput", json!({"profile": "mac18", "preserveRgb": true})).unwrap(); + let mut m = px(128, 128, 128); + DisplayColor::of(&s).unwrap().apply(&mut m); + assert!(m[0][0] < 120, "gamma 1.8 numbers on an sRGB display look darker: {m:?}"); + // Custom… uses (and sets) the custom profile. + let r = s.execute("view.simulateOutput", json!({"profile": "custom", "space": "p3", "preserveRgb": false})).unwrap(); + assert_eq!(r["profile"], "p3"); + assert_eq!(s.state.viewer.custom_simulation, Simulation { profile: SimProfile::P3, preserve_rgb: false }); + assert_eq!(crate::menus::checked(&s, "view.simulateOutput", &json!({"profile": "custom"})), Some(true)); + s.execute("view.simulateOutput", json!({"profile": "none"})).unwrap(); + assert!(DisplayColor::of(&s).is_none()); + assert!(s.execute("view.simulateOutput", json!({"profile": "cmyk"})).is_err()); + // Every menu profile parses. + for p in SimProfile::ALL { + assert_eq!(SimProfile::parse(p.id()), Some(p)); + } +} + +fn red_solid(s: &mut Session, hex: &str) { + s.execute("layer.newSolid", json!({"color": hex, "width": 64, "height": 48})).unwrap(); +} + +#[test] +fn color_engine_aces_working_spaces_and_hdr() { + let mut s = session(); + let cid = s.active_comp_id().unwrap(); + // OCIO engine: ACEScg working space and the tone-mapped display by default. + s.execute("file.projectSettings", json!({"colorEngine": "ocio"})).unwrap(); + let st = s.project.settings.clone(); + assert_eq!((st.color_engine, st.working_space, st.hdr), (ColorEngine::Ocio, Some(ColorSpace::AcesCg), HdrMode::ToneMap)); + assert!(s.execute("file.projectSettings", json!({"workingSpace": "rec709"})).is_err(), "not in the OCIO config"); + s.execute("file.projectSettings", json!({"workingSpace": "aces2065"})).unwrap(); + assert!(s.execute("file.projectSettings", json!({"workingSpace": "rec2100pq"})).is_err(), "not a working space"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.project.settings.working_space, Some(ColorSpace::AcesCg)); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.project.settings.color_engine, ColorEngine::Adobe); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(s.project.settings.color_engine, ColorEngine::Ocio); + // 32 bpc, a solid of 4.0 (over-range) in ACEScg: tone mapped below white, not clipped. + s.execute("file.projectSettings", json!({"bitDepth": 32, "hdr": "toneMap"})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#ffffff", "width": 64, "height": 48})).unwrap(); + let lid = s.active_comp().unwrap().layers[0].id.0; + s.execute("effect.apply", json!({"layers": [lid], "effect": "Exposure"})).unwrap(); + s.execute("prop.set", json!({"layer": lid, "path": "effects/#1/exposure", "value": 2.0})).unwrap(); + let mapped = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + assert!(mapped[0] > 0.5 && mapped[0] < 0.99, "tone mapped: {mapped:?}"); + s.execute("file.projectSettings", json!({"hdr": "clip"})).unwrap(); + let clipped = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + assert!(clipped[0] > 1.1, "32 bpc keeps over-range: {clipped:?}"); + s.execute("file.projectSettings", json!({"hdr": "compand"})).unwrap(); + let comp = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + assert!(comp[0] > 0.9 && comp[0] < 1.0, "companded: {comp:?}"); + // Rec. 2100 PQ output: HDR, no tone mapping: over-range white encodes above SDR white's 0.58. + s.execute("file.projectSettings", json!({"outputSpace": "rec2100pq", "hdr": "toneMap"})).unwrap(); + let pq = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + assert!(pq[0] > 0.6 && pq[0] < 0.8, "PQ: {pq:?}"); + s.execute("file.projectSettings", json!({"outputSpace": "rec2100hlg"})).unwrap(); + let hlg = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + assert!(hlg[0] > 0.85, "HLG: {hlg:?}"); + assert!(s.execute("file.projectSettings", json!({"hdr": "glow"})).is_err()); + // Serde keeps the new settings; older files load with the defaults. + let back: Project = serde_json::from_str(&serde_json::to_string(&*s.project).unwrap()).unwrap(); + assert_eq!(back.settings, s.project.settings); + let mut v = serde_json::to_value(&s.project.settings).unwrap(); + for k in ["color_engine", "hdr", "output_space"] { + v.as_object_mut().unwrap().remove(k); + } + let old: effectcraft_project::ProjectSettings = serde_json::from_value(v).unwrap(); + assert_eq!((old.color_engine, old.hdr, old.output_space), (ColorEngine::Adobe, HdrMode::Clip, None)); +} + +#[test] +fn aces_working_space_matches_the_ocio_config() { + // ACEScg in the colour module and in the OCIO effects' built-in config agree. + let mut s = session(); + s.execute("file.projectSettings", json!({"workingSpace": "acescg"})).unwrap(); + red_solid(&mut s, "#808080"); + let cid = s.active_comp_id().unwrap(); + // Authored ACEScg values (0.5 linear AP1 grey) → sRGB display. + let out = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; + let want = effectcraft_color::linear_to_srgb(128.0 / 255.0); + assert!((out[0] - want).abs() < 0.01 && (out[1] - want).abs() < 0.01, "{out:?} vs {want}"); +} + +#[test] +fn feet_and_frames_display_and_entry() { + let mut s = session(); + let comp = s.active_comp().unwrap().clone(); + s.execute("file.projectSettings", json!({"timeDisplay": "feet35"})).unwrap(); + assert_eq!(s.project.settings.time_display, TimeDisplayStyle::Feet35); + let t = comp.frame_rate.tick_of(16 * 3 + 5); + assert_eq!(crate::commands::time::display_time(&s, &comp, t), "0003+05"); + // Typed feet+frames in the time field. + let r = s.execute("time.set", json!({"timecode": "2+10"})).unwrap(); + assert_eq!(r["frame"], 42); + assert_eq!(r["display"], "0002+10"); + let r = s.execute("time.set", json!({"timecode": "+6"})).unwrap(); + assert_eq!(r["frame"], 48); + s.execute("file.projectSettings", json!({"timeDisplay": "feet16"})).unwrap(); + assert_eq!(crate::commands::time::display_time(&s, &comp, comp.frame_rate.tick_of(95)), "0002+15"); + s.execute("file.projectSettings", json!({"timeDisplay": "frames"})).unwrap(); + assert_eq!(crate::commands::time::display_time(&s, &comp, comp.frame_rate.tick_of(95)), "00095"); + s.execute("file.projectSettings", json!({"timeDisplay": "timecode"})).unwrap(); + assert_eq!(crate::commands::time::display_time(&s, &comp, comp.frame_rate.tick_of(48)), "0:00:02:00"); + assert!(s.execute("file.projectSettings", json!({"timeDisplay": "furlongs"})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.project.settings.time_display, TimeDisplayStyle::Frames); + assert_eq!(serde_json::to_value(TimeDisplayStyle::Feet16).unwrap(), json!("Feet16")); +} diff --git a/crates/engine/src/viewer.rs b/crates/engine/src/viewer.rs index 0c002fa1..bba10122 100644 --- a/crates/engine/src/viewer.rs +++ b/crates/engine/src/viewer.rs @@ -5,6 +5,7 @@ use std::sync::Arc; +use effectcraft_color::{ColorSpace, space}; use effectcraft_geom::vec2; use effectcraft_project::{Layer, LayerId}; use effectcraft_raster::Image; @@ -359,6 +360,273 @@ pub fn display_transform(px: &mut [[u8; 4]], channel: Channel, colorized: bool, } } +// ---------------------------------------------------------------- display colour management + +/// View ▸ Simulate Output: the output device a preview imitates. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SimProfile { + #[default] + None, + /// HDTV (Rec. 709): ITU-R BT.709 primaries, BT.1886 gamma 2.4. + Rec709, + /// UHDTV (Rec. 2020): ITU-R BT.2020 primaries, gamma 2.4. + Rec2020, + /// Display P3: SMPTE EG 432-1 primaries, D65, the sRGB curve. + P3, + /// Internet Standard RGB (sRGB, IEC 61966-2-1). + Srgb, + /// Linear light with Rec. 709 primaries (no transfer curve). + Linear, + /// SDTV NTSC: SMPTE 170M primaries, gamma 2.4. + Ntsc, + /// SDTV PAL: EBU Tech 3213 primaries, gamma 2.4. + Pal, + /// Legacy Macintosh RGB: Rec. 709 primaries, gamma 1.8. + Mac18, +} + +/// A transfer curve of a simulated or display profile. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Curve { + Srgb, + Gamma(f32), + Linear, + /// A colour space's own curve (PQ, HLG…). + Space(space::Curve), +} + +impl Curve { + fn decode(self, v: f32) -> f32 { + let a = v.abs(); + let l = match self { + Curve::Srgb => srgb_to_linear(a), + Curve::Gamma(g) => a.powf(g), + Curve::Linear => a, + Curve::Space(c) => c.decode(a), + }; + l.copysign(v) + } + fn encode(self, v: f32) -> f32 { + let a = v.abs(); + let e = match self { + Curve::Srgb => linear_to_srgb(a), + Curve::Gamma(g) => a.powf(1.0 / g), + Curve::Linear => a, + Curve::Space(c) => c.encode(a), + }; + e.copysign(v) + } +} + +const P709: [[f64; 2]; 3] = [[0.640, 0.330], [0.300, 0.600], [0.150, 0.060]]; + +impl SimProfile { + pub const ALL: [SimProfile; 9] = [ + SimProfile::None, + SimProfile::Rec709, + SimProfile::Ntsc, + SimProfile::Pal, + SimProfile::Mac18, + SimProfile::Srgb, + SimProfile::Rec2020, + SimProfile::P3, + SimProfile::Linear, + ]; + + pub fn id(self) -> &'static str { + match self { + SimProfile::None => "none", + SimProfile::Rec709 => "rec709", + SimProfile::Rec2020 => "rec2020", + SimProfile::P3 => "p3", + SimProfile::Srgb => "srgb", + SimProfile::Linear => "linear", + SimProfile::Ntsc => "ntsc", + SimProfile::Pal => "pal", + SimProfile::Mac18 => "mac18", + } + } + + pub fn label(self) -> &'static str { + match self { + SimProfile::None => "No Output Simulation", + SimProfile::Rec709 => "HDTV (Rec. 709)", + SimProfile::Rec2020 => "UHDTV (Rec. 2020)", + SimProfile::P3 => "Display P3", + SimProfile::Srgb => "Internet Standard RGB (sRGB)", + SimProfile::Linear => "Linear (1.0 Gamma)", + SimProfile::Ntsc => "SDTV NTSC", + SimProfile::Pal => "SDTV PAL", + SimProfile::Mac18 => "Legacy Macintosh RGB (Gamma 1.8)", + } + } + + pub fn parse(s: &str) -> Option { + let k: String = s.chars().filter(|c| c.is_ascii_alphanumeric()).collect::().to_ascii_lowercase(); + SimProfile::ALL + .into_iter() + .find(|p| p.id() == k || p.label().chars().filter(|c| c.is_ascii_alphanumeric()).collect::().to_ascii_lowercase() == k) + .or(match k.as_str() { + "off" => Some(SimProfile::None), + "hdtv" | "bt709" => Some(SimProfile::Rec709), + "bt2020" | "uhdtv" => Some(SimProfile::Rec2020), + "displayp3" => Some(SimProfile::P3), + "lin" | "linearlight" => Some(SimProfile::Linear), + _ => None, + }) + } + + /// (primaries, curve); `None` for No Output Simulation. + pub fn space(self) -> Option<([[f64; 2]; 3], Curve)> { + Some(match self { + SimProfile::None => return None, + SimProfile::Rec709 => (P709, Curve::Gamma(2.4)), + SimProfile::Rec2020 => (ColorSpace::Rec2020.primaries(), Curve::Gamma(2.4)), + SimProfile::P3 => (ColorSpace::DisplayP3.primaries(), Curve::Srgb), + SimProfile::Srgb => (P709, Curve::Srgb), + SimProfile::Linear => (P709, Curve::Linear), + SimProfile::Ntsc => ([[0.630, 0.340], [0.310, 0.595], [0.155, 0.070]], Curve::Gamma(2.4)), + SimProfile::Pal => ([[0.640, 0.330], [0.290, 0.600], [0.150, 0.060]], Curve::Gamma(2.4)), + SimProfile::Mac18 => (P709, Curve::Gamma(1.8)), + }) + } +} + +/// View ▸ Simulate Output setting: the profile, and Preserve RGB (the output numbers are sent +/// to the display unconverted, as on a device that doesn't colour-manage). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(default)] +pub struct Simulation { + pub profile: SimProfile, + pub preserve_rgb: bool, +} + +/// An RGB space for viewer conversions: primaries (D65) and curve. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Prof { + /// Linear RGB → XYZ D65. + xyz: [[f64; 3]; 3], + curve: Curve, +} + +impl Prof { + fn new(prims: [[f64; 2]; 3], curve: Curve) -> Prof { + Prof { xyz: space::rgb_to_xyz(prims, D65), curve } + } + /// A colour space (linear when `linear`). + fn of(cs: ColorSpace, linear: bool) -> Prof { + let curve = if linear { Curve::Linear } else { Curve::Space(cs.curve()) }; + Prof { xyz: cs.to_xyz(), curve } + } +} + +/// A linear-RGB matrix plus curves between two profiles. +#[derive(Clone, Copy, Debug)] +struct Conv { + from: Curve, + m: Option<[[f32; 3]; 3]>, + to: Curve, +} + +impl Conv { + fn new(a: Prof, b: Prof) -> Conv { + let m = (a.xyz != b.xyz).then(|| { + let m = space::mul(&space::invert(&b.xyz), &a.xyz); + m.map(|r| r.map(|v| v as f32)) + }); + Conv { from: a.curve, m, to: b.curve } + } + fn apply(&self, c: [f32; 3]) -> [f32; 3] { + let l = c.map(|v| self.from.decode(v)); + let l = match &self.m { + Some(m) => [0, 1, 2].map(|i| m[i][0] * l[0] + m[i][1] * l[1] + m[i][2] * l[2]), + None => l, + }; + l.map(|v| self.to.encode(v)) + } +} + +const D65: [f64; 2] = [0.3127, 0.3290]; + +/// The viewer's colour conversion from the rendered frame (sRGB-encoded display pixels) to +/// what the monitor shows (View ▸ Use Display Color Management, View ▸ Simulate Output and +/// the display profile in Settings ▸ Previews). `None` when nothing changes. +#[derive(Clone, Copy, Debug)] +pub struct DisplayColor { + sim: Option<(Conv, Option)>, + display: Option, +} + +impl DisplayColor { + /// `working`: the project working space (None = unmanaged: nothing is converted, as in + /// After Effects); `linear`: the working space is linear; `source`: the space rendered + /// frames are in (the project's output space, sRGB by default); `dcm`: Use Display Color + /// Management; `display`: the monitor's space. + pub fn new(working: Option, linear: bool, source: ColorSpace, dcm: bool, display: ColorSpace, sim: Simulation) -> Option { + let ws = working?; + let src = Prof::of(source, false); + if !dcm { + // Without display colour management the working space's numbers go to the screen. + let raw = Conv::new(src, Prof::of(ws, linear)); + return Some(DisplayColor { sim: None, display: Some(raw) }); + } + let sim = sim.profile.space().map(|(prims, curve)| { + let out = Prof::new(prims, curve); + ( + Conv::new(src, out), + Some(if sim.preserve_rgb { + Conv::new(Prof::new(P709, Curve::Srgb), Prof::of(display, false)) + } else { + Conv::new(out, Prof::of(display, false)) + }), + ) + }); + let display = (display != source).then(|| Conv::new(src, Prof::of(display, false))); + (sim.is_some() || display.is_some()).then_some(DisplayColor { sim, display }) + } + + /// The conversion a session's viewer uses. + pub fn of(s: &crate::Session) -> Option { + let st = &s.project.settings; + let display = ColorSpace::parse(&s.prefs.previews.display_profile).unwrap_or(ColorSpace::Srgb); + let v = &s.state.viewer; + let linear = st.working_space.is_some_and(|w| st.linearize || w.is_linear()); + DisplayColor::new(st.working_space, linear, st.output_space.unwrap_or(ColorSpace::Srgb), v.display_color_management, display, v.simulation) + } + + /// One straight colour (0..1 floats). + pub fn apply_straight(&self, c: [f32; 3]) -> [f32; 3] { + let c = match &self.sim { + // Through the output's 8-bit encoding (clipped and quantised like the real output), + // then shown on the display: converted from the output profile, or — Preserve RGB — + // the output numbers taken as if they were sRGB. + Some((to, show)) => { + let o = to.apply(c).map(|v| (v.clamp(0.0, 1.0) * 255.0).round() / 255.0); + show.map_or(o, |b| b.apply(o)) + } + None => self.display.map_or(c, |d| d.apply(c)), + }; + c.map(|v| v.clamp(0.0, 1.0)) + } + + /// Premultiplied 8-bit RGBA pixels in place. + pub fn apply(&self, px: &mut [[u8; 4]]) { + use rayon::prelude::*; + px.par_iter_mut().for_each(|p| { + let a = p[3]; + if a == 0 { + return; + } + let af = a as f32 / 255.0; + let c = [0, 1, 2].map(|i| (p[i] as f32 / 255.0 / af).min(1.0)); + let o = self.apply_straight(c); + for i in 0..3 { + p[i] = (o[i] * af * 255.0 + 0.5) as u8; + } + }); + } +} + // ---------------------------------------------------------------- snapshot /// Take Snapshot (Shift+F5): a rendered frame kept for comparison (Show Snapshot, F5). diff --git a/crates/gpu/src/ops.rs b/crates/gpu/src/ops.rs index 775e86c1..f8248f99 100644 --- a/crates/gpu/src/ops.rs +++ b/crates/gpu/src/ops.rs @@ -144,11 +144,13 @@ pub fn quantize(e: &mut Enc, img: &GpuImage, levels: f32) -> GpuImage { } fn curve_id(s: Option) -> u32 { - use effectcraft_color::ColorSpace::*; - match s { - None => 0, - Some(Srgb | DisplayP3) => 1, - Some(Rec709 | Rec2020) => 2, + use effectcraft_color::space::Curve; + match s.map(|s| s.curve()) { + None | Some(Curve::Linear) => 0, + Some(Curve::Srgb) => 1, + Some(Curve::Gamma24) => 2, + Some(Curve::Pq) => 3, + Some(Curve::Hlg) => 4, } } diff --git a/crates/gpu/src/shaders/common.wgsl b/crates/gpu/src/shaders/common.wgsl index dee86b16..2949c0dc 100644 --- a/crates/gpu/src/shaders/common.wgsl +++ b/crates/gpu/src/shaders/common.wgsl @@ -428,7 +428,44 @@ fn hsl_to_rgb(h: f32, s: f32, l: f32) -> vec3 { return vec3(hsl_channel(h + 1.0 / 3.0, p, q), hsl_channel(h, p, q), hsl_channel(h - 1.0 / 3.0, p, q)); } -// ColorSpace transfer curves: 1 = sRGB (sRGB, Display P3), 2 = 2.4 power (Rec. 709 / 2020). +// ColorSpace transfer curves: 1 = sRGB (sRGB, Display P3), 2 = 2.4 power (Rec. 709 / 2020), +// 3 = SMPTE ST 2084 PQ and 4 = BT.2100 HLG (linear 1.0 = the BT.2408 reference white). +const PQ_M1: f32 = 0.1593017578125; +const PQ_M2: f32 = 78.84375; +const PQ_C1: f32 = 0.8359375; +const PQ_C2: f32 = 18.8515625; +const PQ_C3: f32 = 18.6875; +const PQ_WHITE: f32 = 0.0203; +const HLG_A: f32 = 0.17883277; +const HLG_B: f32 = 0.28466892; +const HLG_C: f32 = 0.55991073; +const HLG_REF: f32 = 0.26496256; + +fn pq_decode(e: f32) -> f32 { + let p = pow(clamp(e, 0.0, 1.0), 1.0 / PQ_M2); + return pow(max(p - PQ_C1, 0.0) / (PQ_C2 - PQ_C3 * p), 1.0 / PQ_M1) / PQ_WHITE; +} + +fn pq_encode(l: f32) -> f32 { + let y = pow(max(l * PQ_WHITE, 0.0), PQ_M1); + return pow((PQ_C1 + PQ_C2 * y) / (1.0 + PQ_C3 * y), PQ_M2); +} + +fn hlg_decode(v: f32) -> f32 { + if (v <= 0.5) { + return v * v / 3.0 / HLG_REF; + } + return (exp((v - HLG_C) / HLG_A) + HLG_B) / 12.0 / HLG_REF; +} + +fn hlg_encode(l: f32) -> f32 { + let e = max(l * HLG_REF, 0.0); + if (e <= 1.0 / 12.0) { + return sqrt(3.0 * e); + } + return HLG_A * log(12.0 * e - HLG_B) + HLG_C; +} + fn curve_decode(curve: u32, v: f32) -> f32 { let a = abs(v); var l = a; @@ -436,6 +473,10 @@ fn curve_decode(curve: u32, v: f32) -> f32 { l = srgb_to_linear(a); } else if (curve == 2u) { l = powz(a, 2.4); + } else if (curve == 3u) { + l = pq_decode(a); + } else if (curve == 4u) { + l = hlg_decode(a); } return select(l, -l, v < 0.0); } @@ -447,6 +488,10 @@ fn curve_encode(curve: u32, v: f32) -> f32 { e = linear_to_srgb(a); } else if (curve == 2u) { e = powz(a, 1.0 / 2.4); + } else if (curve == 3u) { + e = pq_encode(a); + } else if (curve == 4u) { + e = hlg_encode(a); } return select(e, -e, v < 0.0); } diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index ea56df7a..6051b4aa 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -48,6 +48,14 @@ pub(crate) fn render<'g>(e: &mut Enc<'g>, r: &Renderer, comp_id: ItemId, t: Tick canvas = ops::quantize(e, &canvas, l); } } + if r.depth() == 0 + && let Some((lin, mode, enc)) = pipe.output_hdr() + { + on_cpu(e, &mut canvas, |img| { + effectcraft_render::color::output_hdr(img, lin, mode, enc); + pipe.quantize(img); + })?; + } Some(canvas) } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index fb7f7208..018c6abd 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -95,6 +95,62 @@ pub enum TimeDisplayStyle { #[default] Timecode, Frames, + /// Feet + Frames, 35 mm film (16 frames per foot). + Feet35, + /// Feet + Frames, 16 mm film (40 frames per foot). + Feet16, +} + +impl TimeDisplayStyle { + /// Frames per foot of the Feet + Frames styles. + pub fn frames_per_foot(self) -> Option { + match self { + TimeDisplayStyle::Feet35 => Some(16), + TimeDisplayStyle::Feet16 => Some(40), + _ => None, + } + } + pub fn id(self) -> &'static str { + match self { + TimeDisplayStyle::Timecode => "timecode", + TimeDisplayStyle::Frames => "frames", + TimeDisplayStyle::Feet35 => "feet35", + TimeDisplayStyle::Feet16 => "feet16", + } + } + pub fn parse(s: &str) -> Option { + let k: String = s.chars().filter(|c| c.is_ascii_alphanumeric()).collect::().to_ascii_lowercase(); + match k.as_str() { + "timecode" => Some(TimeDisplayStyle::Timecode), + "frames" => Some(TimeDisplayStyle::Frames), + "feet35" | "feetframes" | "feetandframes" | "feet35mm" | "35mm" | "feetframes35mm" => Some(TimeDisplayStyle::Feet35), + "feet16" | "feet16mm" | "16mm" | "feetframes16mm" => Some(TimeDisplayStyle::Feet16), + _ => None, + } + } +} + +/// Project Settings ▸ Color ▸ Color Engine. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorEngine { + /// Built-in colour management (ICC-style working spaces). + #[default] + Adobe, + /// OCIO managed, with the built-in configuration (ACES working spaces). + Ocio, +} + +/// Project Settings ▸ Color ▸ HDR: how over-range (HDR) values reach a standard-dynamic-range +/// display or output. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum HdrMode { + /// Values above 1.0 clip. + #[default] + Clip, + /// Highlights above 80% are companded (an exponential knee) into the remaining range. + Compand, + /// Extended Reinhard tone mapping of luminance (white at 4.0). + ToneMap, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -118,6 +174,16 @@ pub struct ProjectSettings { /// when a GPU adapter exists) or Mercury Software Only (`false`, the CPU compositor). #[serde(default = "yes")] pub gpu_acceleration: bool, + /// Color Engine (Adobe-style built-in or OCIO with ACES working spaces). + #[serde(default)] + pub color_engine: ColorEngine, + /// HDR handling for standard-dynamic-range output (display and renders). + #[serde(default)] + pub hdr: HdrMode, + /// The space the top-level comp is output in (renders and the viewer's source). `None` = + /// sRGB; Rec. 2100 PQ / HLG give HDR output. + #[serde(default)] + pub output_space: Option, } impl Default for ProjectSettings { @@ -131,6 +197,9 @@ impl Default for ProjectSettings { frame_start: 0, audio_sample_rate: 48_000, gpu_acceleration: true, + color_engine: ColorEngine::Adobe, + hdr: HdrMode::Clip, + output_space: None, } } } diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index dbb65cb6..8b8f0e97 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -17,7 +17,7 @@ //! pixels to their parent). Without a working space the sRGB curve is used. use effectcraft_color::{ColorSpace, Conversion}; -use effectcraft_project::ProjectSettings; +use effectcraft_project::{HdrMode, ProjectSettings}; use effectcraft_raster::Image; use rayon::prelude::*; @@ -32,12 +32,24 @@ pub struct Pipe { pub linear: bool, /// Blend in linear light (and the working space is not already linear). pub linear_blend: bool, + /// The top-level output's space (sRGB unless Project Settings choose another). + pub out: ColorSpace, + /// HDR handling for standard-dynamic-range output. + pub hdr: HdrMode, } impl Pipe { pub fn of(s: &ProjectSettings) -> Pipe { - let linear = s.linearize && s.working_space.is_some(); - Pipe { levels: s.bit_depth.levels(), space: s.working_space, linear, linear_blend: s.blend_linear && !linear } + // ACES working spaces are linear light by definition. + let linear = s.working_space.is_some_and(|w| s.linearize || w.is_linear()); + Pipe { + levels: s.bit_depth.levels(), + space: s.working_space, + linear, + linear_blend: s.blend_linear && !linear, + out: s.output_space.unwrap_or(ColorSpace::Srgb), + hdr: s.hdr, + } } /// The space whose curve encodes working-space pixels (sRGB when unmanaged). @@ -67,10 +79,22 @@ impl Pipe { self.linear_blend.then(|| Conversion::delinearize(self.curve())) } - /// Working space → sRGB display (top-level comp output). + /// Working space → the output space (top-level comp output; sRGB by default). `None` when + /// unmanaged or when [`Pipe::output_hdr`] applies. pub fn output(&self) -> Option { let ws = self.space?; - Conversion::new(ws, self.linear, ColorSpace::Srgb, false) + if self.output_hdr().is_some() { + return None; + } + Conversion::new(ws, self.linear, self.out, false) + } + + /// Output with HDR compand / tone mapping (standard-dynamic-range outputs only): working + /// space → linear output primaries, the HDR curve, then the output's encoding. + pub fn output_hdr(&self) -> Option<(Option, HdrMode, Conversion)> { + let ws = self.space?; + (self.hdr != HdrMode::Clip && !self.out.is_hdr()) + .then(|| (Conversion::new(ws, self.linear, self.out, true), self.hdr, Conversion::delinearize(self.out))) } /// One pixel clamped and quantised to the bit depth. @@ -122,6 +146,7 @@ impl Pipe { let mut k = self.levels.map_or(0, |l| l as u64); k = k.wrapping_mul(31).wrapping_add(self.space.map_or(7, |s| s as u64 + 11)); k = k.wrapping_mul(31).wrapping_add(self.linear as u64 * 2 + self.linear_blend as u64); + k = k.wrapping_mul(31).wrapping_add(self.out as u64 * 4 + self.hdr as u64); k } } @@ -202,3 +227,42 @@ pub fn convert(img: &mut Image, c: &Conversion) { *p = [o[0] * a, o[1] * a, o[2] * a, a]; }); } + +/// Compand: an exponential knee from 80%, approaching 1.0 (linear light, per channel). +pub fn compand(v: f32) -> f32 { + const K: f32 = 0.8; + if v <= K { v } else { K + (1.0 - K) * (1.0 - (-(v - K) / (1.0 - K)).exp()) } +} + +/// Extended Reinhard tone mapping of luminance with white at `W` (linear light). +pub fn tone_map(c: [f32; 3]) -> [f32; 3] { + const W: f32 = 4.0; + let l = effectcraft_color::luminance(c[0], c[1], c[2]); + if l <= 0.0 { + return c; + } + let lm = l * (1.0 + l / (W * W)) / (1.0 + l); + let k = lm / l; + c.map(|v| v * k) +} + +/// The HDR step of [`Pipe::output_hdr`] on premultiplied pixels. +pub fn output_hdr(img: &mut Image, to_linear: Option, mode: HdrMode, encode: Conversion) { + img.data.par_iter_mut().for_each(|p| { + let a = p[3]; + if a <= 0.0 { + return; + } + let mut c = [p[0] / a, p[1] / a, p[2] / a]; + if let Some(t) = &to_linear { + c = t.apply(c); + } + c = match mode { + HdrMode::Clip => c, + HdrMode::Compand => c.map(compand), + HdrMode::ToneMap => tone_map(c), + }; + let o = encode.apply(c); + *p = [o[0] * a, o[1] * a, o[2] * a, a]; + }); +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 86d1bd9a..7de8cbd1 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -551,6 +551,12 @@ impl<'a> Renderer<'a> { color::convert(&mut canvas, &c); self.pipe.quantize(&mut canvas); } + if self.depth == 0 + && let Some((lin, mode, enc)) = self.pipe.output_hdr() + { + color::output_hdr(&mut canvas, lin, mode, enc); + self.pipe.quantize(&mut canvas); + } canvas } diff --git a/crates/render/src/three_d/camera.rs b/crates/render/src/three_d/camera.rs index fb307b8c..5d33ea3f 100644 --- a/crates/render/src/three_d/camera.rs +++ b/crates/render/src/three_d/camera.rs @@ -322,11 +322,15 @@ pub enum View3D { Custom1, Custom2, Custom3, + /// The comp's default camera (50 mm, looking at the comp centre), ignoring camera layers; + /// orbitable like the custom views. + Default, } impl View3D { - pub const ALL: [View3D; 10] = [ + pub const ALL: [View3D; 11] = [ View3D::ActiveCamera, + View3D::Default, View3D::Front, View3D::Left, View3D::Top, @@ -349,6 +353,7 @@ impl View3D { View3D::Custom1 => "Custom View 1", View3D::Custom2 => "Custom View 2", View3D::Custom3 => "Custom View 3", + View3D::Default => "Default", } } /// Stable id used by commands (`activeCamera`, `front`, `custom1`…). @@ -364,6 +369,7 @@ impl View3D { View3D::Custom1 => "custom1", View3D::Custom2 => "custom2", View3D::Custom3 => "custom3", + View3D::Default => "default", } } pub fn from_id(s: &str) -> Option { @@ -415,6 +421,7 @@ pub fn default_view_cam(v: View3D, w: f64, h: f64) -> ViewCam { View3D::Custom1 => custom(30.0, -20.0), View3D::Custom2 => custom(0.0, -35.0), View3D::Custom3 => custom(-30.0, -20.0), + View3D::Default => ViewCam { eye: arr(vec3(w / 2.0, h / 2.0, -zoom)), poi: arr(c), down: [0.0, 1.0, 0.0], zoom, ortho: false }, } } diff --git a/crates/time/src/lib.rs b/crates/time/src/lib.rs index c3077490..3d4c2923 100644 --- a/crates/time/src/lib.rs +++ b/crates/time/src/lib.rs @@ -435,6 +435,55 @@ pub fn parse_timecode(input: &str, rate: FrameRate, drop_frame: bool, current: i }) } +/// Format a frame count as Feet + Frames (film footage counting) with `per_foot` frames per +/// foot (35 mm film: 16, 16 mm: 40): `0012+07`. Negative counts get a leading `-`. +pub fn format_feet_frames(frame: i64, per_foot: i64) -> String { + let pf = per_foot.max(1); + let sign = if frame < 0 { "-" } else { "" }; + let a = frame.abs(); + let w = if pf > 100 { 3 } else { 2 }; + format!("{sign}{:04}+{:0w$}", a / pf, a % pf) +} + +/// Parse a Feet + Frames field into a frame count: `12+07` (feet + frames; frames may overflow +/// a foot), `-1+04`, a bare frame count (`250`), or `+n` / `-n` frames relative to `current`. +pub fn parse_feet_frames(input: &str, per_foot: i64, current: i64) -> Result { + let s = input.trim(); + let bad = || ParseError(format!("not feet+frames: `{input}`")); + let num = |t: &str| -> Result { + let t = t.trim(); + if t.is_empty() { + return Ok(0); + } + if !t.bytes().all(|b| b.is_ascii_digit()) { + return Err(bad()); + } + t.parse::().map_err(|_| ParseError("number too large".into())) + }; + if s.is_empty() { + return Err(ParseError("empty feet+frames".into())); + } + let (sign, body) = match s.as_bytes()[0] { + b'-' => (-1, &s[1..]), + b'+' => (1, &s[1..]), + _ => (0, s), + }; + match body.rfind('+') { + Some(i) => { + let v = num(&body[..i])? * per_foot.max(1) + num(&body[i + 1..])?; + // A sign before feet+frames is the sign of the count, not a relative move. + Ok(if sign < 0 { -v } else { v }) + } + None => { + let v = num(body)?; + Ok(match sign { + 0 => v, + k => current + k * v, + }) + } + } +} + /// Common audio sample rates offered in sequence settings. pub const SAMPLE_RATES: [i64; 6] = [32_000, 44_100, 48_000, 88_200, 96_000, 192_000]; @@ -525,4 +574,29 @@ mod tests { prop_assert_eq!(parse_timecode(&txt, r, false, 0).unwrap(), f); } } + + #[test] + fn feet_and_frames() { + // 35 mm: 16 frames per foot; 16 mm: 40. + assert_eq!(format_feet_frames(0, 16), "0000+00"); + assert_eq!(format_feet_frames(16 * 12 + 7, 16), "0012+07"); + assert_eq!(format_feet_frames(95, 40), "0002+15"); + assert_eq!(format_feet_frames(-20, 16), "-0001+04"); + assert_eq!(parse_feet_frames("12+07", 16, 0), Ok(199)); + assert_eq!(parse_feet_frames("0012+07", 16, 0), Ok(199)); + assert_eq!(parse_feet_frames("2+15", 40, 0), Ok(95)); + assert_eq!(parse_feet_frames("-1+04", 16, 0), Ok(-20)); + assert_eq!(parse_feet_frames("1+20", 16, 0), Ok(36), "frames may overflow a foot"); + assert_eq!(parse_feet_frames("250", 16, 0), Ok(250)); + assert_eq!(parse_feet_frames("+10", 16, 100), Ok(110)); + assert_eq!(parse_feet_frames("-10", 16, 100), Ok(90)); + assert_eq!(parse_feet_frames("+", 16, 5), Ok(5)); + assert!(parse_feet_frames("1:2", 16, 0).is_err()); + assert!(parse_feet_frames("", 16, 0).is_err()); + for f in [0, 1, 15, 16, 17, 1234, 99_999] { + for pf in [16, 40] { + assert_eq!(parse_feet_frames(&format_feet_frames(f, pf), pf, 0), Ok(f)); + } + } + } } diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 3e051948..39640ef6 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -286,6 +286,10 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ("HDTV (Rec. 709)", json!("rec709")), ("Rec. 2020", json!("rec2020")), ("Display P3", json!("p3")), + ("ACEScg", json!("acescg")), + ("ACES2065-1", json!("aces2065")), + ("Rec. 2100 PQ", json!("rec2100pq")), + ("Rec. 2100 HLG", json!("rec2100hlg")), ], f.color_profile.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::ALL.iter().position(|x| *x == c).unwrap_or(0)), ), @@ -316,8 +320,27 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { Field::choice( "timeDisplay", "Time display style", - &[("Timecode", json!("timecode")), ("Frames", json!("frames"))], - usize::from(matches!(st.time_display, effectcraft_engine::project::TimeDisplayStyle::Frames)), + &[ + ("Timecode", json!("timecode")), + ("Frames", json!("frames")), + ("Feet + Frames (35mm)", json!("feet35")), + ("Feet + Frames (16mm)", json!("feet16")), + ], + { + use effectcraft_engine::project::TimeDisplayStyle as T; + match st.time_display { + T::Timecode => 0, + T::Frames => 1, + T::Feet35 => 2, + T::Feet16 => 3, + } + }, + ), + Field::choice( + "colorEngine", + "Color engine", + &[("Adobe-style built-in", json!("adobe")), ("OCIO (built-in ACES config)", json!("ocio"))], + usize::from(st.color_engine == effectcraft_engine::project::ColorEngine::Ocio), ), Field::choice( "workingSpace", @@ -328,11 +351,42 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ("HDTV (Rec. 709)", json!("rec709")), ("Rec. 2020", json!("rec2020")), ("Display P3", json!("p3")), + ("ACEScg", json!("acescg")), + ("ACES2065-1", json!("aces2065")), ], - st.working_space.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::ALL.iter().position(|x| *x == c).unwrap_or(0)), + st.working_space.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::WORKING.iter().position(|x| *x == c).unwrap_or(0)), ), Field::bool("linearize", "Linearize working space", st.linearize), Field::bool("blendLinear", "Blend colors using 1.0 gamma", st.blend_linear), + Field::choice( + "hdr", + "HDR on SDR displays and outputs", + &[("Clip", json!("clip")), ("Compand", json!("compand")), ("Tone map", json!("toneMap"))], + st.hdr as usize, + ), + Field::choice( + "outputSpace", + "Output color space", + &[ + ("sRGB IEC61966-2.1 (default)", json!("default")), + ("HDTV (Rec. 709)", json!("rec709")), + ("Rec. 2020", json!("rec2020")), + ("Display P3", json!("p3")), + ("Rec. 2100 PQ (HDR)", json!("rec2100pq")), + ("Rec. 2100 HLG (HDR)", json!("rec2100hlg")), + ], + { + use effectcraft_engine::project::ColorSpace as C; + match st.output_space { + Some(C::Rec709) => 1, + Some(C::Rec2020) => 2, + Some(C::DisplayP3) => 3, + Some(C::Rec2100Pq) => 4, + Some(C::Rec2100Hlg) => 5, + _ => 0, + } + }, + ), // Video Rendering and Effects ▸ Use. Field::choice( "renderer", diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 2c1f95ee..aee74e66 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -40,7 +40,7 @@ pub mod viewer_tools; pub mod waveform; use effectcraft_engine::Session; -use effectcraft_engine::project::{Comp, TimeDisplayStyle}; +use effectcraft_engine::project::Comp; use effectcraft_engine::time::Tick; use egui::Rect; @@ -58,14 +58,10 @@ pub enum DragPayload { Property { layer: u64, prop: u64 }, } -/// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). +/// The current time formatted per project settings (timecode with `;` for drop-frame, frames +/// or Feet + Frames). pub fn timecode(session: &Session, comp: &Comp, t: Tick) -> String { - let fr = comp.frame_rate; - match session.project.settings.time_display { - TimeDisplayStyle::Frames => format!("{:05}", fr.frame_at(t) + session.project.settings.frame_start), - // AE's display: `0:00:02:15` (non-drop-frame counting, as AE shows new comps). - TimeDisplayStyle::Timecode => effectcraft_engine::time::format_timecode_ae(fr.frame_at(t + comp.display_start), fr, false), - } + effectcraft_engine::commands::time::display_time(session, comp, t) } pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rect) { diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 64babbf2..f157e0bb 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -286,6 +286,70 @@ fn aux_views( main } +/// View ▸ Split with New Locked Viewer: the locked viewer on the left of the Composition panel +/// (its comp and 3D view stay put whatever comp is active). Returns the main viewer's rectangle. +fn locked_pane(app: &mut EffectcraftApp, ui: &mut egui::Ui, full: Rect, bg: Color32) -> Rect { + let Some(lv) = app.session.state.locked_viewer else { return full }; + let Some(comp) = app.session.project.comp(lv.comp).cloned() else { return full }; + let name = app.session.project.item(lv.comp).map(|i| i.name.clone()).unwrap_or_default(); + let gap = 2.0; + let mid = full.center().x; + let r = Rect::from_min_max(full.min, pos2(mid - gap / 2.0, full.max.y)); + let main = Rect::from_min_max(pos2(mid + gap / 2.0, full.min.y), full.max); + let ctx = ui.ctx().clone(); + let p = ui.painter().clone(); + p.rect_filled(Rect::from_min_max(pos2(r.max.x, full.min.y), pos2(main.min.x, full.max.y)), 0.0, Color32::from_black_alpha(200)); + p.rect_filled(r, 0.0, bg); + let (cw, ch) = (comp.width as f32, comp.height as f32); + let fit = ((r.width() - 20.0) / cw).min((r.height() - 44.0) / ch).max(0.01); + let cr = Rect::from_center_size(r.center(), vec2(cw * fit, ch * fit)); + let scale = (fit * ctx.pixels_per_point()).min(1.0) as f64; + let t = app.session.time_of(lv.comp); + let cam = (lv.view != View3D::ActiveCamera && comp.has_3d()) + .then(|| app.session.state.views3d.get(&lv.comp).cloned().unwrap_or_default().cam(lv.view, comp.width as f64, comp.height as f64).state()); + let dc = effectcraft_engine::viewer::DisplayColor::of(&app.session); + let key = { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + (app.session.revision, lv.comp.0, t.0, scale.to_bits(), format!("{cam:?}{}", dc.is_some())).hash(&mut h); + h.finish() + }; + let id = egui::Id::new("viewer-locked"); + let cached: Option<(u64, egui::TextureHandle)> = ctx.data(|d| d.get_temp(id)); + let tex = match cached { + Some((k, tex)) if k == key => tex, + _ => { + let opts = effectcraft_engine::render::RenderOpts { scale, view: cam, ..Default::default() }; + let img = app.session.render(lv.comp, t, opts); + let mut ci = crate::frames::to_color_image(&img); + if let Some(dc) = dc { + let mut px: Vec<[u8; 4]> = ci.pixels.iter().map(|c| c.to_array()).collect(); + dc.apply(&mut px); + ci = egui::ColorImage::new(ci.size, px.into_iter().map(|a| Color32::from_rgba_premultiplied(a[0], a[1], a[2], a[3])).collect()); + } + let tex = ctx.load_texture("viewer-locked", ci, egui::TextureOptions::LINEAR); + ctx.data_mut(|d| d.insert_temp(id, (key, tex.clone()))); + tex + } + }; + let b = comp.background; + p.rect_filled(cr, 0.0, Color32::from_rgb((b[0] * 255.0) as u8, (b[1] * 255.0) as u8, (b[2] * 255.0) as u8)); + p.image(tex.id(), cr, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + p.rect_stroke(cr, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); + let label = format!("Locked: {name} \u{2014} {}", if comp.has_3d() { lv.view.label() } else { "Active Camera" }); + p.text(r.left_top() + vec2(8.0, 8.0), Align2::LEFT_TOP, &label, Tokens::ui(11.0), Color32::from_white_alpha(210)); + app.auto.add("viewer.locked", r, &label); + // Close the locked viewer. + let close = Rect::from_min_size(pos2(r.max.x - 26.0, r.min.y + 4.0), vec2(20.0, 20.0)); + let resp = ui.interact(close, egui::Id::new("viewer-locked-close"), Sense::click()); + p.text(close.center(), Align2::CENTER_CENTER, "\u{00d7}", Tokens::ui(14.0), if resp.hovered() { Color32::WHITE } else { Color32::from_white_alpha(170) }); + app.auto.add("viewer.locked.close", close, "Close Locked Viewer"); + if resp.clicked() { + let _ = app.session.execute("view.closeLockedViewer", serde_json::json!({})); + } + main +} + pub(crate) fn checker(p: &egui::Painter, r: Rect) { let s = 10.0; p.rect_filled(r, 0.0, Color32::from_gray(0xcc)); @@ -354,6 +418,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let pasteboard = app.ui.viewer.pasteboard.map(|[r, g, b]| Color32::from_rgb(r, g, b)).unwrap_or(t.pasteboard); // View ▸ Switch View Layout: extra views (Top / Front / Right) beside the main view, which // keeps the overlays and the interaction. + let full = locked_pane(app, ui, full, pasteboard); let area = aux_views(app, ui, &comp, cid, full, pasteboard); p.rect_filled(area, 0.0, pasteboard); // View ▸ Show Rulers takes a strip on the top and left. diff --git a/crates/ui-egui/src/panels/viewer_tools.rs b/crates/ui-egui/src/panels/viewer_tools.rs index 79688e19..4ad14510 100644 --- a/crates/ui-egui/src/panels/viewer_tools.rs +++ b/crates/ui-egui/src/panels/viewer_tools.rs @@ -307,8 +307,13 @@ pub(crate) fn showing_snapshot(app: &EffectcraftApp, ctx: &egui::Context) -> boo app.session.snapshot.is_some() && (app.session.state.viewer.show_snapshot || f5) } -fn transformed(img: &egui::ColorImage, ch: Channel, colorized: bool, stops: f32) -> egui::ColorImage { +/// Show Channel and exposure, then the display colour conversion (View ▸ Use Display Color +/// Management / Simulate Output). +fn transformed(img: &egui::ColorImage, ch: Channel, colorized: bool, stops: f32, dc: Option) -> egui::ColorImage { let mut px: Vec<[u8; 4]> = img.pixels.iter().map(|c| c.to_array()).collect(); + if let Some(dc) = dc.filter(|_| matches!(ch, Channel::Rgb | Channel::RgbStraight)) { + dc.apply(&mut px); + } vw::display_transform(&mut px, ch, colorized, stops); egui::ColorImage::new(img.size, px.into_iter().map(|a| Color32::from_rgba_premultiplied(a[0], a[1], a[2], a[3])).collect()) } @@ -336,14 +341,18 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: } return; } - let plain = opts.channel == Channel::Rgb && opts.exposure == 0.0; + let dc = vw::DisplayColor::of(&app.session); + // The display conversion's inputs, for the texture caches. + let dc_key = format!("{:?}{:?}{}", dc.is_some(), opts.simulation, app.session.prefs.previews.display_profile) + + &format!("{:?}{:?}{}", app.session.project.settings.working_space, app.session.project.settings.output_space, opts.display_color_management); + let plain = opts.channel == Channel::Rgb && opts.exposure == 0.0 && dc.is_none(); let uv = Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)); if snap { let Some(s) = app.session.snapshot.clone() else { return }; let key = { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); - (std::sync::Arc::as_ptr(&s.image) as usize, opts.channel.id(), opts.colorized, opts.exposure.to_bits()).hash(&mut h); + (std::sync::Arc::as_ptr(&s.image) as usize, opts.channel.id(), opts.colorized, opts.exposure.to_bits(), &dc_key).hash(&mut h); h.finish() }; let id = egui::Id::new("viewer-snapshot-tex"); @@ -351,7 +360,7 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: Some((k, tex)) if k == key => tex, _ => { let img = crate::frames::to_color_image(&s.image); - let img = if plain { img } else { transformed(&img, opts.channel, opts.colorized, opts.exposure) }; + let img = if plain { img } else { transformed(&img, opts.channel, opts.colorized, opts.exposure, dc) }; let tex = ctx.load_texture("viewer-snapshot", img, egui::TextureOptions::LINEAR); ctx.data_mut(|d| d.insert_temp(id, (key, tex.clone()))); tex @@ -378,14 +387,14 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: let key = { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); - (std::sync::Arc::as_ptr(&src) as usize, opts.channel.id(), opts.colorized, opts.exposure.to_bits()).hash(&mut h); + (std::sync::Arc::as_ptr(&src) as usize, opts.channel.id(), opts.colorized, opts.exposure.to_bits(), &dc_key).hash(&mut h); h.finish() }; let id = egui::Id::new("viewer-display-tex"); let tex = match ctx.data(|d| d.get_temp::<(u64, egui::TextureHandle)>(id)) { Some((k, tex)) if k == key => tex, _ => { - let tex = ctx.load_texture("viewer-display", transformed(&src, opts.channel, opts.colorized, opts.exposure), egui::TextureOptions::LINEAR); + let tex = ctx.load_texture("viewer-display", transformed(&src, opts.channel, opts.colorized, opts.exposure, dc), egui::TextureOptions::LINEAR); ctx.data_mut(|d| d.insert_temp(id, (key, tex.clone()))); tex } diff --git a/docs/parity.md b/docs/parity.md index 3517c484..5e344ae2 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -22,8 +22,7 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur -and blend modes; a few viewer and colour-management menu items; OpenType features and variable +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and the "better than After Effects" items (a plug-in API, branching history, GPU particles). @@ -120,10 +119,10 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 76% | 10.0 | multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | +| 3D | 84% | 7.0 | multi-view layouts, collapsed 3D precomps in Advanced 3D, the Extended Viewer; stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | | Effects | ≈ 75% | 5.0 | GPU versions of more effects (every After Effects effect exists since M9.11, M12.5 and M12.6) | | Interface | 70% | 6.0 | Timeline horizontal scrolling of many columns, a richer Learn area (native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | -| Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | +| Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | @@ -140,7 +139,7 @@ Mocha and Cineware are third-party and not counted. |---|---|---| | 3D Channel | 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, Fog 3D, ID Matte, IDentifier | Read auxiliary channels: multi-layer OpenEXR (any `layer.channel`, Cryptomatte manifests from the header) or the compositor's own depth / layer-ID / normal / UV / Cryptomatte pass for precomps of 3D comps. | | Immersive Video | VR Blur, VR Chromatic Aberrations, VR Color Gradients, VR Converter, VR De-Noise, VR Digital Glitch, VR Fractal Noise, VR Glow, VR Plane to Sphere, VR Rotate Sphere, VR Sharpen, VR Sphere To Plane | Seam-aware equirectangular processing (wrap-around, pole continuation, latitude-scaled filters), mono / over-under / side-by-side layouts; VR Converter between equirectangular, cube maps (4:3, 3:2, 6:1), sphere map and 2D. | -| Color Correction | OCIO CDL / Color Space / Display / File / Look Transform, Color Stabilizer | Built-in minimal OCIO-style config (ACES2065-1, ACEScg, ACEScct, sRGB, Rec.709, Rec.2020, Display P3, linear variants, XYZ) from published matrices and transfer functions; `.cube` / `.3dl` / `.csp` LUTs and ASC `.cc` / `.ccc` / `.cdl`. Custom `.ocio` config files are not read yet. | +| Color Correction | OCIO CDL / Color Space / Display / File / Look Transform, Color Stabilizer | Built-in minimal OCIO-style config (ACES2065-1, ACEScg, ACEScct, sRGB, Rec.709, Rec.2020, Display P3, linear variants, XYZ) from published matrices and transfer functions; `.cube` / `.3dl` / `.csp` LUTs and ASC `.cc` / `.ccc` / `.cdl`. Custom `.ocio` config files are read (Configuration ▸ Custom: colorspaces, roles, displays/views; Matrix, File, Exponent, Log, LogAffine, CDL, Range and Group transforms). | | Distort | CC Flo Motion, Liquify, Rolling Shutter Repair | Liquify strokes are effect data replayed into a displacement mesh (`liquify.stroke` / `liquify.clear` commands; no interactive viewer brush yet). Rolling Shutter Repair uses the KLT tracker. | | Blur & Sharpen | Camera-Shake Deblur, CC Radial Blur | Deblur substitutes aligned patches from sharper neighbouring frames. | | Audio | Compressor, Distortion, Gate | Applied in the mixdown like the other audio effects. | diff --git a/docs/preferences.md b/docs/preferences.md index cb05229e..4e37704e 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -89,6 +89,7 @@ These rows are in the dialog for After Effects parity and are stored, but nothin - `composition.hardwareAcceleratePanels`: Hardware Accelerate Composition, Layer and Footage Panels - `previews.showInternalWireframes`: Show Internal Wireframes - `previews.zoomQuality`: Viewer Zoom Quality +- `previews.displayProfile`: Display Color Space - `appearance.useLabelColorForTabs`: Use Label Color for Related Tabs - `appearance.cycleMaskColors`: Cycle Mask Colors - `appearance.useGradients`: Use Gradients From fcc09cf09555c91c12584386514cc1ffcb45378c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:42:43 -0400 Subject: [PATCH 07/36] M12.7: GPU distort/stylize effects (Turbulent Displace, Displacement Map, Wave Warp, Ripple, Twirl, Bulge, CC Lens, Mesh Warp, Mosaic, Find Edges, Emboss) Compute kernels in shaders/fx_distort.wgsl mirroring the CPU effects; per-row/column structure (map coordinates, wave phase, tile bounds, mesh triangle buckets) is computed in f64 on the CPU and uploaded. Displacement Map fetches its map layer through the effect host as the CPU does and samples it on the GPU. CPU oracle tests in tests_fx_distort.rs. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/distort2.rs | 3 +- crates/effects/src/lib.rs | 12 + crates/gpu/src/fx_distort.rs | 405 +++++++++++++++++- crates/gpu/src/lib.rs | 2 + crates/gpu/src/shaders/fx_distort.wgsl | 552 +++++++++++++++++++++++++ crates/gpu/src/tests_fx_distort.rs | 119 ++++++ 6 files changed, 1087 insertions(+), 6 deletions(-) create mode 100644 crates/gpu/src/tests_fx_distort.rs diff --git a/crates/effects/src/distort2.rs b/crates/effects/src/distort2.rs index 66f038e0..2d52f039 100644 --- a/crates/effects/src/distort2.rs +++ b/crates/effects/src/distort2.rs @@ -414,7 +414,8 @@ fn grid_warp(src: &Image, w: u32, h: u32, nx: usize, ny: usize, dest: &[Pt], src out } -fn parse_mesh(s: &str) -> Vec { +/// Mesh Warp's `mesh` parameter: the vertex offsets (layer pixels), row-major. +pub fn parse_mesh(s: &str) -> Vec<(f64, f64)> { s.split(|c: char| c.is_whitespace() || c == ';') .filter(|t| !t.is_empty()) .filter_map(|t| { diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index a5b4f346..7c6e0574 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -62,6 +62,7 @@ use std::sync::OnceLock; pub use color_fx::{exposure_settings, levels_clip}; pub use color2::Curve; +pub use distort2::parse_mesh; use effectcraft_keyframe::Value; use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; @@ -641,6 +642,17 @@ pub const GPU_EFFECTS: &[&str] = &[ "ec.color.exposure", "ec.channel.invert", "ec.distort.transform", + "ec.distort.turbulentdisplace", + "ec.distort.displacementmap", + "ec.distort.wavewarp", + "ec.distort.ripple", + "ec.distort.twirl", + "ec.distort.bulge", + "ec.distort.cclens", + "ec.distort.meshwarp", + "ec.stylize.mosaic", + "ec.stylize.findedges", + "ec.stylize.emboss", ]; /// Effects whose output depends on [`EffectCtx::time`] directly (not only through animated diff --git a/crates/gpu/src/fx_distort.rs b/crates/gpu/src/fx_distort.rs index fe92ad91..bcd7214f 100644 --- a/crates/gpu/src/fx_distort.rs +++ b/crates/gpu/src/fx_distort.rs @@ -1,18 +1,413 @@ //! GPU effects, distortion and stylize family (kernels in `shaders/fx_distort.wgsl`): the CPU effects' exact //! steps with the pixel loops as compute kernels. +//! +//! Parameter conversions, padding and anything with a per-row or per-column structure (map +//! coordinates, tile bounds, Mesh Warp's triangle buckets) are computed here in f64 exactly as +//! the CPU does and uploaded; the kernels do the per-pixel work. + +use std::f64::consts::TAU; use effectcraft_effects::EffectCtx; +use effectcraft_effects::util::{Src, src_at}; -use crate::context::Enc; +use crate::context::{Enc, Params}; use crate::effects::GBuf; /// Compute entry points in `fx_distort.wgsl`. -pub(crate) const KERNELS: &[&str] = &[]; +pub(crate) const KERNELS: &[&str] = &[ + "dst_twirl", + "dst_bulge", + "dst_wavewarp", + "dst_ripple", + "dst_cclens", + "dst_turbulent", + "dst_displace", + "dst_mesh", + "dst_mosaic_rows", + "dst_mosaic_tiles", + "dst_mosaic_out", + "dst_mosaic_sharp", + "dst_findedges", + "dst_emboss", +]; /// Effect ids implemented here. -pub(crate) const IDS: &[&str] = &[]; +pub(crate) const IDS: &[&str] = &[ + "ec.distort.turbulentdisplace", + "ec.distort.displacementmap", + "ec.distort.wavewarp", + "ec.distort.ripple", + "ec.distort.twirl", + "ec.distort.bulge", + "ec.distort.cclens", + "ec.distort.meshwarp", + "ec.stylize.mosaic", + "ec.stylize.findedges", + "ec.stylize.emboss", +]; /// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. -pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { - None +pub(crate) fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + match id { + "ec.distort.turbulentdisplace" => turbulent(e, ctx, b), + "ec.distort.displacementmap" => displacement_map(e, ctx, b), + "ec.distort.wavewarp" => wave_warp(e, ctx, b), + "ec.distort.ripple" => ripple(e, ctx, b), + "ec.distort.twirl" => twirl(e, ctx, b), + "ec.distort.bulge" => bulge(e, ctx, b), + "ec.distort.cclens" => cc_lens(e, ctx, b), + "ec.distort.meshwarp" => mesh_warp(e, ctx, b), + "ec.stylize.mosaic" => mosaic(e, ctx, b), + "ec.stylize.findedges" => find_edges(e, ctx, b), + "ec.stylize.emboss" => emboss(e, ctx, b), + _ => None, + } +} + +/// Run a same-size per-pixel kernel over the buffer. +fn run(e: &mut Enc, entry: &str, p: &Params, mut b: GBuf, data: Option<&wgpu::Buffer>) -> Option { + let out = e.image(b.img.width, b.img.height); + e.pixels(entry, p, &b.img, None, &out, data); + b.img = out; + Some(b) +} + +/// Buf::layer_w / layer_h (distort.rs). +fn layer_wh(ctx: &EffectCtx) -> (f64, f64) { + (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)) +} + +/// util::layer_rect. +fn layer_rect(ctx: &EffectCtx, b: &GBuf) -> [f32; 4] { + [b.offset[0] as f32, b.offset[1] as f32, (ctx.layer_size[0] * b.scale) as f32, (ctx.layer_size[1] * b.scale) as f32] +} + +// ---------------------------------------------------------------- distort.rs + +fn twirl(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let ang = ctx.params.f("angle").to_radians(); + let (lw, lh) = layer_wh(ctx); + let r = (ctx.params.f("radius") / 100.0 * lw.max(lh) * b.scale).max(1.0); + let c = b.to_px(ctx.params.v2("center")); + let mut p = Params::default(); + p.f[0] = [c.0 as f32, c.1 as f32, r as f32, ang as f32]; + run(e, "dst_twirl", &p, b, None) +} + +fn bulge(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let rx = (ctx.params.f("hradius") * b.scale).max(1.0); + let ry = (ctx.params.f("vradius") * b.scale).max(1.0); + let c = b.to_px(ctx.params.v2("center")); + let h = ctx.params.f("height"); + let taper = 1.0 + ctx.params.f("taperRadius").max(0.0) / 100.0 * 3.0; + let pin = ctx.params.b("pinAllEdges"); + let (lw, lh) = layer_wh(ctx); + let (lw, lh) = (lw * b.scale, lh * b.scale); + let edge = (lw.min(lh) * 0.1).max(1.0); + let mut p = Params::default(); + p.f[0] = [c.0 as f32, c.1 as f32, rx as f32, ry as f32]; + p.f[1] = [h as f32, taper as f32, if pin { 1.0 } else { 0.0 }, edge as f32]; + p.f[2] = [b.offset[0] as f32, b.offset[1] as f32, lw as f32, lh as f32]; + run(e, "dst_bulge", &p, b, None) +} + +fn wave_warp(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let h = ctx.params.f("height") * b.scale; + let w = (ctx.params.f("width") * b.scale).max(1.0); + let dir = ctx.params.f("direction").to_radians(); + let speed = ctx.params.f("speed"); + let phase = ctx.params.f("phase").to_radians() + ctx.time * speed * TAU; + let (dx, dy) = (dir.sin(), -dir.cos()); + if !ctx.adjustment { + b.pad(e, h.abs().ceil() as u32 + 1)?; + } + let (lw, lh) = layer_wh(ctx); + let rect = [b.offset[0] as f32, b.offset[1] as f32, (lw * b.scale).max(1.0) as f32, (lh * b.scale).max(1.0) as f32]; + // The wave phase in cycles, (x·dx + y·dy) / width + phase / 2π, is separable: per column + // and per row (fraction, whole cycles) in f64, so the kernel's sum keeps the fraction exact + // to f32 (the circle shapes have vertical tangents where a phase error is amplified). + let (bw, bh) = (b.img.width as usize, b.img.height as usize); + let mut data = vec![0.0f32; 2 * (bw + bh)]; + let split = |v: f64| (v - v.floor(), v.floor()); + for x in 0..bw { + let (f, i) = split((x as f64 + 0.5) * dx / w); + data[x] = f as f32; + data[bw + x] = i as f32; + } + for y in 0..bh { + let (f, i) = split((y as f64 + 0.5) * dy / w + phase / TAU); + data[2 * bw + y] = f as f32; + data[2 * bw + bh + y] = i as f32; + } + let mut p = Params::default(); + p.u[0] = [ctx.params.e("waveType"), ctx.params.e("pinning"), ctx.seed ^ 0x3a7e, bw as u32]; + p.f[0] = [dx as f32, dy as f32, h as f32, 0.0]; + p.f[2] = rect; + let buf = e.data(&data); + run(e, "dst_wavewarp", &p, b, Some(&buf)) +} + +fn ripple(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let (lw, lh) = layer_wh(ctx); + let r = (ctx.params.f("radius") / 100.0 * lw.max(lh) * 0.5 * b.scale).max(1.0); + let asymmetric = ctx.params.e("conversion") == 0; + let c = b.to_px(ctx.params.v2("center")); + let w = (ctx.params.f("waveWidth") * b.scale).max(1.0); + let h = ctx.params.f("waveHeight") * b.scale; + let phase = ctx.params.f("phase").to_radians() - ctx.time * ctx.params.f("speed") * TAU; + let mut p = Params::default(); + p.u[0][0] = asymmetric as u32; + p.f[0] = [c.0 as f32, c.1 as f32, r as f32, w as f32]; + p.f[1] = [h as f32, phase.rem_euclid(TAU) as f32, 0.0, 0.0]; + run(e, "dst_ripple", &p, b, None) +} + +// ---------------------------------------------------------------- distort2.rs + +fn cc_lens(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let c = b.to_px(ctx.params.v2("center")); + let (lw, lh) = (ctx.layer_size[0] * b.scale, ctx.layer_size[1] * b.scale); + let r = (ctx.params.f("size") / 100.0 * lw.max(lh) * 0.5).max(0.5); + let k = 2f64.powf(ctx.params.f("convergence") / 50.0); + let mut p = Params::default(); + p.f[0] = [c.0 as f32, c.1 as f32, r as f32, k as f32]; + run(e, "dst_cclens", &p, b, None) +} + +fn turbulent(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let amount = ctx.params.f("amount") * b.scale; + if amount.abs() < 1e-9 { + return Some(b); + } + let kind = ctx.params.e("displacement"); + let size = (ctx.params.f("size") * b.scale).max(1.0); + if ctx.params.b("resizeLayer") && !ctx.adjustment { + b.pad(e, amount.abs().ceil() as u32 + 1)?; + } + let off = b.to_px(ctx.params.v2("offset")); + let oct = ctx.params.f("complexity").clamp(1.0, 10.0); + let evo = ctx.params.f("evolution") / 360.0; + let cycle = ctx.params.b("evolutionOptions/cycleEvolution").then(|| ctx.params.f("evolutionOptions/cycle").round().max(1.0)); + let falloff = if (3..=5).contains(&kind) { 0.3 } else { 0.5 }; + let rect = layer_rect(ctx, &b); + let edge = ((ctx.layer_size[0] * b.scale).min(ctx.layer_size[1] * b.scale) * 0.1).max(1.0); + let seed = (ctx.seed ^ 0x7d15).wrapping_add((ctx.params.f("evolutionOptions/randomSeed") as i64 as u32).wrapping_mul(0x9e37_79b9)); + let (z0, z1, t) = match cycle { + Some(c) => { + let z = evo.rem_euclid(c); + (z as f32, (z - c) as f32, (z / c) as f32) + } + None => (evo as f32, 0.0, 0.0), + }; + let mut p = Params::default(); + p.u[0] = [kind, ctx.params.e("pinning"), seed, oct.ceil().max(1.0) as u32]; + p.u[1][0] = cycle.is_some() as u32; + p.f[0] = [off.0 as f32, off.1 as f32, size as f32, amount as f32]; + p.f[1] = [z0, z1, t, (oct - oct.floor()) as f32]; + p.f[2] = rect; + p.f[3] = [edge as f32, falloff, 0.0, 0.0]; + run(e, "dst_turbulent", &p, b, None) +} + +fn displacement_map(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let hs = src_at(ctx.params.e("useForHorizontal")); + let vs = src_at(ctx.params.e("useForVertical")); + let mh = ctx.params.f("maxHorizontal") * b.scale; + let mv = ctx.params.f("maxVertical") * b.scale; + let wrap = ctx.params.b("wrapPixelsAround"); + if ctx.params.b("expandOutput") && !wrap && !ctx.adjustment { + b.pad(e, mh.abs().max(mv.abs()).ceil() as u32)?; + } + let layer = ctx.layer_param("displacementMapLayer", false); + let behavior = ctx.params.e("displacementMapBehavior"); + let (w, h) = (b.img.width as usize, b.img.height as usize); + let inv = 1.0 / b.scale.max(1e-9); + let ls = ctx.layer_size; + // Per column / row: the map layer's buffer coordinate (fit_layer / Tile Map) and whether + // the pixel lies inside the centred map (Center Map). + let mut data = vec![0.0f32; 2 * (w + h)]; + let mut mode = 0u32; + let mut center = false; + let map = match &layer { + Some(o) => { + if o.buf.img.is_empty() { + return None; + } + let os = o.size; + let axis = |i: usize, k: usize| -> (f64, bool) { + let q = (i as f64 + 0.5 - b.offset[k]) * inv; + let map_q = match behavior { + 2 => (q + (os[k] - ls[k]) * 0.5).rem_euclid(os[k].max(1.0)), + 1 if ls[0] > 0.0 && ls[1] > 0.0 => q * (os[k] / ls[k]), + 1 => q, + _ => q + (os[k] - ls[k]) * 0.5, + }; + let c = q + (os[k] - ls[k]) * 0.5; + (map_q * o.buf.scale + o.buf.offset[k], (0.0..os[k]).contains(&c)) + }; + for x in 0..w { + let (m, inside) = axis(x, 0); + data[x] = m as f32; + data[w + h + x] = inside as u32 as f32; + } + for y in 0..h { + let (m, inside) = axis(y, 1); + data[w + y] = m as f32; + data[2 * w + h + y] = inside as u32 as f32; + } + mode = if behavior == 2 { 2 } else { 1 }; + center = behavior == 0; + Some(e.g.upload_image(&o.buf.img)?) + } + None => None, + }; + let idx = |s: Src| effectcraft_effects::util::SRC_ORDER.iter().position(|x| *x == s).unwrap_or(10) as u32; + let mut p = Params::default(); + p.u[0] = [idx(hs), idx(vs), mode, wrap as u32 | (center as u32) << 1]; + p.u[1] = [w as u32, h as u32, 0, 0]; + p.f[0] = [mh as f32, mv as f32, 0.0, 0.0]; + let buf = e.data(&data); + let out = e.image(b.img.width, b.img.height); + e.pixels("dst_displace", &p, &b.img, map.as_ref(), &out, Some(&buf)); + b.img = out; + Some(b) +} + +fn mesh_warp(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let rows = ctx.params.f("rows").round().clamp(1.0, 31.0) as usize; + let cols = ctx.params.f("columns").round().clamp(1.0, 31.0) as usize; + let q = ctx.params.f("quality").round().clamp(1.0, 10.0) as usize; + let offs = effectcraft_effects::parse_mesh(ctx.params.s("mesh")); + if offs.len() != (rows + 1) * (cols + 1) || offs.iter().all(|o| o.0 == 0.0 && o.1 == 0.0) { + return Some(b); + } + let (lw, lh) = (ctx.layer_size[0], ctx.layer_size[1]); + let vert = |i: usize, j: usize| { + let o = offs[j * (cols + 1) + i]; + b.to_px([i as f64 / cols as f64 * lw + o.0, j as f64 / rows as f64 * lh + o.1]) + }; + let (nx, ny) = (cols * q, rows * q); + let mut dest = Vec::with_capacity((nx + 1) * (ny + 1)); + let mut srcp = Vec::with_capacity((nx + 1) * (ny + 1)); + for gj in 0..=ny { + for gi in 0..=nx { + let (ci, cj) = ((gi / q).min(cols - 1), (gj / q).min(rows - 1)); + let (u, v) = ((gi - ci * q) as f64 / q as f64, (gj - cj * q) as f64 / q as f64); + let (p00, p10, p01, p11) = (vert(ci, cj), vert(ci + 1, cj), vert(ci, cj + 1), vert(ci + 1, cj + 1)); + let x = (p00.0 * (1.0 - u) + p10.0 * u) * (1.0 - v) + (p01.0 * (1.0 - u) + p11.0 * u) * v; + let y = (p00.1 * (1.0 - u) + p10.1 * u) * (1.0 - v) + (p01.1 * (1.0 - u) + p11.1 * u) * v; + dest.push((x, y)); + srcp.push(b.to_px([gi as f64 / nx as f64 * lw, gj as f64 / ny as f64 * lh])); + } + } + // distort2::grid_warp: triangles in order, bucketed per output row. + let idx = |i: usize, j: usize| j * (nx + 1) + i; + let hh = b.img.height as usize; + let mut tri_data: Vec = Vec::with_capacity(nx * ny * 24); + let mut buckets: Vec> = vec![Vec::new(); hh]; + let mut t = 0u32; + for j in 0..ny { + for i in 0..nx { + let (a, b2, c, d) = (idx(i, j), idx(i + 1, j), idx(i + 1, j + 1), idx(i, j + 1)); + for tri in [[a, b2, c], [a, c, d]] { + let [ia, ib, ic] = tri; + let (pa, pb, pc) = (dest[ia], dest[ib], dest[ic]); + let den: f64 = (pb.1 - pc.1) * (pa.0 - pc.0) + (pc.0 - pb.0) * (pa.1 - pc.1); + // Degenerate triangles are skipped per pixel on the CPU; leaving them out of + // the buckets is the same. + if den.abs() < 1e-12 { + continue; + } + let ys = [pa.1, pb.1, pc.1]; + let y0 = (ys[0].min(ys[1]).min(ys[2]) - 0.5).floor().max(0.0) as usize; + let y1 = (ys[0].max(ys[1]).max(ys[2]) + 0.5).ceil().max(0.0) as usize; + for bucket in buckets.iter_mut().take(y1.min(hh)).skip(y0) { + bucket.push(t); + } + let (sa, sb, sc) = (srcp[ia], srcp[ib], srcp[ic]); + tri_data.extend( + [(pb.1 - pc.1) / den, (pc.0 - pb.0) / den, (pc.1 - pa.1) / den, (pa.0 - pc.0) / den, pc.0, pc.1, sa.0, sa.1, sb.0, sb.1, sc.0, sc.1] + .map(|v| v as f32), + ); + t += 1; + } + } + } + let row_starts = tri_data.len(); + let list_start = row_starts + hh + 1; + let total: usize = buckets.iter().map(Vec::len).sum(); + if list_start + total >= 1 << 24 { + // Indices past 2^24 are not exact in the f32 data buffer. + return None; + } + let mut data = tri_data; + data.reserve(hh + 1 + total); + let mut n = 0usize; + for bucket in &buckets { + data.push(n as f32); + n += bucket.len(); + } + data.push(n as f32); + for bucket in &buckets { + data.extend(bucket.iter().map(|&t| t as f32)); + } + let mut p = Params::default(); + p.u[0] = [row_starts as u32, list_start as u32, 0, 0]; + let buf = e.data(&data); + run(e, "dst_mesh", &p, b, Some(&buf)) +} + +// ---------------------------------------------------------------- misc.rs (Stylize) + +fn mosaic(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let hb = ctx.params.f("horizontal").round().max(1.0) as usize; + let vb = ctx.params.f("vertical").round().max(1.0) as usize; + let (w, h) = (b.img.width as usize, b.img.height as usize); + if w == 0 || h == 0 { + return Some(b); + } + let cw = (w as f64 / hb as f64).max(1.0); + let ch = (h as f64 / vb as f64).max(1.0); + let mut p = Params::default(); + if ctx.params.b("sharpColors") { + let centre = |i: usize, c: f64| (((i as f64 / c).floor() + 0.5) * c) as i64 as f32; + let data: Vec = (0..w).map(|x| centre(x, cw)).chain((0..h).map(|y| centre(y, ch))).collect(); + p.u[0][0] = w as u32; + let buf = e.data(&data); + return run(e, "dst_mosaic_sharp", &p, b, Some(&buf)); + } + let (nx, ny) = ((w as f64 / cw).ceil() as usize, (h as f64 / ch).ceil() as usize); + let tx: Vec = (0..w).map(|x| ((x as f64 / cw) as usize).min(nx - 1)).collect(); + let ty: Vec = (0..h).map(|y| ((y as f64 / ch) as usize).min(ny - 1)).collect(); + // Tile bounds: tile i covers the columns with tx == i (tx is non-decreasing). + let bounds = |t: &[usize], n: usize| -> Vec { (0..=n).map(|i| t.partition_point(|&v| v < i) as f32).collect() }; + let mut data: Vec = tx.iter().chain(&ty).map(|&v| v as f32).collect(); + data.extend(bounds(&tx, nx)); + data.extend(bounds(&ty, ny)); + p.u[0] = [w as u32, h as u32, nx as u32, ny as u32]; + let buf = e.data(&data); + let rows = e.image(nx as u32, h as u32); + e.pixels("dst_mosaic_rows", &p, &b.img, None, &rows, Some(&buf)); + let tiles = e.image(nx as u32, ny as u32); + e.pixels("dst_mosaic_tiles", &p, &rows, None, &tiles, Some(&buf)); + let out = e.image(b.img.width, b.img.height); + e.pixels("dst_mosaic_out", &p, &tiles, None, &out, Some(&buf)); + b.img = out; + Some(b) +} + +fn find_edges(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let mut p = Params::default(); + p.u[0][0] = ctx.params.b("invert") as u32; + p.f[0][0] = ctx.params.f("blend") as f32 / 100.0; + run(e, "dst_findedges", &p, b, None) +} + +fn emboss(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let ang = ctx.params.f("direction").to_radians(); + let relief = ctx.params.f("relief") * b.scale; + let (dx, dy) = (ang.cos() * relief, -ang.sin() * relief); + let mut p = Params::default(); + p.f[0] = [dx as f32, dy as f32, ctx.params.f("contrast") as f32 / 100.0, ctx.params.f("blend") as f32 / 100.0]; + run(e, "dst_emboss", &p, b, None) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 9ebce209..b49dbcc6 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -149,3 +149,5 @@ mod tests_3d; mod tests_adjust; #[cfg(test)] mod tests_adv3d; +#[cfg(test)] +mod tests_fx_distort; diff --git a/crates/gpu/src/shaders/fx_distort.wgsl b/crates/gpu/src/shaders/fx_distort.wgsl index b3ce3463..a5493621 100644 --- a/crates/gpu/src/shaders/fx_distort.wgsl +++ b/crates/gpu/src/shaders/fx_distort.wgsl @@ -1,2 +1,554 @@ // GPU effects (distort family): see src/fx_distort.rs. Each entry point mirrors the CPU effect in // effectcraft-effects operation for operation; parameter layouts are documented per entry. +// Coordinates are pixel centres (x + 0.5, y + 0.5) in buffer pixels, as in util::remap. + +const DST_TAU: f32 = 6.2831855; +const DST_HALF_PI: f32 = 1.5707964; + +fn dst_centre(p: vec2) -> vec2 { + return vec2(f32(p.x) + 0.5, f32(p.y) + 0.5); +} + +fn dst_inside(p: vec2) -> bool { + let d = out_dims(); + return p.x < d.x && p.y < d.y; +} + +// ---------------------------------------------------------------- Twirl (distort::twirl) +// f[0] = (cx, cy, radius, angle in radians) +@compute @workgroup_size(16, 16) +fn dst_twirl(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let c = P.f[0].xy; + let r = P.f[0].z; + let dx = q.x - c.x; + let dy = q.y - c.y; + let d = sqrt(dx * dx + dy * dy); + var s = q; + if (d < r) { + let t = 1.0 - d / r; + let a = -P.f[0].w * t * t; + let sn = sin(a); + let co = cos(a); + s = vec2(c.x + dx * co - dy * sn, c.y + dx * sn + dy * co); + } + textureStore(out, p, sample_bilinear(src, s.x, s.y)); +} + +// ---------------------------------------------------------------- Bulge (distort::bulge) +// f[0] = (cx, cy, rx, ry); f[1] = (height, taper, pin, edge); f[2] = layer rect (x, y, w, h) +@compute @workgroup_size(16, 16) +fn dst_bulge(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let c = P.f[0].xy; + let rx = P.f[0].z; + let ry = P.f[0].w; + let dx = (q.x - c.x) / rx; + let dy = (q.y - c.y) / ry; + let d2 = dx * dx + dy * dy; + var s = q; + if (d2 < 1.0) { + var amt = P.f[1].x * pow(1.0 - d2, P.f[1].y) * 0.5; + if (P.f[1].z > 0.5) { + let l = P.f[2]; + let edge = P.f[1].w; + let ex = clamp(min(q.x - l.x - 0.5, l.x + l.z - 0.5 - q.x) / edge, 0.0, 1.0); + let ey = clamp(min(q.y - l.y - 0.5, l.y + l.w - 0.5 - q.y) / edge, 0.0, 1.0); + amt *= min(ex, ey); + } + let k = 1.0 - amt; + s = vec2(c.x + dx * rx * k, c.y + dy * ry * k); + } + textureStore(out, p, sample_bilinear(src, s.x, s.y)); +} + +// ---------------------------------------------------------------- Wave Warp (distort::wave_warp) + +fn dst_half_circle(u: f32) -> f32 { + let v = 2.0 * u - 1.0; + return sqrt(max(1.0 - v * v, 0.0)); +} + +fn dst_wave_noise(i: f32, seed: u32) -> f32 { + return hash_noise(bitcast(i32(i)), 0x5eedu, seed) * 2.0 - 1.0; +} + +// distort::wave_shape at the phase `cf + ci` in cycles (t / 2π), split into a fraction cf in +// [0, 2) and whole cycles ci so the fraction keeps full precision. +fn dst_wave_shape(kind: u32, cf: f32, ci: f32, seed: u32) -> f32 { + let ph = cf - floor(cf); + switch kind { + case 1u: { + return select(-1.0, 1.0, ph < 0.5); + } + case 2u: { + return 1.0 - 4.0 * abs(ph - 0.5); + } + case 3u: { + return 2.0 * ph - 1.0; + } + case 4u: { + if (ph < 0.5) { + return dst_half_circle(ph * 2.0); + } + return -dst_half_circle(ph * 2.0 - 1.0); + } + case 5u: { + return 2.0 * dst_half_circle(fract(ph * 2.0)) - 1.0; + } + case 6u: { + return 1.0 - 2.0 * dst_half_circle(fract(ph * 2.0)); + } + case 7u: { + return dst_wave_noise(floor(cf * 2.0) + 2.0 * ci, seed); + } + case 8u: { + let u = cf * 2.0; + let i = floor(u); + let f = u - i; + let s = f * f * (3.0 - 2.0 * f); + let n = i + 2.0 * ci; + return dst_wave_noise(n, seed) * (1.0 - s) + dst_wave_noise(n + 1.0, seed) * s; + } + default: { + return sin(ph * DST_TAU); + } + } +} + +// distort::wave_pin. +fn dst_wave_pin(pin: u32, x: f32, y: f32, r: vec4) -> f32 { + let u = clamp((x - r.x - 0.5) / max(r.z - 1.0, 1.0), 0.0, 1.0); + let v = clamp((y - r.y - 0.5) / max(r.w - 1.0, 1.0), 0.0, 1.0); + let horiz = clamp(min(v, 1.0 - v) * 2.0, 0.0, 1.0); + let vert = clamp(min(u, 1.0 - u) * 2.0, 0.0, 1.0); + switch pin { + case 1u: { return min(horiz, vert); } + case 2u: { return min(sqrt((u - 0.5) * (u - 0.5) + (v - 0.5) * (v - 0.5)) * 2.0, 1.0); } + case 3u: { return u; } + case 4u: { return v; } + case 5u: { return 1.0 - u; } + case 6u: { return 1.0 - v; } + case 7u: { return horiz; } + case 8u: { return vert; } + default: { return 1.0; } + } +} + +// u[0] = (wave type, pinning, seed, buffer width); f[0] = (dx, dy, height); f[2] = layer rect +// (x, y, w, h). data: the phase in cycles (along / width + phase / 2π) as per-column and per-row +// (fraction, whole cycles) pairs: fx (w), ix (w), fy (h), iy (h). +@compute @workgroup_size(16, 16) +fn dst_wavewarp(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let dx = P.f[0].x; + let dy = P.f[0].y; + let w = P.u[0].w; + let x = u32(p.x); + let y = u32(p.y); + let cf = data[x] + data[2u * w + y]; + let ci = data[w + x] + data[2u * w + u32(out_dims().y) + y]; + let disp = dst_wave_shape(P.u[0].x, cf, ci, P.u[0].z) * P.f[0].z * dst_wave_pin(P.u[0].y, q.x, q.y, P.f[2]); + textureStore(out, p, sample_bilinear(src, q.x + dy * disp, q.y - dx * disp)); +} + +// ---------------------------------------------------------------- Ripple (distort::ripple) +// u[0].x = asymmetric; f[0] = (cx, cy, radius, wave width); f[1] = (wave height, phase mod 2π) +@compute @workgroup_size(16, 16) +fn dst_ripple(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let c = P.f[0].xy; + let r = P.f[0].z; + let dx = q.x - c.x; + let dy = q.y - c.y; + let d = sqrt(dx * dx + dy * dy); + var s = q; + if (d < r && d != 0.0) { + let h = P.f[1].x; + let fall = 1.0 - d / r; + let arg = d / P.f[0].w * DST_TAU + P.f[1].y; + let o = sin(arg) * h * fall; + var t = 0.0; + if (P.u[0].x != 0u) { + t = cos(arg) * h * fall * 0.5; + } + let ux = dx / d; + let uy = dy / d; + s = vec2(q.x + ux * o - uy * t, q.y + uy * o + ux * t); + } + textureStore(out, p, sample_bilinear(src, s.x, s.y)); +} + +// ---------------------------------------------------------------- CC Lens (distort2::cc_lens) +// f[0] = (cx, cy, radius, convergence exponent) +@compute @workgroup_size(16, 16) +fn dst_cclens(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let c = P.f[0].xy; + let r = P.f[0].z; + let dx = q.x - c.x; + let dy = q.y - c.y; + let d = sqrt(dx * dx + dy * dy); + if (d >= r) { + textureStore(out, p, vec4(0.0)); + return; + } + var s = q; + if (d >= 1e-9) { + let g = powz(asin(min(d / r, 1.0)) / DST_HALF_PI, P.f[0].w); + let k = r * g / d; + s = vec2(c.x + dx * k, c.y + dy * k); + } + textureStore(out, p, sample_bilinear(src, s.x, s.y)); +} + +// ---------------------------------------------------------------- Turbulent Displace (distort2) + +// distort2::fbm. u[0].w = octave count; f[1].w = fraction of the last octave; f[3].y = falloff. +fn dst_fbm(u: f32, v: f32, z: f32, seed: u32) -> f32 { + let n = P.u[0].w; + let frac = P.f[1].w; + let falloff = P.f[3].y; + var sum = 0.0; + var norm = 0.0; + var amp = 1.0; + var f = 1.0; + for (var o = 0u; o < n; o++) { + var w = 1.0; + if (o + 1u == n && frac > 0.0) { + w = frac; + } + sum += value_noise(u * f, v * f, z + f32(o) * 5.17, seed + o * 101u) * amp * w; + norm += amp * w; + amp *= falloff; + f *= 2.0; + } + return sum / max(norm, 1e-6); +} + +// The centred noise `n(u, v, s)` (cross-faded over the evolution cycle when u[1].x is set). +// f[1] = (z, z - cycle, cycle position t). +fn dst_turb_n(u: f32, v: f32, s: u32) -> f32 { + let seed = P.u[0].z + s; + if (P.u[1].x != 0u) { + let t = P.f[1].z; + return dst_fbm(u, v, P.f[1].x, seed) * (1.0 - t) + dst_fbm(u, v, P.f[1].y, seed) * t - 0.5; + } + return dst_fbm(u, v, P.f[1].x, seed) - 0.5; +} + +// u[0] = (displacement kind, pinning, seed, octaves); u[1].x = cycle evolution; +// f[0] = (offset x, offset y, size, amount); f[1] = (z0, z1, t, last-octave fraction); +// f[2] = layer rect (x, y, w, h); f[3] = (edge, falloff) +@compute @workgroup_size(16, 16) +fn dst_turbulent(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let kind = P.u[0].x; + let amount = P.f[0].w; + let u = (q.x - P.f[0].x) / P.f[0].z; + let v = (q.y - P.f[0].y) / P.f[0].z; + var d = vec2(0.0); + switch kind { + case 1u, 2u, 4u, 5u: { + let e = 0.05; + let gx = (dst_turb_n(u + e, v, 0u) - dst_turb_n(u - e, v, 0u)) / (2.0 * e); + let gy = (dst_turb_n(u, v + e, 0u) - dst_turb_n(u, v - e, 0u)) / (2.0 * e); + let k = amount * 0.25; + if (kind == 1u || kind == 4u) { + d = vec2(gx * k, gy * k); + } else { + d = vec2(-gy * k, gx * k); + } + } + case 6u: { + d = vec2(0.0, dst_turb_n(u, v, 17u) * amount); + } + case 7u: { + d = vec2(dst_turb_n(u, v, 0u) * amount, 0.0); + } + case 8u: { + let a = dst_turb_n(u, v, 0u) * amount; + d = vec2(a, a); + } + default: { + d = vec2(dst_turb_n(u, v, 0u) * amount, dst_turb_n(u, v, 17u) * amount); + } + } + let r = P.f[2]; + let edge = P.f[3].x; + let l = clamp((q.x - r.x - 0.5) / edge, 0.0, 1.0); + let rr = clamp((r.x + r.z - 0.5 - q.x) / edge, 0.0, 1.0); + let t = clamp((q.y - r.y - 0.5) / edge, 0.0, 1.0); + let bt = clamp((r.y + r.w - 0.5 - q.y) / edge, 0.0, 1.0); + var k = 1.0; + switch P.u[0].y { + case 1u: { k = min(min(l, rr), min(t, bt)); } + case 2u: { k = min(t, bt); } + case 3u: { k = min(l, rr); } + case 4u: { k = l; } + case 5u: { k = t; } + case 6u: { k = rr; } + case 7u: { k = bt; } + default: {} + } + textureStore(out, p, sample_bilinear(src, q.x + d.x * k, q.y + d.y * k)); +} + +// ---------------------------------------------------------------- Displacement Map (distort2) + +// util::pick for a straight colour (source index in util::SRC_ORDER). +fn dst_pick(s: u32, c: vec3, a: f32) -> f32 { + switch s { + case 0u: { return c.x; } + case 1u: { return c.y; } + case 2u: { return c.z; } + case 3u: { return a; } + case 4u: { return luminance(c); } + case 5u: { return rgb_to_hsl(c).x; } + case 6u: { return rgb_to_hsl(c).z; } + case 7u: { return rgb_to_hsl(c).y; } + case 8u: { return 1.0; } + case 9u: { return 0.5; } + default: { return 0.0; } + } +} + +// u[0] = (horizontal source, vertical source, map mode: 0 self, 1 fitted layer (transparent +// outside), 2 tiled layer (edges repeat), flags: 1 wrap, 2 Center Map bounds); +// u[1] = (buffer width, buffer height); f[0] = (max horizontal, max vertical). +// data: per-column map x, per-row map y (the map layer's buffer pixels), per-column and +// per-row "inside the centred map" flags. aux = the map layer. +@compute @workgroup_size(16, 16) +fn dst_displace(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let w = P.u[1].x; + let h = P.u[1].y; + let x = u32(p.x); + let y = u32(p.y); + let px = textureLoad(src, p, 0); + let flags = P.u[0].w; + if ((flags & 2u) != 0u && (data[w + h + x] < 0.5 || data[2u * w + h + y] < 0.5)) { + textureStore(out, p, px); + return; + } + var m = px; + let mode = P.u[0].z; + if (mode == 1u) { + m = sample_bilinear(aux, data[x], data[w + y]); + } else if (mode == 2u) { + m = sample_bilinear_clamped(aux, data[x], data[w + y]); + } + var c = vec3(0.0); + let a = m.w; + if (a > 1e-6) { + c = m.xyz / a; + } + let al = max(a, 0.0); + var dx = 0.0; + var dy = 0.0; + if (P.u[0].x != 10u) { + dx = (dst_pick(P.u[0].x, c, al) - 0.5) * 2.0 * P.f[0].x; + } + if (P.u[0].y != 10u) { + dy = (dst_pick(P.u[0].y, c, al) - 0.5) * 2.0 * P.f[0].y; + } + if (dx == 0.0 && dy == 0.0) { + textureStore(out, p, px); + return; + } + var sx = f32(p.x) + 0.5 + dx; + var sy = f32(p.y) + 0.5 + dy; + if ((flags & 1u) != 0u) { + let fw = f32(w); + let fh = f32(h); + sx = sx - floor(sx / fw) * fw; + sy = sy - floor(sy / fh) * fh; + textureStore(out, p, sample_bilinear_clamped(src, sx, sy)); + } else { + textureStore(out, p, sample_bilinear(src, sx, sy)); + } +} + +// ---------------------------------------------------------------- Mesh Warp (distort2::grid_warp) +// u[0] = (row starts offset, list offset). data: 12 floats per triangle (the barycentric +// coefficients k11 k12 k21 k22 relative to vertex c, c, then the source points a, b, c), then +// per output row the start of its triangle list (height + 1 entries), then the lists (triangle +// indices in the CPU's bucket order). +@compute @workgroup_size(16, 16) +fn dst_mesh(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let rs = P.u[0].x; + let lo = P.u[0].y; + let i0 = u32(data[rs + u32(p.y)]); + let i1 = u32(data[rs + u32(p.y) + 1u]); + var cur = vec4(0.0); + for (var i = i0; i < i1; i++) { + if (cur.w > 0.0) { + break; + } + let b = u32(data[lo + i]) * 12u; + let ex = q.x - data[b + 4u]; + let ey = q.y - data[b + 5u]; + let l1 = data[b] * ex + data[b + 1u] * ey; + let l2 = data[b + 2u] * ex + data[b + 3u] * ey; + let l3 = 1.0 - l1 - l2; + // The CPU tests against -1e-9 in f64; f32 rounding needs a wider margin so pixels on + // shared edges are not dropped by both triangles. + let e = -1e-5; + if (l1 < e || l2 < e || l3 < e) { + continue; + } + let sx = data[b + 6u] * l1 + data[b + 8u] * l2 + data[b + 10u] * l3; + let sy = data[b + 7u] * l1 + data[b + 9u] * l2 + data[b + 11u] * l3; + cur = sample_bilinear(src, sx, sy); + } + textureStore(out, p, cur); +} + +// ---------------------------------------------------------------- Mosaic (misc::mosaic) +// data: column tile index (w), row tile index (h), tile column bounds (nx + 1), tile row bounds +// (ny + 1); u[0] = (w, h, nx, ny). + +// Row sums of each tile's columns: out is nx × h. +@compute @workgroup_size(16, 16) +fn dst_mosaic_rows(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let w = P.u[0].x; + let h = P.u[0].y; + let xb = w + h + u32(p.x); + var s = vec4(0.0); + for (var x = u32(data[xb]); x < u32(data[xb + 1u]); x++) { + s += textureLoad(src, vec2(i32(x), p.y), 0); + } + textureStore(out, p, s); +} + +// Tile averages: out is nx × ny (src = the row sums). +@compute @workgroup_size(16, 16) +fn dst_mosaic_tiles(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let w = P.u[0].x; + let h = P.u[0].y; + let xb = w + h + u32(p.x); + let yb = w + h + P.u[0].z + 1u + u32(p.y); + let y0 = u32(data[yb]); + let y1 = u32(data[yb + 1u]); + var s = vec4(0.0); + for (var y = y0; y < y1; y++) { + s += textureLoad(src, vec2(p.x, i32(y)), 0); + } + let n = f32((u32(data[xb + 1u]) - u32(data[xb])) * (y1 - y0)); + textureStore(out, p, s / max(n, 1.0)); +} + +// Each pixel takes its tile's average (src = the tile averages). +@compute @workgroup_size(16, 16) +fn dst_mosaic_out(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let tx = i32(data[u32(p.x)]); + let ty = i32(data[P.u[0].x + u32(p.y)]); + textureStore(out, p, textureLoad(src, vec2(tx, ty), 0)); +} + +// Sharp Colors: data = per-column / per-row source pixel; u[0].x = w. +@compute @workgroup_size(16, 16) +fn dst_mosaic_sharp(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let sx = i32(data[u32(p.x)]); + let sy = i32(data[P.u[0].x + u32(p.y)]); + textureStore(out, p, tex_get_clamped(src, sx, sy)); +} + +// ---------------------------------------------------------------- Find Edges / Emboss (misc) + +fn dst_blend_with(orig: vec4, fx: vec4, k: f32) -> vec4 { + return fx + (orig - fx) * k; +} + +fn dst_lum_at(x: i32, y: i32) -> f32 { + return luminance(tex_get_clamped(src, x, y).xyz); +} + +// u[0].x = invert; f[0].x = blend with original. +@compute @workgroup_size(16, 16) +fn dst_findedges(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let x = p.x; + let y = p.y; + let gx = dst_lum_at(x + 1, y - 1) + 2.0 * dst_lum_at(x + 1, y) + dst_lum_at(x + 1, y + 1) - dst_lum_at(x - 1, y - 1) - 2.0 * dst_lum_at(x - 1, y) + - dst_lum_at(x - 1, y + 1); + let gy = dst_lum_at(x - 1, y + 1) + 2.0 * dst_lum_at(x, y + 1) + dst_lum_at(x + 1, y + 1) - dst_lum_at(x - 1, y - 1) - 2.0 * dst_lum_at(x, y - 1) + - dst_lum_at(x + 1, y - 1); + let e = min(sqrt(gx * gx + gy * gy), 1.0); + var v = 1.0 - e; + if (P.u[0].x != 0u) { + v = e; + } + let px = textureLoad(src, p, 0); + let a = px.w; + textureStore(out, p, dst_blend_with(px, vec4(v * a, v * a, v * a, a), P.f[0].x)); +} + +// f[0] = (dx, dy, contrast, blend with original). +@compute @workgroup_size(16, 16) +fn dst_emboss(@builtin(global_invocation_id) gid: vec3) { + let p = vec2(gid.xy); + if (!dst_inside(p)) { + return; + } + let q = dst_centre(p); + let d = P.f[0].xy; + let a = sample_bilinear_clamped(src, q.x + d.x, q.y + d.y); + let c = sample_bilinear_clamped(src, q.x - d.x, q.y - d.y); + let l = luminance(a.xyz) - luminance(c.xyz); + let v = clamp(0.5 + l * P.f[0].z, 0.0, 1.0); + let px = textureLoad(src, p, 0); + let al = px.w; + textureStore(out, p, dst_blend_with(px, vec4(v * al, v * al, v * al, al), P.f[0].w)); +} diff --git a/crates/gpu/src/tests_fx_distort.rs b/crates/gpu/src/tests_fx_distort.rs new file mode 100644 index 00000000..9732f99c --- /dev/null +++ b/crates/gpu/src/tests_fx_distort.rs @@ -0,0 +1,119 @@ +//! Distort / stylize GPU effects vs the CPU effects (the oracle): see `fx_distort.rs`. + +use effectcraft_keyframe::Value; +use effectcraft_project::BitDepth; +use effectcraft_render::RenderOpts; +use effectcraft_time::Tick; + +use crate::tests::{Scene, check, compare_at, effect_case, n, opts, set}; + +fn e(v: u32) -> Value { + Value::Enum(v) +} + +fn on() -> Value { + Value::Bool(true) +} + +#[test] +fn twirl_bulge_ripple_wave_warp() { + effect_case("ec.distort.twirl", &[("angle", n(120.0)), ("radius", n(40.0))]); + effect_case("ec.distort.twirl", &[("angle", n(-400.0)), ("radius", n(70.0)), ("center", Value::Vec2([25.0, 18.0]))]); + effect_case("ec.distort.bulge", &[("hradius", n(30.0)), ("vradius", n(20.0)), ("height", n(1.8))]); + effect_case("ec.distort.bulge", &[("height", n(-2.5)), ("taperRadius", n(60.0)), ("pinAllEdges", on()), ("center", Value::Vec2([30.0, 25.0]))]); + effect_case("ec.distort.ripple", &[("radius", n(80.0)), ("waveWidth", n(9.0)), ("waveHeight", n(4.0))]); + effect_case("ec.distort.ripple", &[("conversion", e(1)), ("phase", n(70.0)), ("speed", n(3.0)), ("center", Value::Vec2([20.0, 30.0]))]); + effect_case("ec.distort.wavewarp", &[("height", n(6.0)), ("width", n(25.0)), ("direction", n(30.0))]); + effect_case("ec.distort.wavewarp", &[("waveType", e(2)), ("pinning", e(1)), ("phase", n(400.0)), ("speed", n(-2.0))]); + effect_case("ec.distort.wavewarp", &[("waveType", e(8)), ("pinning", e(2)), ("height", n(-5.0)), ("direction", n(200.0))]); + effect_case("ec.distort.wavewarp", &[("waveType", e(4)), ("pinning", e(7)), ("width", n(33.0))]); +} + +#[test] +fn turbulent_displace_and_cc_lens() { + effect_case("ec.distort.turbulentdisplace", &[("size", n(30.0)), ("amount", n(20.0))]); + effect_case( + "ec.distort.turbulentdisplace", + &[("displacement", e(1)), ("complexity", n(3.5)), ("evolution", n(90.0)), ("size", n(25.0)), ("pinning", e(2)), ("resizeLayer", on())], + ); + effect_case( + "ec.distort.turbulentdisplace", + &[ + ("displacement", e(5)), + ("evolutionOptions/cycleEvolution", on()), + ("evolutionOptions/cycle", n(2.0)), + ("evolution", n(500.0)), + ("evolutionOptions/randomSeed", n(7.0)), + ("pinning", e(0)), + ("amount", n(-30.0)), + ], + ); + effect_case("ec.distort.turbulentdisplace", &[("displacement", e(8)), ("size", n(15.0)), ("pinning", e(6))]); + effect_case("ec.distort.cclens", &[("size", n(70.0))]); + effect_case("ec.distort.cclens", &[("size", n(45.0)), ("convergence", n(60.0)), ("center", Value::Vec2([30.0, 20.0]))]); + effect_case("ec.distort.cclens", &[("convergence", n(-70.0))]); +} + +#[test] +fn mesh_warp() { + // 2 rows × 3 columns: 12 vertices. + let mesh = "0,0 0,0 0,0 0,0 0,0 6,-4 -5,3 0,0 0,0 0,0 0,0 0,0"; + effect_case("ec.distort.meshwarp", &[("rows", n(2.0)), ("columns", n(3.0)), ("quality", n(3.0)), ("mesh", Value::Str(mesh.into()))]); + let mesh = "-4,-3 2,0 8,5 0,-2 9,9 1,0 -3,4 2,2 4,-6"; + effect_case("ec.distort.meshwarp", &[("rows", n(2.0)), ("columns", n(2.0)), ("quality", n(5.0)), ("mesh", Value::Str(mesh.into()))]); +} + +#[test] +fn mosaic_find_edges_emboss() { + effect_case("ec.stylize.mosaic", &[]); + effect_case("ec.stylize.mosaic", &[("horizontal", n(7.0)), ("vertical", n(3.0))]); + effect_case("ec.stylize.mosaic", &[("horizontal", n(13.0)), ("vertical", n(9.0)), ("sharpColors", on())]); + effect_case("ec.stylize.findedges", &[]); + effect_case("ec.stylize.findedges", &[("invert", on()), ("blend", n(30.0))]); + effect_case("ec.stylize.emboss", &[]); + effect_case("ec.stylize.emboss", &[("direction", n(160.0)), ("relief", n(4.0)), ("contrast", n(250.0)), ("blend", n(20.0))]); +} + +#[test] +fn displacement_map_self() { + effect_case("ec.distort.displacementmap", &[("maxHorizontal", n(8.0)), ("maxVertical", n(-5.0))]); + effect_case("ec.distort.displacementmap", &[("useForHorizontal", e(4)), ("useForVertical", e(3)), ("wrapPixelsAround", on())]); + effect_case( + "ec.distort.displacementmap", + &[("useForHorizontal", e(10)), ("useForVertical", e(6)), ("maxVertical", n(12.0)), ("expandOutput", Value::Bool(false))], + ); +} + +/// Displacement Map reading another layer (Center / Stretch / Tile), composited. +#[test] +fn displacement_map_layer() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + for (behavior, wrap) in [(0, false), (1, false), (2, true), (2, false)] { + let mut s = Scene::new(depth); + let bg = s.solid([0.15, 0.1, 0.2], 97, 61); + s.push(bg); + let mut map = s.footage(50, 36); + map.switches.video = false; + let mid = s.push(map); + let mut l = s.footage(70, 44); + s.effect( + &mut l, + "ec.distort.displacementmap", + &[ + ("displacementMapLayer", Value::Layer(Some(mid.0))), + ("useForHorizontal", e(0)), + ("useForVertical", e(4)), + ("maxHorizontal", n(9.0)), + ("maxVertical", n(6.0)), + ("displacementMapBehavior", e(behavior)), + ("wrapPixelsAround", Value::Bool(wrap)), + ], + ); + set(&mut l, "transform/rotation", Value::Scalar(8.0)); + s.push(l); + let label = format!("displacement map {behavior} {wrap} {depth:?}"); + check(&label, compare_at(&s, opts(), Tick::ZERO), 0.0); + check(&format!("{label} half"), compare_at(&s, RenderOpts { scale: 0.5, ..opts() }, Tick::ZERO), 0.0); + } + } +} From 0e3b354520ec7b62979f7a0362c7f1485beff9b9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:44:36 -0400 Subject: [PATCH 08/36] M12.7: GPU particles (CC Particle World, CC Particle Systems II, Particle Playground cannon): compute-shader simulation with GPU checkpoints, CPU as the oracle Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/lib.rs | 8 + crates/effects/src/psim.rs | 106 ++++++++++ crates/effects/src/sim.rs | 88 ++++++++- crates/effects/src/sim3.rs | 76 +++++++- crates/gpu/src/context.rs | 5 + crates/gpu/src/lib.rs | 13 ++ crates/gpu/src/particles.rs | 209 ++++++++++++++++++++ crates/gpu/src/shaders/particles.wgsl | 270 ++++++++++++++++++++++++++ crates/gpu/src/tests_particles.rs | 155 +++++++++++++++ crates/render/src/lib.rs | 8 + 10 files changed, 925 insertions(+), 13 deletions(-) create mode 100644 crates/effects/src/psim.rs create mode 100644 crates/gpu/src/particles.rs create mode 100644 crates/gpu/src/shaders/particles.wgsl create mode 100644 crates/gpu/src/tests_particles.rs diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index a5b4f346..8840a383 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -40,6 +40,7 @@ mod ocio; pub mod paint; mod perspective; mod perspective2; +pub mod psim; pub mod puppet; pub mod roto; mod sim; @@ -67,6 +68,8 @@ use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; pub use effectcraft_raster::{AuxChannels, Image}; pub use misc::{INVERT_ALPHA, INVERT_CHANNELS}; +pub use sim::particle_state; +pub use sim3::playground_state; /// Effect categories in Effects & Presets order. pub const CATEGORIES: &[&str] = &[ @@ -260,6 +263,11 @@ pub trait EffectHost: Sync { fn aux(&self) -> Option> { None } + /// A particle simulation backend (GPU particles; see [`psim`]). `None` = simulate on the + /// CPU. + fn particles(&self) -> Option<&dyn psim::ParticleSim> { + None + } } /// Extra context the renderer may supply (all optional; `Default` is "nothing known"). diff --git a/crates/effects/src/psim.rs b/crates/effects/src/psim.rs new file mode 100644 index 00000000..aabbcdce --- /dev/null +++ b/crates/effects/src/psim.rs @@ -0,0 +1,106 @@ +//! Pluggable particle simulation (GPU particles, M12.7). +//! +//! The stepped particle systems (CC Particle World, CC Particle Systems II and Particle +//! Playground's Cannon without interacting forces) evolve every particle independently: births +//! come from a rate (the same `carry` arithmetic as the CPU step), and each particle integrates +//! its own motion with the same fixed step. An accelerator ([`ParticleSim`], reached through +//! [`crate::EffectHost::particles`]) can therefore simulate all of them in parallel and return +//! the state the CPU's `SimCache` would reach. The CPU stays the oracle: anything the backend +//! declines (interacting forces, caps that would drop births) simulates on the CPU. + +/// CC Particle World / CC Particle Systems II physics (see `sim::Phys`), with the per-step and +/// per-spawn constants already evaluated on the CPU (bit-identical to the CPU's). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EngineDesc { + pub life: f32, + pub producer: [f32; 3], + pub radius: [f32; 3], + /// Animation (0 Explosive … 9 Fractal Uni). + pub anim: u32, + pub speed: f32, + pub gravity: [f32; 3], + /// Normalised direction axis. + pub axis: [f32; 3], + pub extra: f32, + /// Cone Axis spread factor `|tan(extra angle / 2)|` (≤ 20). + pub cone: f32, + /// Velocity damping per step `exp(−drag · dt)`. + pub damp: f32, + pub seed: u32, + /// 2D systems keep z = 0. + pub flat: bool, +} + +/// Particle Playground's Cannon with gravity only (see `sim3::Pg`). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CannonDesc { + pub pos: [f32; 2], + pub barrel_angle: f32, + pub barrel_radius: f32, + pub dir_spread: f32, + pub vel: f32, + pub vel_spread: f32, + pub gravity: [f32; 2], + pub grav_spread: f32, + pub seed: u32, + /// Particles leaving `[x0, y0, x1, y1]` are removed. + pub bounds: [f32; 4], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ParticleSystem { + Engine(EngineDesc), + Cannon(CannonDesc), +} + +/// One simulation request: the state after `steps` steps of system `key`. +#[derive(Clone, Debug)] +pub struct SimRequest<'a> { + /// Identifies the system (parameters, seed); checkpoints are reused per key. + pub key: u64, + pub steps: u64, + /// Birth step of every particle id born before `steps` (ascending). + pub births: &'a [u32], + pub system: ParticleSystem, +} + +/// A live particle (ascending ids, as the CPU state keeps them). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SimParticle { + pub p: [f32; 3], + pub v: [f32; 3], + pub age: f32, + pub life: f32, + pub rnd: f32, + pub id: u32, +} + +/// A particle simulation backend (the GPU). +pub trait ParticleSim: Send + Sync { + /// The live particles after `req.steps` steps, or `None` (simulate on the CPU). + fn simulate(&self, req: &SimRequest) -> Option>; +} + +/// Birth step of every particle born in the first `steps` steps at `rate` particles per second +/// (the CPU step's `carry` arithmetic, exactly). `None` when more than `cap` would be born (the +/// CPU drops births at its live-particle cap, which an independent simulation can't know). +pub fn births(rate: f64, steps: u64, cap: usize) -> Option> { + let per = rate / SPS; + if per * steps as f64 > cap as f64 + 2.0 { + return None; + } + let mut out = Vec::new(); + let mut carry = 0.0f64; + for i in 0..steps { + carry += per; + let n = carry.floor() as u32; + carry -= n as f64; + for _ in 0..n { + out.push(i as u32); + } + } + (out.len() <= cap).then_some(out) +} + +/// Steps per second of the stepped simulations. +pub const SPS: f64 = crate::sim::SPS; diff --git a/crates/effects/src/sim.rs b/crates/effects/src/sim.rs index 7f90af6b..18beeb76 100644 --- a/crates/effects/src/sim.rs +++ b/crates/effects/src/sim.rs @@ -818,6 +818,60 @@ impl Phys { } } +impl Phys { + /// The backend description (per-step and per-spawn constants evaluated here, in f32). + pub(crate) fn desc(&self) -> crate::psim::EngineDesc { + let mut drag = self.resistance; + if self.anim == 3 { + drag += 3.0 * self.extra.max(0.1); + } + let dt = (1.0 / SPS) as f32; + crate::psim::EngineDesc { + life: self.life, + producer: self.producer, + radius: self.radius, + anim: self.anim, + speed: self.speed, + gravity: self.gravity, + axis: norm3(self.axis), + extra: self.extra, + cone: (self.extra_angle.to_radians() * 0.5).tan().abs().min(20.0), + damp: (-drag * dt).exp(), + seed: self.seed, + flat: self.flat, + } + } +} + +/// The state at layer time `ctx.time` from the host's particle backend (GPU particles), or +/// `None` (simulate on the CPU). +fn accelerated(ctx: &EffectCtx, key: u64, phys: &Phys) -> Option> { + let backend = ctx.env.host?.particles()?; + let steps = steps_at(ctx.time); + let births = crate::psim::births(phys.rate, steps, phys.max)?; + let req = crate::psim::SimRequest { key, steps, births: &births, system: crate::psim::ParticleSystem::Engine(phys.desc()) }; + let parts = backend.simulate(&req)?; + Some(std::sync::Arc::new(PState { + parts: parts.into_iter().map(|q| Particle { p: q.p, v: q.v, age: q.age, life: q.life, rnd: q.rnd, id: q.id }).collect(), + carry: 0.0, + next_id: births.len() as u32, + })) +} + +/// The live particles of a stepped particle effect (`ec.sim.ccparticleworld`, +/// `ec.sim.ccparticlesystems2`) at `ctx.time`: from the host's particle backend when it has +/// one (GPU particles), else from the CPU simulation. For comparing the two. +pub fn particle_state(id: &str, ctx: &EffectCtx) -> Option> { + let (phys, cache, salt) = match id { + "ec.sim.ccparticleworld" => (pw_phys(ctx), &PW_CACHE, 1), + "ec.sim.ccparticlesystems2" => (ps2_phys(ctx), &PS2_CACHE, 2), + _ => return None, + }; + let key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, salt); + let st = accelerated(ctx, key, &phys).unwrap_or_else(|| simulate(cache, key, ctx.time, &phys)); + Some(st.parts.iter().map(|q| crate::psim::SimParticle { p: q.p, v: q.v, age: q.age, life: q.life, rnd: q.rnd, id: q.id }).collect()) +} + /// Run a particle system to layer time `t` through `cache`. pub(crate) fn simulate(cache: &SimCache, key: u64, t: f64, phys: &Phys) -> std::sync::Arc { cache.run(key, steps_at(t), PState::default, |st, i| phys.step(st, i)) @@ -859,10 +913,10 @@ fn type_sprite(kind: u32, x: f32, y: f32, r: f32, c: [f32; 4], vx: f32, vy: f32, static PW_CACHE: SimCache = SimCache::new(6); -fn particle_world(ctx: &EffectCtx, mut b: Buf) -> Buf { +/// CC Particle World's physics. +fn pw_phys(ctx: &EffectCtx) -> Phys { let pr = ctx.params; - let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); - let phys = Phys { + Phys { rate: pr.f("birthRate").max(0.0) * 120.0, life: pr.f("longevity").max(0.0) as f32, producer: [pr.f("producer/producerX") as f32, pr.f("producer/producerY") as f32, pr.f("producer/producerZ") as f32], @@ -882,11 +936,18 @@ fn particle_world(ctx: &EffectCtx, mut b: Buf) -> Buf { seed: (pr.f("extras/randomSeed") as u32).wrapping_mul(0x9e3779b9) ^ ctx.seed, max: 40_000, flat: false, - }; + } +} + +fn particle_world(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + let phys = pw_phys(ctx); if phys.rate <= 0.0 && ctx.time <= 0.0 { return b; } - let st = simulate(&PW_CACHE, params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 1), ctx.time, &phys); + let key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 1); + let st = accelerated(ctx, key, &phys).unwrap_or_else(|| simulate(&PW_CACHE, key, ctx.time, &phys)); let kind = pr.e("particle/particleType"); let birth = pr.color("particle/birthColor"); let death = pr.color("particle/deathColor"); @@ -936,13 +997,14 @@ fn particle_world(ctx: &EffectCtx, mut b: Buf) -> Buf { static PS2_CACHE: SimCache = SimCache::new(6); -fn particle_systems2(ctx: &EffectCtx, mut b: Buf) -> Buf { +/// CC Particle Systems II's physics. +fn ps2_phys(ctx: &EffectCtx) -> Phys { let pr = ctx.params; let lh = ctx.layer_size[1] as f32; let unit = lh.max(1.0); let pos = pr.v2("producer/position"); let dir = (pr.f("physics/direction") as f32).to_radians(); - let phys = Phys { + Phys { rate: pr.f("birthRate").max(0.0) * 60.0, life: pr.f("longevity").max(0.0) as f32, producer: [pos[0] as f32 / unit, pos[1] as f32 / unit, 0.0], @@ -958,8 +1020,16 @@ fn particle_systems2(ctx: &EffectCtx, mut b: Buf) -> Buf { seed: (pr.f("randomSeed") as u32).wrapping_mul(0x85ebca6b) ^ ctx.seed, max: 40_000, flat: true, - }; - let st = simulate(&PS2_CACHE, params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 2), ctx.time, &phys); + } +} + +fn particle_systems2(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let lh = ctx.layer_size[1] as f32; + let unit = lh.max(1.0); + let phys = ps2_phys(ctx); + let key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 2); + let st = accelerated(ctx, key, &phys).unwrap_or_else(|| simulate(&PS2_CACHE, key, ctx.time, &phys)); let kind = pr.e("particle/particleType"); let birth = pr.color("particle/birthColor"); let death = pr.color("particle/deathColor"); diff --git a/crates/effects/src/sim3.rs b/crates/effects/src/sim3.rs index 5560917a..f7425df7 100644 --- a/crates/effects/src/sim3.rs +++ b/crates/effects/src/sim3.rs @@ -231,6 +231,38 @@ impl Pg { static PG_CACHE: SimCache = SimCache::new(6); +/// Particle Playground cannons with gravity only, simulated by the host's particle backend. +fn pg_accelerated(ctx: &EffectCtx, key: u64, pg: &Pg) -> Option> { + let backend = ctx.env.host?.particles()?; + let steps = steps_at(ctx.time); + let births = crate::psim::births(pg.rate, steps, 60_000)?; + let desc = crate::psim::CannonDesc { + pos: pg.pos, + barrel_angle: pg.barrel_angle, + barrel_radius: pg.barrel_radius, + dir_spread: pg.dir_spread, + vel: pg.vel, + vel_spread: pg.vel_spread, + gravity: pg.gravity, + grav_spread: pg.grav_spread, + seed: pg.seed, + bounds: pg.bounds, + }; + let req = crate::psim::SimRequest { key, steps, births: &births, system: crate::psim::ParticleSystem::Cannon(desc) }; + let parts = backend.simulate(&req)?; + Some(std::sync::Arc::new(PgState { + parts: parts + .into_iter() + .map(|q| { + let born = pg.spawn(q.id); + PgParticle { p: [q.p[0], q.p[1]], v: [q.v[0], q.v[1]], ..born } + }) + .collect(), + carry: 0.0, + next_id: births.len() as u32, + })) +} + fn image_key(img: &Image) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -244,14 +276,15 @@ fn image_key(img: &Image) -> u64 { h.finish() } -fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { +/// Particle Playground's producers and forces. +fn pg_of(ctx: &EffectCtx) -> Pg { let pr = ctx.params; let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); let grav_dir = (pr.f("gravity/gravityDirection") as f32).to_radians(); let gf = pr.f("gravity/gravityForce") as f32; let wall = (pr.f("wall/wallBoundary").round() as usize).checked_sub(1).and_then(|i| ctx.env.masks.get(i)).map(|m| m.points.clone()); let mapper = if pr.b("mapperEnabled") { ctx.layer_param("persistentPropertyMapper/useLayerAsMap", true).map(|lp| MapImg::from(ctx, lp)) } else { None }; - let pg = Pg { + Pg { cannon: pr.b("cannonEnabled"), pos: { pr.v2("cannon/cannonPosition").map(|v| v as f32) }, barrel_angle: pr.f("cannon/barrelAngle") as f32, @@ -275,7 +308,34 @@ fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { ], seed: (pr.f("randomSeed") as u32).wrapping_mul(0x9e37_79b9) ^ ctx.seed, bounds: [-lw * 2.0, -lh * 2.0, lw * 3.0, lh * 3.0], - }; + } +} + +/// The live particles of a Particle Playground whose cannon is the only producer and force +/// besides gravity, at `ctx.time`: from the host's particle backend when it has one (GPU +/// particles), else from the CPU simulation (`None` for other set-ups). For comparing the two. +pub fn playground_state(ctx: &EffectCtx) -> Option> { + let pg = pg_of(ctx); + let pr = ctx.params; + let grid = pr.b("gridEnabled") && pr.f("grid/particlesAcross").round() > 0.0 && pr.f("grid/particlesDown").round() > 0.0; + let exploder = pr.b("exploderEnabled") && pr.get("layerExploder/explodeLayer").and_then(Value::as_layer).is_some(); + if !pg.cannon || grid || exploder || (pg.repel != 0.0 && pg.repel_radius > 0.0) || pg.wall.is_some() || pg.mapper.is_some() { + return None; + } + let key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 0x7067); + let st = pg_accelerated(ctx, key, &pg).unwrap_or_else(|| PG_CACHE.run(key, steps_at(ctx.time), PgState::default, |st, _| pg.step(st))); + Some( + st.parts + .iter() + .map(|q| crate::psim::SimParticle { p: [q.p[0], q.p[1], 0.0], v: [q.v[0], q.v[1], 0.0], age: 0.0, life: 0.0, rnd: 0.0, id: q.id }) + .collect(), + ) +} + +fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + let pg = pg_of(ctx); // Initial particles: Grid and Layer Exploder. let grid_on = pr.b("gridEnabled"); let across = pr.f("grid/particlesAcross").max(0.0).round() as u32; @@ -347,7 +407,15 @@ fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { } st }; - let st = PG_CACHE.run(key, steps_at(ctx.time), init, |st, _| pg.step(st)); + // A cannon with gravity only: every particle moves on its own (GPU particles). + let independent = pg.cannon + && !(grid_on && across > 0 && down > 0) + && exploder.is_none() + && !(pg.repel != 0.0 && pg.repel_radius > 0.0) + && pg.wall.is_none() + && pg.mapper.is_none(); + let gpu = if independent { pg_accelerated(ctx, key, &pg) } else { None }; + let st = gpu.unwrap_or_else(|| PG_CACHE.run(key, steps_at(ctx.time), init, |st, _| pg.step(st))); let s = b.scale as f32; let sprites: Vec = st .parts diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index e7cda93e..7dd14ba8 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -102,6 +102,9 @@ pub struct GpuContext { uploads: Mutex, /// Advanced 3D render pipelines (built on first use). pub(crate) adv3d: std::sync::OnceLock, + /// GPU particle pipeline (built on first use) and simulation checkpoints. + pub(crate) particles: crate::particles::PipesCell, + pub(crate) particle_states: crate::particles::StatesCell, } fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry { @@ -263,6 +266,8 @@ impl GpuContext { dummy_buf, uploads: Mutex::new(Uploads::default()), adv3d: std::sync::OnceLock::new(), + particles: Default::default(), + particle_states: Default::default(), }) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 9ebce209..83f27b26 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,6 +29,7 @@ mod fx_color; mod fx_distort; mod fx_generate; mod ops; +mod particles; mod walk; use std::sync::Arc; @@ -139,6 +140,16 @@ impl Accelerator for Gpu { fn raster_3d(&self, scene: &effectcraft_render::three_d::adv::Scene) -> Option { adv3d::render(&self.ctx, scene) } + + fn particles(&self) -> Option<&dyn effectcraft_effects::psim::ParticleSim> { + self.ctx.can_readback().then_some(self as &dyn effectcraft_effects::psim::ParticleSim) + } +} + +impl effectcraft_effects::psim::ParticleSim for Gpu { + fn simulate(&self, req: &effectcraft_effects::psim::SimRequest) -> Option> { + particles::simulate(&self.ctx, req) + } } #[cfg(test)] @@ -149,3 +160,5 @@ mod tests_3d; mod tests_adjust; #[cfg(test)] mod tests_adv3d; +#[cfg(test)] +mod tests_particles; diff --git a/crates/gpu/src/particles.rs b/crates/gpu/src/particles.rs new file mode 100644 index 00000000..669a4ba0 --- /dev/null +++ b/crates/gpu/src/particles.rs @@ -0,0 +1,209 @@ +//! GPU particles (`particles.wgsl`): the stepped particle systems of `effectcraft-effects` +//! ([`effectcraft_effects::psim`]) simulated with one invocation per particle. States stay on +//! the GPU as checkpoints (per system key, like the CPU's `SimCache`), so playback advances from +//! the previous frame's state instead of simulating from layer time 0. + +use std::sync::{Mutex, OnceLock}; + +use effectcraft_effects::psim::{ParticleSystem, SimParticle, SimRequest}; +use wgpu::util::DeviceExt; + +use crate::context::GpuContext; + +/// Bytes of one particle state (`PState` in the shader). +const STATE_BYTES: u64 = 48; +/// Checkpoints kept (all keys). +const CHECKPOINTS: usize = 24; + +pub(crate) struct Pipes { + bgl: wgpu::BindGroupLayout, + pipe: wgpu::ComputePipeline, +} + +struct Checkpoint { + key: u64, + steps: u64, + n: u32, + buf: wgpu::Buffer, + used: u64, +} + +#[derive(Default)] +pub(crate) struct Checkpoints { + list: Vec, + clock: u64, +} + +fn pipes(g: &GpuContext) -> &Pipes { + g.particles.get_or_init(|| { + let d = &g.device; + let src = [include_str!("shaders/common.wgsl"), include_str!("shaders/kernels.wgsl"), include_str!("shaders/particles.wgsl")].concat(); + let module = + d.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("effectcraft particles"), source: wgpu::ShaderSource::Wgsl(src.into()) }); + let buffer = |binding: u32, ty: wgpu::BufferBindingType| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None }, + count: None, + }; + let bgl = d.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effectcraft particles"), + entries: &[ + buffer(0, wgpu::BufferBindingType::Uniform), + buffer(1, wgpu::BufferBindingType::Storage { read_only: true }), + buffer(2, wgpu::BufferBindingType::Storage { read_only: true }), + buffer(3, wgpu::BufferBindingType::Storage { read_only: false }), + ], + }); + let layout = d.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effectcraft particles"), + bind_group_layouts: &[None, Some(&bgl)], + immediate_size: 0, + }); + let pipe = d.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("particles"), + layout: Some(&layout), + module: &module, + entry_point: Some("particles"), + compilation_options: Default::default(), + cache: None, + }); + Pipes { bgl, pipe } + }) +} + +/// The shader's parameter block. +fn uniforms(req: &SimRequest, n: u32, s0: u64, prev_n: u32) -> Vec { + let dt = (1.0 / effectcraft_effects::psim::SPS) as f32; + let mut u = [n, s0 as u32, req.steps as u32, 0]; + let u2; + let mut f = [[0.0f32; 4]; 6]; + match &req.system { + ParticleSystem::Engine(e) => { + u2 = [e.anim, e.flat as u32, e.seed, prev_n]; + f[0] = [e.life, e.speed, e.extra, e.cone]; + f[1] = [e.producer[0], e.producer[1], e.producer[2], 0.0]; + f[2] = [e.radius[0], e.radius[1], e.radius[2], 0.0]; + f[3] = [e.gravity[0], e.gravity[1], e.gravity[2], 0.0]; + f[4] = [e.axis[0], e.axis[1], e.axis[2], 0.0]; + f[5] = [e.damp, dt, 0.0, 0.0]; + } + ParticleSystem::Cannon(c) => { + u[3] = 1; + u2 = [0, 1, c.seed, prev_n]; + f[0] = [c.pos[0], c.pos[1], c.barrel_angle, c.barrel_radius]; + f[1] = [c.dir_spread, c.vel, c.vel_spread, c.grav_spread]; + f[2] = [c.gravity[0], c.gravity[1], dt, 0.0]; + f[3] = c.bounds; + } + } + let mut out = Vec::with_capacity(128); + for x in u.iter().chain(&u2) { + out.extend_from_slice(&x.to_le_bytes()); + } + for x in f.iter().flatten() { + out.extend_from_slice(&x.to_le_bytes()); + } + out +} + +/// Simulate `req` (see [`effectcraft_effects::psim::ParticleSim`]). +pub(crate) fn simulate(g: &GpuContext, req: &SimRequest) -> Option> { + if !g.can_readback() || req.steps > u32::MAX as u64 { + return None; + } + let n = req.births.len() as u32; + if n == 0 { + return Some(vec![]); + } + let size = n as u64 * STATE_BYTES; + if size > g.device.limits().max_storage_buffer_binding_size { + return None; + } + let p = pipes(g); + // The latest checkpoint at or before the target step. + let (s0, prev, prev_n) = { + let mut c = g.particle_states.lock().ok()?; + c.clock += 1; + let now = c.clock; + match c.list.iter_mut().filter(|k| k.key == req.key && k.steps <= req.steps).max_by_key(|k| k.steps) { + Some(k) => { + k.used = now; + (k.steps, Some(k.buf.clone()), k.n) + } + None => (0, None, 0), + } + }; + let d = &g.device; + let births: Vec = req.births.iter().flat_map(|b| b.to_le_bytes()).collect(); + let births = d.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("births"), contents: &births, usage: wgpu::BufferUsages::STORAGE }); + let prev = prev.unwrap_or_else(|| { + d.create_buffer(&wgpu::BufferDescriptor { label: Some("particles"), size: STATE_BYTES, usage: wgpu::BufferUsages::STORAGE, mapped_at_creation: false }) + }); + let next = d.create_buffer(&wgpu::BufferDescriptor { + label: Some("particles"), + size, + usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC, + mapped_at_creation: false, + }); + let ub = d.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("particles"), + contents: &uniforms(req, n, s0, prev_n), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bg = d.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &p.bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: ub.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: births.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: prev.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: next.as_entire_binding() }, + ], + }); + let read = d.create_buffer(&wgpu::BufferDescriptor { + label: Some("particles readback"), + size, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let mut enc = d.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("particles") }); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("particles"), timestamp_writes: None }); + pass.set_pipeline(&p.pipe); + pass.set_bind_group(1, &bg, &[]); + pass.dispatch_workgroups(n.div_ceil(64), 1, 1); + } + enc.copy_buffer_to_buffer(&next, 0, &read, 0, size); + g.queue.submit([enc.finish()]); + let (tx, rx) = std::sync::mpsc::channel(); + read.map_async(wgpu::MapMode::Read, .., move |r| { + let _ = tx.send(r); + }); + g.device.poll(wgpu::PollType::wait_indefinitely()).ok()?; + rx.recv().ok()?.ok()?; + let view = read.get_mapped_range(..).ok()?; + let f: Vec = view.chunks_exact(4).map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])).collect(); + drop(view); + read.unmap(); + let parts = f + .chunks_exact(12) + .enumerate() + .filter(|(_, s)| s[9] != 0.0) + .map(|(id, s)| SimParticle { p: [s[0], s[1], s[2]], v: [s[4], s[5], s[6]], age: s[3], life: s[7], rnd: s[8], id: id as u32 }) + .collect(); + // Keep the new state as a checkpoint. + if let Ok(mut c) = g.particle_states.lock() { + let now = c.clock; + c.list.retain(|k| !(k.key == req.key && k.steps == req.steps)); + c.list.push(Checkpoint { key: req.key, steps: req.steps, n, buf: next, used: now }); + while c.list.len() > CHECKPOINTS { + let Some(i) = c.list.iter().enumerate().min_by_key(|(_, k)| k.used).map(|(i, _)| i) else { break }; + c.list.remove(i); + } + } + Some(parts) +} + +pub(crate) type PipesCell = OnceLock; +pub(crate) type StatesCell = Mutex; diff --git a/crates/gpu/src/shaders/particles.wgsl b/crates/gpu/src/shaders/particles.wgsl new file mode 100644 index 00000000..694d6fb6 --- /dev/null +++ b/crates/gpu/src/shaders/particles.wgsl @@ -0,0 +1,270 @@ +// GPU particles: effects::psim systems, one invocation per particle id. Mirrors the CPU step +// (effects::sim::Phys::spawn / step, effects::sim3::Pg::spawn / step) operation for operation. +// +// Concatenated after common.wgsl and kernels.wgsl (hash_noise, value_noise, powz); bindings in +// group 1 (group 0 is unused here). + +struct PSim { + // Particles, from step, to step, system (0 engine, 1 cannon). + u: vec4, + // Animation, flat, seed, particles in `prev`. + u2: vec4, + // Engine: f[0] = (life, speed, extra, cone), f[1] = producer, f[2] = radius, + // f[3] = gravity, f[4] = axis, f[5] = (damp, dt). + // Cannon: f[0] = (pos.x, pos.y, barrel angle, barrel radius), f[1] = (dir spread, vel, + // vel spread, grav spread), f[2] = (gravity.x, gravity.y, dt), f[3] = bounds. + f: array, 6>, +} + +struct PState { + // Position, age. + p: vec4, + // Velocity, life. + v: vec4, + // rnd, alive (1 / 0), id. + m: vec4, +} + +@group(1) @binding(0) var S: PSim; +@group(1) @binding(1) var births: array; +@group(1) @binding(2) var prev: array; +@group(1) @binding(3) var next: array; + +const PS_TAU: f32 = 6.2831855; + +fn ps_h(i: u32, k: u32) -> f32 { + return hash_noise(i, k, S.u2.z); +} + +fn ps_hs(i: u32, k: u32) -> f32 { + return hash_noise(i, k, S.u2.z) * 2.0 - 1.0; +} + +// sim::norm3. +fn ps_norm3(v: vec3) -> vec3 { + let l = sqrt(v.x * v.x + v.y * v.y + v.z * v.z); + if (l > 1e-9) { + return v / l; + } + return vec3(0.0, -1.0, 0.0); +} + +// sim::rand_dir. +fn ps_rand_dir(i: u32, k: u32, flat: bool) -> vec3 { + if (flat) { + let a = ps_h(i, k) * PS_TAU; + return vec3(cos(a), sin(a), 0.0); + } + let z = ps_hs(i, k); + let a = ps_h(i, k + 1u) * PS_TAU; + let r = sqrt(max(1.0 - z * z, 0.0)); + return vec3(r * cos(a), r * sin(a), z); +} + +struct Part { + p: vec3, + v: vec3, + age: f32, + life: f32, + rnd: f32, +} + +// sim::Phys::spawn (+ the step's sub-step birth offset). +fn ps_spawn_engine(id: u32) -> Part { + let flat = S.u2.y != 0u; + let anim = S.u2.x; + let producer = S.f[1].xyz; + let radius = S.f[2].xyz; + let u = ps_rand_dir(id, 1u, flat); + var e = 1.0 / 3.0; + if (flat) { + e = 0.5; + } + let rr = powz(ps_h(id, 3u), e); + var p = producer + u * radius * rr; + if (flat) { + p.z = 0.0; + } + let ax = S.f[4].xyz; + let jitter = ps_rand_dir(id, 5u, flat); + var dir = jitter; + switch anim { + case 1u, 9u: { + dir = ps_norm3(ax + jitter * 0.08); + } + case 2u: { + let k = S.f[0].w; + dir = ps_norm3(ax + jitter * k * ps_h(id, 8u)); + } + case 5u: { + var z = jitter.z; + if (flat) { + z = 0.0; + } + dir = vec3(jitter.x, 0.0, z); + } + case 6u: { + dir = ps_norm3(vec3(jitter.x * 0.25, -1.0, jitter.z * 0.25)); + } + case 7u: { + dir = ps_norm3(vec3(1.0, jitter.y * 0.05, jitter.z * 0.05)); + } + default: {} + } + let sp = S.f[0].y * (0.5 + 0.5 * ps_h(id, 9u)); + var v = dir * sp; + if (flat) { + v.z = 0.0; + } + let life = S.f[0].x * (0.8 + 0.4 * ps_h(id, 10u)); + let dt = S.f[5].y; + let frac = ps_h(id, 12u) * dt; + return Part(p + v * frac, v, frac, life, ps_h(id, 11u)); +} + +// sim::Phys::step for one particle at step `j`. +fn ps_step_engine(q: Part, j: u32) -> Part { + let flat = S.u2.y != 0u; + let anim = S.u2.x; + let seed = S.u2.z; + let dt = S.f[5].y; + let t = f32(j) * dt; + let producer = S.f[1].xyz; + let gravity = S.f[3].xyz; + let extra = S.f[0].z; + var a = gravity; + switch anim { + case 4u: { + let dx = q.p.x - producer.x; + let dz = q.p.z - producer.z; + let k = 4.0 * extra; + a.x += -dz * k; + a.z += dx * k; + if (flat) { + let dy = q.p.y - producer.y; + a.x += -dy * k; + a.y += dx * k; + } + } + case 5u: { + let dx = q.p.x - producer.x; + let dy = q.p.y - producer.y; + let k = 6.0 * extra; + a.x += -dy * k - dx * 2.0; + a.y += dx * k - dy * 2.0; + } + case 6u: { + let n = value_noise(q.p.x * 8.0, q.p.y * 8.0, t * 3.0, seed) - 0.5; + a.x += n * 4.0 * extra; + a.y -= gravity.y * 1.5 + 0.3 * extra; + } + case 8u, 9u: { + let f = 6.0 * extra; + a.x += (value_noise(q.p.x * 5.0, q.p.y * 5.0, q.p.z * 5.0 + t, seed) - 0.5) * f; + a.y += (value_noise(q.p.x * 5.0 + 17.0, q.p.y * 5.0, q.p.z * 5.0 + t, seed) - 0.5) * f; + if (!flat) { + a.z += (value_noise(q.p.x * 5.0, q.p.y * 5.0 + 31.0, q.p.z * 5.0 + t, seed) - 0.5) * f; + } + } + default: {} + } + let damp = S.f[5].x; + var o = q; + o.v = (q.v + a * dt) * damp; + o.p = q.p + o.v * dt; + if (flat) { + o.p.z = 0.0; + o.v.z = 0.0; + } + o.age = q.age + dt; + return o; +} + +// sim3::Pg::spawn (+ the sub-step birth offset). +fn ps_spawn_cannon(id: u32) -> Part { + let pos = S.f[0].xy; + let angle = S.f[0].z; + let r = S.f[0].w; + let spread = S.f[1].x; + let vel = S.f[1].y; + let vel_spread = S.f[1].z; + let a = radians(angle + ps_hs(id, 1u) * spread * 0.5); + let d = vec2(sin(a), -cos(a)); + let speed = max(vel + ps_hs(id, 3u) * vel_spread, 0.0); + var off: vec2; + if (r >= 0.0) { + off = vec2(ps_hs(id, 2u) * r, ps_hs(id, 4u) * r); + } else { + let aa = ps_h(id, 5u) * PS_TAU; + let rr = sqrt(ps_h(id, 6u)) * -r; + off = vec2(cos(aa) * rr, sin(aa) * rr); + } + let v = d * speed; + let dt = S.f[2].z; + let frac = ps_h(id, 9u) * dt; + let p = pos + off + v * frac; + return Part(vec3(p, 0.0), vec3(v, 0.0), 0.0, 0.0, 0.0); +} + +// sim3::Pg::step for one particle (gravity only); returns false when it leaves the bounds. +fn ps_step_cannon(q: ptr, id: u32) -> bool { + let dt = S.f[2].z; + let spread = S.f[1].w; + var g = 1.0; + if (spread > 0.0) { + g = 1.0 + (ps_h(id, 7u) * 2.0 - 1.0) * spread; + } + let a = S.f[2].xy * g; + let v = ((*q).v.xy + a * dt) * 1.0; + let p = (*q).p.xy + v * dt; + (*q).v = vec3(v, 0.0); + (*q).p = vec3(p, 0.0); + let b = S.f[3]; + return p.x > b.x && p.x < b.z && p.y > b.y && p.y < b.w; +} + +@compute @workgroup_size(64, 1) +fn particles(@builtin(global_invocation_id) gid: vec3) { + let id = gid.x; + if (id >= S.u.x) { + return; + } + let s0 = S.u.y; + let s1 = S.u.z; + let cannon = S.u.w == 1u; + let b = births[id]; + var q: Part; + var j: u32; + if (b < s0 && id < S.u2.w) { + let o = prev[id]; + if (o.m.y == 0.0) { + next[id] = o; + return; + } + q = Part(o.p.xyz, o.v.xyz, o.p.w, o.v.w, o.m.x); + j = s0; + } else { + if (cannon) { + q = ps_spawn_cannon(id); + } else { + q = ps_spawn_engine(id); + } + j = b + 1u; + } + var alive = true; + for (; j < s1; j++) { + if (cannon) { + if (!ps_step_cannon(&q, id)) { + alive = false; + break; + } + } else { + q = ps_step_engine(q, j); + if (q.age >= q.life) { + alive = false; + break; + } + } + } + next[id] = PState(vec4(q.p, q.age), vec4(q.v, q.life), vec4(q.rnd, select(0.0, 1.0, alive), 0.0, 0.0)); +} diff --git a/crates/gpu/src/tests_particles.rs b/crates/gpu/src/tests_particles.rs new file mode 100644 index 00000000..e3ef97a8 --- /dev/null +++ b/crates/gpu/src/tests_particles.rs @@ -0,0 +1,155 @@ +//! GPU particles vs the CPU simulation (the oracle): CC Particle World, CC Particle Systems II +//! and Particle Playground's cannon at fixed seeds must give the same live particles (same +//! ids) at positions within tolerance, from scratch and when advancing / seeking from GPU +//! checkpoints. + +use effectcraft_effects::psim::{ParticleSim, SimParticle}; +use effectcraft_effects::{EffectCtx, EffectEnv, EffectHost, LayerPixels, Params}; +use effectcraft_keyframe::Value; + +use crate::Gpu; +use crate::tests::{gpu, n}; + +struct Host(&'static Gpu); + +impl EffectHost for Host { + fn layer(&self, _id: u64, _masks_and_effects: bool) -> Option { + None + } + fn audio(&self, _id: u64, _start: f64, _frames: usize, _rate: u32) -> Option> { + None + } + fn particles(&self) -> Option<&dyn ParticleSim> { + Some(self.0) + } +} + +fn params(id: &str, vals: &[(&str, Value)]) -> Params { + let spec = effectcraft_effects::find(id).unwrap(); + let size = [320.0, 240.0]; + let mut p = Params { values: spec.params.iter().map(|ps| (ps.id.to_string(), effectcraft_effects::default_value(ps, size))).collect() }; + for (k, v) in vals { + assert!(p.values.contains_key(*k), "{id}: no {k}"); + p.values.insert(k.to_string(), v.clone()); + } + p +} + +/// (CPU, GPU) particles of effect `id` at layer time `t`. +fn states(id: &str, p: &Params, t: f64, host: &Host) -> (Vec, Vec) { + let ctx = |env| EffectCtx { params: p, time: t, layer_size: [320.0, 240.0], seed: 5, adjustment: false, env }; + let run = + |c: &EffectCtx| if id == "ec.sim.particleplayground" { effectcraft_effects::playground_state(c) } else { effectcraft_effects::particle_state(id, c) }; + let cpu = run(&ctx(EffectEnv::default())).expect("cpu state"); + let gpu = run(&ctx(EffectEnv { host: Some(host), ..Default::default() })).expect("gpu state"); + (cpu, gpu) +} + +/// Same ids alive (up to `id_slack` particles dying a step apart), positions within `tol` +/// (world units) for 99 % of them and on average within `tol / 10`. +fn agree(label: &str, cpu: &[SimParticle], gpu: &[SimParticle], tol: f32, id_slack: usize) { + assert!(cpu.len() > 20, "{label}: the scene has particles ({})", cpu.len()); + let gi: std::collections::HashMap = gpu.iter().map(|q| (q.id, q)).collect(); + let mut missing = 0; + let mut errs = vec![]; + for q in cpu { + match gi.get(&q.id) { + Some(g) => errs.push((0..3).map(|k| (q.p[k] - g.p[k]).abs()).fold(0.0f32, f32::max)), + None => missing += 1, + } + } + let extra = gpu.len() + missing - cpu.len(); + assert!(missing + extra <= id_slack, "{label}: {missing} CPU particles missing on the GPU, {extra} extra (of {})", cpu.len()); + errs.sort_by(f32::total_cmp); + let mean = errs.iter().sum::() / errs.len() as f32; + let p99 = errs[(errs.len() * 99 / 100).min(errs.len() - 1)]; + eprintln!("{label}: {} particles, mean err {mean:.2e}, p99 {p99:.2e}, max {:.2e}", cpu.len(), errs.last().unwrap()); + assert!(p99 <= tol && mean <= tol / 10.0, "{label}: positions differ (mean {mean}, p99 {p99})"); +} + +#[test] +fn particle_world_matches_the_cpu() { + let Some(g) = gpu() else { return }; + let host = Host(g); + for anim in [0u32, 1, 2, 3, 4, 5, 6, 7, 8, 9] { + let p = params( + "ec.sim.ccparticleworld", + &[ + ("physics/animation", Value::Enum(anim)), + ("birthRate", n(2.0)), + ("longevity", n(1.2)), + ("physics/extra", n(0.6)), + ("physics/resistance", n(0.5)), + ("extras/randomSeed", n(7.0)), + ], + ); + let (cpu, gpu) = states("ec.sim.ccparticleworld", &p, 1.5, &host); + // Fractal / fire animations sample value noise along the path: rounding differences can + // grow, so they get a looser bound. + let tol = if matches!(anim, 6 | 8 | 9) { 1e-3 } else { 1e-4 }; + agree(&format!("particle world anim {anim}"), &cpu, &gpu, tol, cpu.len() / 100 + 1); + } +} + +#[test] +fn particle_systems_ii_matches_the_cpu() { + let Some(g) = gpu() else { return }; + let host = Host(g); + for anim in [0u32, 1, 2, 4, 5, 8] { + let p = params( + "ec.sim.ccparticlesystems2", + &[("physics/animation", Value::Enum(anim)), ("birthRate", n(3.0)), ("longevity", n(1.5)), ("physics/gravity", n(0.8)), ("randomSeed", n(3.0))], + ); + let (cpu, gpu) = states("ec.sim.ccparticlesystems2", &p, 2.0, &host); + let tol = if anim == 8 { 1e-3 } else { 1e-4 }; + agree(&format!("particle systems ii anim {anim}"), &cpu, &gpu, tol, cpu.len() / 100 + 1); + } +} + +#[test] +fn playground_cannon_matches_the_cpu_and_checkpoints_reuse() { + let Some(g) = gpu() else { return }; + let host = Host(g); + for (radius, spread) in [(4.0, 0.0), (-10.0, 50.0)] { + let p = params( + "ec.sim.particleplayground", + &[ + ("grid/particlesAcross", n(0.0)), + ("cannon/barrelRadius", n(radius)), + ("cannon/particlesPerSecond", n(90.0)), + ("cannon/directionRandomSpread", n(30.0)), + ("cannon/velocityRandomSpread", n(40.0)), + ("gravity/gravityForceRandomSpread", n(spread)), + ], + ); + // Advance through checkpoints (0.5 → 1.0 → 2.0), seek back (1.5), compare each. + for t in [0.5, 1.0, 2.0, 1.5] { + let (cpu, gpu) = states("ec.sim.particleplayground", &p, t, &host); + // Positions are layer pixels here. + agree(&format!("cannon r {radius} spread {spread} t {t}"), &cpu, &gpu, 0.01, cpu.len() / 100 + 1); + } + } +} + +/// Through the renderer: the GPU backend's effect host hands the simulation to the GPU, and +/// the composited frames agree with the CPU render. +#[test] +fn particle_effects_render_like_the_cpu() { + use effectcraft_project::BitDepth; + use effectcraft_time::Tick; + + use crate::tests::{Scene, check, compare_at, opts}; + for (id, vals) in [ + ("ec.sim.ccparticleworld", vec![("birthRate", n(1.0)), ("physics/animation", Value::Enum(4))]), + ("ec.sim.ccparticlesystems2", vec![("birthRate", n(2.0))]), + ("ec.sim.particleplayground", vec![("grid/particlesAcross", n(0.0)), ("cannon/particlesPerSecond", n(60.0))]), + ] { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let mut s = Scene::new(depth); + let mut l = s.solid([0.1, 0.1, 0.15], 97, 61); + s.effect(&mut l, id, &vals); + s.push(l); + check(&format!("{id} {depth:?}"), compare_at(&s, opts(), Tick::from_seconds_f64(1.0)), 0.002); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index ffb55936..44f4876a 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -82,6 +82,9 @@ struct FxHost<'r, 'a, 'c> { const MAX_FX_DEPTH: usize = 8; impl EffectHost for FxHost<'_, '_, '_> { + fn particles(&self) -> Option<&dyn effectcraft_effects::psim::ParticleSim> { + self.r.active_accel().and_then(|a| a.particles()) + } fn layer(&self, id: u64, masks_and_effects: bool) -> Option { let other = self.ctx.layer(effectcraft_project::LayerId(id))?; if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { @@ -240,6 +243,11 @@ pub trait Accelerator: Send + Sync { fn raster_3d(&self, _scene: &three_d::adv::Scene) -> Option { None } + /// A particle simulation backend (GPU particles) for the stepped particle effects, with + /// the CPU simulation's semantics. `None` = they simulate on the CPU. + fn particles(&self) -> Option<&dyn effectcraft_effects::psim::ParticleSim> { + None + } } /// Where an effect stack runs (see [`Renderer::run_effects_on`]): a CPU buffer, or an image From f855a4b55bfe13000698862ae0d74cc8aed251a7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:45:44 -0400 Subject: [PATCH 09/36] M14.5: wire the remaining settings; keyframe assistants and labels; AE menu entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings: 56 of 59 not-yet-wired settings now change behaviour (nested-comp switches, constant vertex count, related-item time sync, match-name pick whip, split-layer order, Home on open, motion path span, internal wireframes, zoom quality, label-coloured tabs, mask colour cycling, gradients, grid/guide styles, text engine, font preview/recent/native names, missing frames, unlabeled alpha, drag import, default output folder, segmenting, bit depth names, preview sample rate, disk/media/conformed-audio caches, RAM reserve and low-memory reduction, Video Preview window, 3D reference axes/extended viewer/draft shadows, script file gate + File.execute warning, expression editor options and error banner). Three stay display-only and are documented. Features: Convert Audio to Keyframes, RPF Camera Import (JSON/CSV camera data), keyframe colour labels + Select Keyframe Label Group, Recent Animation Presets. Menus: Edit > History, File > Import Recent Footage, Replace Footage > With Layered Comp, Save a Copy As XML (.ecprojx, opens again), Composition > VR (Create VR Environment, Extract Cubemap), Assign Shortcut to 3D view / workspace, Window viewer labels + open viewers, Help > System Compatibility Report / Enable Logging / Reveal Logging File. Dynamic menu nodes (@dynamic:…) shared by the egui and native menus. Co-Authored-By: Claude Opus 5.5 --- apps/effectcraft/src/audio_out.rs | 10 +- apps/effectcraft/src/main.rs | 7 +- crates/engine/src/commands/animation.rs | 6 + crates/engine/src/commands/app_more.rs | 592 +++++++++++++++++ crates/engine/src/commands/edit.rs | 11 +- crates/engine/src/commands/file.rs | 78 ++- crates/engine/src/commands/keys_more.rs | 298 +++++++++ crates/engine/src/commands/layer.rs | 8 +- crates/engine/src/commands/layer_menu.rs | 51 +- crates/engine/src/commands/link.rs | 13 +- crates/engine/src/commands/mask.rs | 96 ++- crates/engine/src/commands/mod.rs | 11 + crates/engine/src/commands/prop.rs | 2 +- crates/engine/src/commands/render_queue.rs | 11 +- crates/engine/src/commands/stubs.rs | 3 - crates/engine/src/lib.rs | 117 ++++ crates/engine/src/logging.rs | 99 +++ crates/engine/src/media_cache.rs | 65 ++ crates/engine/src/menus.rs | 167 ++++- crates/engine/src/menus/tests.rs | 2 +- crates/engine/src/prefs.rs | 303 +++++++-- crates/engine/src/render_queue.rs | 130 +++- crates/engine/src/session_settings.rs | 57 +- crates/engine/src/sysinfo.rs | 140 ++++ crates/engine/src/tests_m145.rs | 666 +++++++++++++++++++ crates/engine/src/xml_project.rs | 265 ++++++++ crates/export/src/lib.rs | 4 + crates/export/tests/roundtrip.rs | 8 +- crates/host/src/lib.rs | 1 + crates/keyframe/src/lib.rs | 10 + crates/media/src/pool.rs | 108 ++- crates/render/src/lib.rs | 69 +- crates/render/src/three_d/adv/scene.rs | 9 +- crates/render/src/three_d/compose.rs | 8 +- crates/script/src/prelude.js | 2 +- crates/script/src/runtime.rs | 5 + crates/text/src/fonts.rs | 10 +- crates/text/src/sfnt.rs | 27 +- crates/ui-egui/src/audio.rs | 45 +- crates/ui-egui/src/dock.rs | 4 + crates/ui-egui/src/dock_ui.rs | 25 + crates/ui-egui/src/lib.rs | 9 +- crates/ui-egui/src/menus.rs | 96 ++- crates/ui-egui/src/native_menu.rs | 65 +- crates/ui-egui/src/panels/expr_editor.rs | 394 +++++++++++ crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/text_panels.rs | 86 ++- crates/ui-egui/src/panels/timeline.rs | 29 +- crates/ui-egui/src/panels/viewer.rs | 101 ++- crates/ui-egui/src/panels/viewer_overlays.rs | 7 +- crates/ui-egui/src/panels/viewer_tools.rs | 12 +- crates/ui-egui/src/panels/waveform.rs | 17 +- crates/ui-egui/src/prefs_live.rs | 141 ++++ crates/ui-egui/src/theme.rs | 25 + crates/ui-egui/tests/ui_text_edit.rs | 4 +- docs/preferences.md | 132 ++-- 56 files changed, 4378 insertions(+), 284 deletions(-) create mode 100644 crates/engine/src/commands/app_more.rs create mode 100644 crates/engine/src/commands/keys_more.rs create mode 100644 crates/engine/src/logging.rs create mode 100644 crates/engine/src/media_cache.rs create mode 100644 crates/engine/src/sysinfo.rs create mode 100644 crates/engine/src/tests_m145.rs create mode 100644 crates/engine/src/xml_project.rs create mode 100644 crates/ui-egui/src/panels/expr_editor.rs create mode 100644 crates/ui-egui/src/prefs_live.rs diff --git a/apps/effectcraft/src/audio_out.rs b/apps/effectcraft/src/audio_out.rs index eea7ffaa..06ab9785 100644 --- a/apps/effectcraft/src/audio_out.rs +++ b/apps/effectcraft/src/audio_out.rs @@ -47,7 +47,15 @@ fn device(name: &str) -> Option { pub fn open(out: &AudioOutput) -> Option> { let dev = device(&out.device)?; let cfg = dev.default_output_config().ok()?; - Some(Box::new(CpalOut { out: out.clone(), rate: cfg.sample_rate().0, latency: Arc::new(AtomicU64::new(0)), stop: None, thread: None })) + // Settings ▸ Audio ▸ Preview Sample Rate: open the device at that rate when it supports it + // (otherwise the preview is mixed at that rate and resampled to the device's). + let supports = |r: u32| { + dev.supported_output_configs() + .map(|mut c| c.any(|c| c.channels() == cfg.channels() && c.min_sample_rate().0 <= r && r <= c.max_sample_rate().0)) + .unwrap_or(false) + }; + let rate = if out.rate > 0 && supports(out.rate) { out.rate } else { cfg.sample_rate().0 }; + Some(Box::new(CpalOut { out: out.clone(), rate, latency: Arc::new(AtomicU64::new(0)), stop: None, thread: None })) } fn run>( diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index b6426f5f..157ffba2 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -24,6 +24,8 @@ fn app_icon() -> egui::IconData { } fn main() -> eframe::Result { + // Help ▸ Enable Logging writes through this logger; warnings feed the compatibility report. + effectcraft_engine::logging::install(); let mut control_port: Option = std::env::var("EFFECTCRAFT_CONTROL_PORT").ok().and_then(|p| p.parse().ok()); let mut files = Vec::new(); let mut demo = true; @@ -94,8 +96,9 @@ fn main() -> eframe::Result { app.hooks.pick_save = Some(Box::new(|name: &str| { rfd::FileDialog::new().add_filter("EffectCraft Project", &["ecproj"]).set_file_name(name).save_file().map(|p| p.to_string_lossy().to_string()) })); - app.hooks.pick_open_project = - Some(Box::new(|| rfd::FileDialog::new().add_filter("EffectCraft Project", &["ecproj"]).pick_file().map(|p| p.to_string_lossy().to_string()))); + app.hooks.pick_open_project = Some(Box::new(|| { + rfd::FileDialog::new().add_filter("EffectCraft Project", &["ecproj", "ecprojx"]).pick_file().map(|p| p.to_string_lossy().to_string()) + })); app.hooks.audio_device = Some(Box::new(audio_out::open)); app.hooks.audio_devices = Some(Box::new(audio_out::devices)); app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string()))); diff --git a/crates/engine/src/commands/animation.rs b/crates/engine/src/commands/animation.rs index a6390504..aa74c3f2 100644 --- a/crates/engine/src/commands/animation.rs +++ b/crates/engine/src/commands/animation.rs @@ -103,6 +103,12 @@ fn apply_preset(s: &mut Session, p: &Value) -> Result { if ids.is_empty() { return Err(bad("anim.applyPreset", "select a layer")); } + // Animation ▸ Recent Animation Presets. + if let Some(path) = str_p(p, "path") { + s.prefs.push_recent_preset(path); + s.save_prefs(); + s.prefs_revision += 1; + } let effects: Vec = doc["effects"].as_array().map(|a| a.iter().filter_map(|v| serde_json::from_value(v.clone()).ok()).collect()).unwrap_or_default(); let props: Vec<(String, Property)> = doc["properties"] diff --git a/crates/engine/src/commands/app_more.rs b/crates/engine/src/commands/app_more.rs new file mode 100644 index 00000000..37604853 --- /dev/null +++ b/crates/engine/src/commands/app_more.rs @@ -0,0 +1,592 @@ +//! Menu entries of M14.5: Edit ▸ History, File ▸ Import Recent Footage, Replace Footage ▸ With +//! Layered Comp, Save a Copy As XML, Animation ▸ Recent Animation Presets, Composition ▸ VR +//! (Create VR Environment, Extract Cubemap), the "Assign Shortcut to …" submenus, Help ▸ Enable +//! Logging / Reveal Logging File / System Compatibility Report, and `File.execute()` for scripts. + +use effectcraft_geom::{Mat4, Vec3, vec3}; +use effectcraft_project::{ItemId, ItemKind, LayerSource}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, comp_id, has_comp, has_layers, has_project_selection, str_p}; +use crate::{EngineError, Event, Result, Session, cmd, query}; + +/// Run `f` (which may run several undoable commands) as one undo step named `label`; on error +/// the project is restored. +pub(crate) fn grouped(s: &mut Session, label: &str, f: impl FnOnce(&mut Session) -> Result) -> Result { + // The steps `f` records go to an empty stack (so the undo limit can't drop the snapshot + // taken before them), then become one step on the real one. + let saved = std::mem::take(&mut s.history.undo); + let before = s.project.clone(); + let r = f(s); + let changed = !s.history.undo.is_empty(); + s.history.undo = saved; + s.history.merge_key = None; + match r { + Ok(v) => { + if changed { + s.history.undo.push((label.to_string(), before)); + let levels = s.prefs.general.undo_levels.max(1) as usize; + if s.history.undo.len() > levels { + let extra = s.history.undo.len() - levels; + s.history.undo.drain(..extra); + } + } + Ok(v) + } + Err(e) => { + if changed { + s.project = before; + s.sanitize_state(); + s.bump(); + } + Err(e) + } + } +} + +fn id_of(v: &Value, key: &str) -> Result { + v.get(key).and_then(Value::as_u64).ok_or_else(|| EngineError::Other(format!("expected `{key}` in {v}"))) +} + +// ---------------------------------------------------------------- Edit ▸ History + +/// Edit ▸ History: `{steps: n}` undoes the last n steps (the entry n−1 of the list), `{redo: n}` +/// redoes n; without either, lists the steps (newest first). +fn history(s: &mut Session, p: &Value) -> Result { + if let Some(n) = p.get("steps").and_then(Value::as_u64) { + if n as usize > s.history.undo.len() { + return Err(bad("edit.history", format!("only {} undo steps", s.history.undo.len()))); + } + for _ in 0..n { + s.undo(); + } + } else if let Some(n) = p.get("redo").and_then(Value::as_u64) { + if n as usize > s.history.redo.len() { + return Err(bad("edit.history", format!("only {} redo steps", s.history.redo.len()))); + } + for _ in 0..n { + s.redo(); + } + } + let undo: Vec<&str> = s.history.undo.iter().rev().map(|(l, _)| l.as_str()).collect(); + let redo: Vec<&str> = s.history.redo.iter().rev().map(|(l, _)| l.as_str()).collect(); + Ok(json!({"undo": undo, "redo": redo})) +} + +fn has_history(s: &Session) -> std::result::Result<(), String> { + if s.history.undo.is_empty() && s.history.redo.is_empty() { Err("nothing to undo".into()) } else { Ok(()) } +} + +// ---------------------------------------------------------------- recent footage / presets + +fn import_recent(s: &mut Session, p: &Value) -> Result { + let path = match (str_p(p, "path"), p.get("index").and_then(Value::as_u64)) { + (Some(x), _) => x.to_string(), + (None, Some(i)) => s.prefs.recent_footage.get(i as usize).cloned().ok_or_else(|| bad("file.importRecent", format!("no recent footage #{i}")))?, + _ => s.prefs.recent_footage.first().cloned().ok_or_else(|| bad("file.importRecent", "no recent footage"))?, + }; + super::file::import(s, &json!({"paths": [path]})) +} + +fn has_recent_footage(s: &Session) -> std::result::Result<(), String> { + if s.prefs.recent_footage.is_empty() { Err("no recent footage".into()) } else { Ok(()) } +} + +fn clear_recent_footage(s: &mut Session, _: &Value) -> Result { + s.prefs.recent_footage.clear(); + s.save_prefs(); + s.prefs_revision += 1; + Ok(Value::Null) +} + +fn apply_recent_preset(s: &mut Session, p: &Value) -> Result { + let path = match (str_p(p, "path"), p.get("index").and_then(Value::as_u64)) { + (Some(x), _) => x.to_string(), + (None, Some(i)) => s.prefs.recent_presets.get(i as usize).cloned().ok_or_else(|| bad("anim.applyRecentPreset", format!("no recent preset #{i}")))?, + _ => s.prefs.recent_presets.first().cloned().ok_or_else(|| bad("anim.applyRecentPreset", "no recent animation presets"))?, + }; + let mut q = json!({"path": path}); + if let Some(l) = p.get("layers") { + q["layers"] = l.clone(); + } + s.execute("anim.applyPreset", q) +} + +fn has_recent_presets(s: &Session) -> std::result::Result<(), String> { + has_layers(s)?; + if s.prefs.recent_presets.is_empty() { Err("no recent animation presets".into()) } else { Ok(()) } +} + +fn clear_recent_presets(s: &mut Session, _: &Value) -> Result { + s.prefs.recent_presets.clear(); + s.save_prefs(); + s.prefs_revision += 1; + Ok(Value::Null) +} + +// ---------------------------------------------------------------- File menu + +fn save_copy_xml(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("file.saveCopyAsXml", "missing `path` (.ecprojx)"))?; + let xml = crate::xml_project::to_xml(&s.project.to_json()).map_err(EngineError::Other)?; + s.services.write_file(path, xml.as_bytes()).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; + s.toast(format!("Saved an XML copy to {path}")); + Ok(json!({"path": path, "bytes": xml.len()})) +} + +/// File ▸ Replace Footage ▸ With Layered Comp: a layered (Photoshop) footage item becomes a +/// composition of its layers; every layer using the footage now uses the comp, and the footage +/// item goes. One undo step. +fn replace_with_layered_comp(s: &mut Session, p: &Value) -> Result { + let item = p.get("item").and_then(Value::as_u64).map(ItemId).or_else(|| s.state.project_selection.first().copied()); + let item = item.ok_or_else(|| bad("file.replaceWithLayeredComp", "select a layered footage item"))?; + let path = match s.project.item(item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.codec == "PSD" || f.path.to_ascii_lowercase().ends_with(".psd") => f.path.clone(), + _ => return Err(bad("file.replaceWithLayeredComp", "the item is not layered footage (a Photoshop file)")), + }; + let bytes = s.services.read_file(&path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + grouped(s, "Replace Footage", |s| { + let (comp, _, _) = super::file::import_psd_comp(s, &path, bytes, false)?; + let comp = ItemId(comp); + s.edit("Replace Footage", None, |proj, st| { + let ids: Vec = proj.items.keys().copied().collect(); + for cid in ids { + if cid == comp { + continue; + } + if let Some(c) = proj.comp_mut(cid) { + for l in &mut c.layers { + if matches!(&l.source, LayerSource::Footage { item: i } if *i == item) { + l.source = LayerSource::Comp { item: comp }; + } + } + } + } + proj.items.remove(&item); + st.project_selection = vec![comp]; + Ok(()) + })?; + Ok(json!({"comp": comp.0})) + }) +} + +fn has_footage(s: &Session) -> std::result::Result<(), String> { + has_project_selection(s)?; + let ok = s.state.project_selection.iter().any(|i| matches!(s.project.item(*i).map(|x| &x.kind), Some(ItemKind::Footage(_)))); + if ok { Ok(()) } else { Err("select a footage item".into()) } +} + +/// `File.execute()` from scripts (and agents): open a file with its default application. +/// Needs Allow Scripts to Write Files and Access Network; with Warn User When Executing Files on, +/// the frontend asks first (`confirmed: true` skips the question). +fn execute_file(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("file.executeFile", "missing `path`"))?; + if !s.prefs.scripting.allow_scripts_write_files { + return Err(bad( + "file.executeFile", + "scripts can't execute files unless Settings ▸ Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network is on", + )); + } + if !std::path::Path::new(path).exists() { + return Err(bad("file.executeFile", format!("no file {path}"))); + } + if s.prefs.scripting.warn_executing_files && !b_p(p, "confirmed").unwrap_or(false) { + s.events.push(Event::Frontend { command: "app.confirmExecute".into(), params: json!({"path": path}) }); + return Ok(json!({"needsConfirmation": path})); + } + let url = format!("file://{}", path.replace(' ', "%20")); + s.events.push(Event::OpenUrl(url)); + Ok(json!({"opened": path})) +} + +// ---------------------------------------------------------------- Assign Shortcut to … + +/// The F10 / F11 / F12 (3D views) or Shift+F10 / F11 / F12 (workspaces) slots. +fn slot_p(p: &Value, cmd: &str, shift: bool) -> Result { + let raw = str_p(p, "slot").ok_or_else(|| bad(cmd, "missing `slot` (F10, F11 or F12)"))?; + let k = raw.trim().trim_start_matches("Shift+").to_ascii_uppercase(); + if !matches!(k.as_str(), "F10" | "F11" | "F12") { + return Err(bad(cmd, "slot: F10, F11 or F12")); + } + Ok(if shift { format!("Shift+{k}") } else { k }) +} + +/// Bind `keys` to the bindable `key`, taking it from whatever had it. +fn reassign(s: &mut Session, key: &str, slot: &str, cmd: &str) -> Result { + let t = s.shortcuts(); + let b = t.find(key).cloned().ok_or_else(|| bad(cmd, format!("`{key}` can't have a shortcut")))?; + let holders: Vec<(crate::shortcuts::Bindable, Vec)> = t + .bindables + .iter() + .filter(|x| x.key != key && t.keys.get(&x.key).is_some_and(|k| k.iter().any(|k| k == slot))) + .map(|x| (x.clone(), t.keys.get(&x.key).cloned().unwrap_or_default())) + .collect(); + let mine = t.keys.get(key).cloned().unwrap_or_default(); + let replaced: Vec = holders.iter().map(|(h, _)| h.label.clone()).collect(); + for (h, keys) in &holders { + let rest: Vec = keys.iter().filter(|k| *k != slot).cloned().collect(); + s.keymaps.set(h, &rest).map_err(|e| bad(cmd, e))?; + } + let mut keys = vec![slot.to_string()]; + keys.extend(mine.into_iter().filter(|k| k != slot)); + let created = s.keymaps.set(&b, &keys).map_err(|e| bad(cmd, e))?; + s.shortcuts_changed(); + if let Some(n) = &created { + s.toast(format!("The default shortcuts can't be changed: created the preset \"{n}\"")); + } + Ok(json!({"command": key, "keys": keys, "replaced": replaced, "preset": s.keymaps.active})) +} + +/// The command id of a 3D view. +pub(crate) fn view_command(v: effectcraft_render::three_d::View3D) -> &'static str { + use effectcraft_render::three_d::View3D::*; + match v { + ActiveCamera => "view.3d.activeCamera", + Front => "view.3d.front", + Left => "view.3d.left", + Top => "view.3d.top", + Back => "view.3d.back", + Right => "view.3d.right", + Bottom => "view.3d.bottom", + Custom1 => "view.3d.custom1", + Custom2 => "view.3d.custom2", + Custom3 => "view.3d.custom3", + } +} + +/// View ▸ Assign Shortcut to "": F10, F11 or F12 switches to the view. +fn assign_view_shortcut(s: &mut Session, p: &Value) -> Result { + let slot = slot_p(p, "view.assign3dShortcut", false)?; + let cid = comp_id(s, p)?; + let cur = s.state.views3d.get(&cid).map(|v| v.current).unwrap_or_default(); + reassign(s, view_command(cur), &slot, "view.assign3dShortcut") +} + +/// Window ▸ Assign Shortcut to "" Workspace: Shift+F10, Shift+F11 or Shift+F12. +/// The frontend fills in the current workspace. +fn assign_workspace_shortcut(s: &mut Session, p: &Value) -> Result { + let slot = slot_p(p, "window.assignWorkspaceShortcut", true)?; + let Some(ws) = str_p(p, "workspace") else { return super::frontend(s, "window.assignWorkspaceShortcut", p) }; + let key = crate::shortcuts::binding_key("window.workspace", &json!({"name": ws})); + reassign(s, &key, &slot, "window.assignWorkspaceShortcut") +} + +// ---------------------------------------------------------------- Help + +fn log_file(s: &Session) -> std::path::PathBuf { + crate::logging::log_path(s.config.as_ref().and_then(|c| c.dir()).as_deref()) +} + +fn enable_logging(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "on").unwrap_or(!crate::logging::is_enabled()); + let file = log_file(s); + crate::logging::set_enabled(on, file.clone()).map_err(|e| EngineError::Other(format!("cannot write {}: {e}", file.display())))?; + s.prefs_revision += 1; + s.toast(if on { format!("Logging to {}", file.display()) } else { "Logging off".into() }); + Ok(json!({"enabled": on, "path": file.to_string_lossy()})) +} + +fn reveal_log(s: &mut Session, _: &Value) -> Result { + let file = crate::logging::current_file().unwrap_or_else(|| log_file(s)); + let path = file.to_string_lossy().to_string(); + let url = super::layer_menu::folder_url(&path); + s.events.push(Event::OpenUrl(url.clone())); + Ok(json!({"path": path, "url": url})) +} + +/// Media formats File ▸ Import reads (with the desktop host's decoders). +pub const IMPORT_FORMATS: &[&str] = &[ + "MP4/MOV (H.264, HEVC, ProRes, AV1)", + "WebM/MKV (VP8, VP9, AV1)", + "PNG", + "JPEG", + "GIF", + "WebP", + "TIFF", + "BMP", + "OpenEXR", + "Photoshop (PSD)", + "SVG", + "WAV", + "AIFF", + "MP3", + "FLAC", + "AAC", + "Opus", + "glTF/GLB", + "OBJ", + "Lottie JSON", +]; + +/// Help ▸ System Compatibility Report: OS, CPU, memory, GPU adapter, import/export formats. +pub fn system_report(s: &Session) -> Value { + let mem = crate::sysinfo::memory(); + let gb = |b: u64| format!("{:.1} GB", b as f64 / (1u64 << 30) as f64); + let export: Vec<&str> = s.exporter.as_ref().map(|e| e.formats().into_iter().map(|f| f.label()).collect()).unwrap_or_default(); + let gpu = s.accel.as_ref().map(|a| a.name()); + let budgets = s.prefs.cache_budgets(mem.or(s.sys_memory)); + let mut issues = vec![]; + if gpu.is_none() { + issues.push("No GPU compositor: rendering uses the CPU (all features still work).".to_string()); + } + if mem.is_some_and(|m| m.total < 8 << 30) { + issues.push("Less than 8 GB of memory: use lower preview resolutions for long or large comps.".to_string()); + } + if s.exporter.is_none() { + issues.push("Export is not available in this build.".to_string()); + } + json!({ + "app": format!("EffectCraft {}", env!("CARGO_PKG_VERSION")), + "os": crate::sysinfo::os_version(), + "arch": std::env::consts::ARCH, + "cpu": crate::sysinfo::cpu_name(), + "cores": crate::sysinfo::cpu_cores(), + "memory": mem.map(|m| json!({"total": gb(m.total), "available": gb(m.available)})), + "gpu": gpu, + "caches": {"layer": gb(budgets.layer as u64), "footage": gb(budgets.media as u64), "preview": gb(budgets.preview as u64)}, + "import": IMPORT_FORMATS, + "export": export, + "logging": crate::logging::is_enabled(), + "recentWarnings": crate::logging::recent().into_iter().rev().take(20).collect::>(), + "issues": issues, + }) +} + +/// The report as text (the dialog and `--` CLI output). +pub fn report_text(r: &Value) -> String { + let st = |k: &str| r.get(k).and_then(Value::as_str).unwrap_or("unknown").to_string(); + let list = |k: &str| r.get(k).and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_str).collect::>().join(", ")).unwrap_or_default(); + let mut t = format!( + "{}\nOperating system: {} ({})\nCPU: {} ({} logical cores)\n", + st("app"), + st("os"), + st("arch"), + st("cpu"), + r.get("cores").and_then(Value::as_u64).unwrap_or(1) + ); + match r.get("memory").filter(|m| !m.is_null()) { + Some(m) => t.push_str(&format!("Memory: {} ({} available)\n", m["total"].as_str().unwrap_or("?"), m["available"].as_str().unwrap_or("?"))), + None => t.push_str("Memory: unknown\n"), + } + t.push_str(&format!("GPU: {}\n", r.get("gpu").and_then(Value::as_str).unwrap_or("none (CPU compositing)"))); + let c = &r["caches"]; + t.push_str(&format!( + "Cache budgets: layer {}, footage {}, preview {}\n", + c["layer"].as_str().unwrap_or("?"), + c["footage"].as_str().unwrap_or("?"), + c["preview"].as_str().unwrap_or("?") + )); + t.push_str(&format!("Import: {}\nExport: {}\n", list("import"), list("export"))); + let issues = list("issues"); + t.push_str(&if issues.is_empty() { "No compatibility issues found.".to_string() } else { format!("Issues: {issues}") }); + t +} + +fn report(s: &mut Session, p: &Value) -> Result { + let r = system_report(s); + if !b_p(p, "quiet").unwrap_or(false) { + s.events.push(Event::Frontend { command: "app.showReport".into(), params: json!({"title": "System Compatibility Report", "text": report_text(&r)}) }); + } + Ok(r) +} + +// ---------------------------------------------------------------- Composition ▸ VR + +/// Orientation (degrees, AE order) of a camera whose +X, +Y (down) and +Z axes point along the +/// given world directions: searched over quarter turns, which is all the cube faces need. +fn orientation_for_axes(x: Vec3, y: Vec3, z: Vec3) -> [f64; 3] { + let mut best = ([0.0; 3], f64::MAX); + for a in 0..4 { + for b in 0..4 { + for c in 0..4 { + let o = vec3(a as f64 * 90.0, b as f64 * 90.0, c as f64 * 90.0); + let m = Mat4::orientation(o); + let err = (m.apply_vec(vec3(1.0, 0.0, 0.0)) - x).length() + + (m.apply_vec(vec3(0.0, 1.0, 0.0)) - y).length() + + (m.apply_vec(vec3(0.0, 0.0, 1.0)) - z).length(); + if err < best.1 { + best = ([o.x, o.y, o.z], err); + } + } + } + } + best.0 +} + +/// Cube faces in the VR Converter's order (+X, −X, +Y, −Y, +Z, −Z): name, forward, right and up +/// in the VR convention (X right, Y up, Z front). +const FACES: [(&str, [f64; 3], [f64; 3], [f64; 3]); 6] = [ + ("Right", [1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), + ("Left", [-1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), + ("Top", [0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, -1.0]), + ("Bottom", [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ("Front", [0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ("Back", [0.0, 0.0, -1.0], [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), +]; +/// (column, row) of each face in the 3:2 cube map. +const CELLS_3X2: [(u32, u32); 6] = [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1)]; + +/// VR world direction → After Effects world (Y down). +fn ae(v: [f64; 3]) -> Vec3 { + vec3(v[0], -v[1], v[2]) +} + +/// Composition ▸ VR ▸ Create VR Environment: renders the active comp's 3D scene in every +/// direction. Six face comps (a 90° camera each at `position`, default the comp centre, with a +/// copy of the scene's layers), a 3:2 cube map of them, and an equirectangular output comp +/// (VR Converter, cube map → equirectangular). One undo step. +fn create_vr_environment(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let src = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let size = p.get("size").and_then(Value::as_u64).unwrap_or(1024).clamp(64, 8192); + let center = match p.get("position").and_then(Value::as_array) { + Some(a) => [0, 1, 2].map(|i| a.get(i).and_then(Value::as_f64).unwrap_or(0.0)), + None => [src.width as f64 / 2.0, src.height as f64 / 2.0, 0.0], + }; + let fps = src.frame_rate.as_f64(); + let dur = src.duration.seconds(); + let r = grouped(s, "Create VR Environment", |s| { + let mut faces = vec![]; + for (fname, fw, r, u) in FACES { + let fc = id_of( + &s.execute( + "comp.new", + json!({"name": format!("{name} VR {fname}"), "width": size, "height": size, "frameRate": fps, "duration": dur, "open": false}), + )?, + "comp", + )?; + let o = orientation_for_axes(ae(r), ae([-u[0], -u[1], -u[2]]), ae(fw)); + let cam = + id_of(&s.execute("layer.newCamera", json!({"comp": fc, "name": format!("{fname} Camera"), "angleOfView": 90.0, "type": "oneNode"}))?, "layer")?; + s.execute("prop.set", json!({"comp": fc, "layer": cam, "path": "transform/position", "value": center}))?; + s.execute("prop.set", json!({"comp": fc, "layer": cam, "path": "transform/orientation", "value": o}))?; + // The scene's layers (not its cameras), in the scene's coordinates, under the face + // camera. + let layers: Vec = src.layers.iter().filter(|l| !l.is_camera()).cloned().collect(); + s.edit("Create VR Environment", None, |proj, _| { + let mut next = proj.next_id; + let mut map = std::collections::BTreeMap::new(); + let mut copies = vec![]; + for l in &layers { + let mut c = l.clone(); + super::edit::reid(&mut c, &mut next); + map.insert(l.id, c.id); + copies.push(c); + } + for c in &mut copies { + c.parent = c.parent.and_then(|p| map.get(&p).copied()); + if let Some(m) = &mut c.track_matte { + m.layer = map.get(&m.layer).copied().unwrap_or(m.layer); + } + } + proj.next_id = next; + let comp = proj.comp_mut(ItemId(fc)).ok_or(EngineError::NoComp)?; + comp.layers.extend(copies); + Ok(()) + })?; + faces.push(fc); + } + let cube = id_of( + &s.execute( + "comp.new", + json!({"name": format!("{name} VR Cube Map"), "width": size * 3, "height": size * 2, "frameRate": fps, "duration": dur, "open": false}), + )?, + "comp", + )?; + for (i, fc) in faces.iter().enumerate() { + let l = id_of(&s.execute("layer.addItem", json!({"comp": cube, "item": fc, "time": 0.0}))?, "layer")?; + let (c, rr) = CELLS_3X2[i]; + let pos = [(c as f64 + 0.5) * size as f64, (rr as f64 + 0.5) * size as f64]; + s.execute("prop.set", json!({"comp": cube, "layer": l, "path": "transform/position", "value": pos}))?; + } + let out = id_of( + &s.execute( + "comp.new", + json!({"name": format!("{name} VR Output"), "width": size * 2, "height": size, "frameRate": fps, "duration": dur, "open": false}), + )?, + "comp", + )?; + let l = id_of(&s.execute("layer.addItem", json!({"comp": out, "item": cube, "time": 0.0}))?, "layer")?; + s.execute("prop.set", json!({"comp": out, "layer": l, "path": "transform/scale", "value": [200.0 / 3.0, 50.0, 100.0]}))?; + s.execute("effect.apply", json!({"comp": out, "layers": [l], "effect": "ec.vr.converter"}))?; + s.execute("prop.set", json!({"comp": out, "layer": l, "path": "effects/#1/sourceProjection", "value": 2}))?; + s.execute("prop.set", json!({"comp": out, "layer": l, "path": "effects/#1/targetProjection", "value": 0}))?; + Ok(json!({"faces": faces, "cubeMap": cube, "output": out})) + })?; + s.open_comp(ItemId(r["output"].as_u64().unwrap_or(0))); + Ok(r) +} + +/// Composition ▸ VR ▸ Extract Cubemap: a 3:2 cube-map comp of an equirectangular comp (VR +/// Converter, equirectangular → cube map), for editing the faces. One undo step. +fn extract_cubemap(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let src = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let face = p.get("faceSize").and_then(Value::as_u64).unwrap_or((src.width / 4).max(16) as u64).clamp(16, 8192); + let (w, h) = (face * 3, face * 2); + let r = grouped(s, "Extract Cubemap", |s| { + let cube = id_of( + &s.execute( + "comp.new", + json!({"name": format!("{name} Cubemap"), "width": w, "height": h, "frameRate": src.frame_rate.as_f64(), "duration": src.duration.seconds(), "open": false}), + )?, + "comp", + )?; + let l = id_of(&s.execute("layer.addItem", json!({"comp": cube, "item": cid.0, "time": 0.0}))?, "layer")?; + let scale = [w as f64 / src.width.max(1) as f64 * 100.0, h as f64 / src.height.max(1) as f64 * 100.0, 100.0]; + s.execute("prop.set", json!({"comp": cube, "layer": l, "path": "transform/scale", "value": scale}))?; + s.execute("effect.apply", json!({"comp": cube, "layers": [l], "effect": "ec.vr.converter"}))?; + s.execute("prop.set", json!({"comp": cube, "layer": l, "path": "effects/#1/sourceProjection", "value": 0}))?; + s.execute("prop.set", json!({"comp": cube, "layer": l, "path": "effects/#1/targetProjection", "value": 2}))?; + Ok(json!({"cubeMap": cube, "layer": l, "faceSize": face})) + })?; + s.open_comp(ItemId(r["cubeMap"].as_u64().unwrap_or(0))); + Ok(r) +} + +pub fn specs() -> Vec { + vec![ + cmd!("edit.history", "History", [], None, "{steps?: n (undo n steps), redo?: n}", has_history, history), + cmd!("file.importRecent", "Import Recent Footage", [], None, "{index? | path?}", has_recent_footage, import_recent), + cmd!("file.clearRecentFootage", "Clear Recent Footage", ["File", "Import Recent Footage"], None, "{}", has_recent_footage, clear_recent_footage), + cmd!("anim.applyRecentPreset", "Recent Animation Presets", [], None, "{index? | path?, layers?}", has_recent_presets, apply_recent_preset), + cmd!( + "anim.clearRecentPresets", + "Clear Recent Presets", + ["Animation", "Recent Animation Presets"], + None, + "{}", + |s| if s.prefs.recent_presets.is_empty() { Err("no recent animation presets".into()) } else { Ok(()) }, + clear_recent_presets + ), + cmd!("file.saveCopyAsXml", "Save a Copy As XML...", ["File", "Save As"], None, "{path: .ecprojx}", always, save_copy_xml), + cmd!("file.replaceWithLayeredComp", "With Layered Comp", ["File", "Replace Footage"], None, "{item?}", has_footage, replace_with_layered_comp), + cmd!("file.executeFile", "Execute File", [], None, "{path, confirmed?}", always, execute_file), + cmd!("view.assign3dShortcut", "Assign Shortcut to 3D View", [], None, "{slot: F10|F11|F12, comp?}", has_comp, assign_view_shortcut), + cmd!( + "window.assignWorkspaceShortcut", + "Assign Shortcut to Workspace", + [], + None, + "{slot: Shift+F10|Shift+F11|Shift+F12, workspace?}", + always, + assign_workspace_shortcut + ), + cmd!("help.enableLogging", "Enable Logging", ["Help"], None, "{on?}", always, enable_logging), + cmd!("help.revealLogFile", "Reveal Logging File", ["Help"], None, "{}", always, reveal_log), + cmd!("help.systemReport", "System Compatibility Report...", ["Help"], None, "{quiet?}", always, report), + cmd!( + "comp.vr.createEnvironment", + "Create VR Environment...", + ["Composition", "VR"], + None, + "{comp?, size?: face pixels (1024), position?: [x,y,z]}", + has_comp, + create_vr_environment + ), + cmd!("comp.vr.extractCubemap", "Extract Cubemap...", ["Composition", "VR"], None, "{comp?, faceSize?}", has_comp, extract_cubemap), + query!("help.systemInfo", "System Information", "{}", |s, _| Ok(system_report(s))), + ] +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index 03cd8087..44f49466 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -384,6 +384,8 @@ fn paste(s: &mut Session, p: &Value) -> Result { fn split(s: &mut Session, p: &Value) -> Result { let (cid, ids) = layers_p(s, p)?; let t = s.time(); + // Settings ▸ General ▸ Create Split Layers Above Original Layer. + let above = s.prefs.general.create_split_layers_above; let new = s.edit("Split Layer", None, |proj, st| { let mut next = proj.next_id; let comp = proj.comp_mut(cid).ok_or(crate::EngineError::NoComp)?; @@ -398,7 +400,7 @@ fn split(s: &mut Session, p: &Value) -> Result { b.in_point = t; comp.layers[i].out_point = t; created.push(b.id); - comp.layers.insert(i, b); + comp.layers.insert(if above { i } else { i + 1 }, b); } proj.next_id = next; st.selected_layers = created.clone(); @@ -408,6 +410,11 @@ fn split(s: &mut Session, p: &Value) -> Result { } fn label(s: &mut Session, p: &Value) -> Result { + // Keyframe colour labels: selected keyframes take the label (unless layers are named). + let explicit_layers = p.get("layers").is_some() || p.get("layer").is_some(); + if p.get("keys").is_some() || (!s.state.selected_keys.is_empty() && !explicit_layers && str_p(p, "target") != Some("layers")) { + return super::keys_more::label_keys(s, p); + } let name = str_p(p, "label").unwrap_or("Red"); let lab = s.prefs.label_from_name(name).ok_or_else(|| super::bad("edit.label", format!("unknown label `{name}`")))?; let (cid, ids) = layers_p(s, p)?; @@ -624,7 +631,7 @@ pub fn specs() -> Vec { cmd!("edit.extractWorkArea", "Extract Work Area", ["Edit"], None, "{layers?}", has_comp, extract), cmd!("edit.selectAll", "Select All", ["Edit"], Some("Cmd+A"), "{}", has_comp, select_all), cmd!("edit.deselectAll", "Deselect All", ["Edit"], Some("Cmd+Shift+A"), "{}", always_ok, deselect_all), - cmd!("edit.label", "Label", ["Edit", "Label"], None, "{label: Red|Yellow|Aqua|…, layers?}", always_ok, label), + cmd!("edit.label", "Label", ["Edit", "Label"], None, "{label: Red|Yellow|Aqua|…, layers?, keys?, target?: layers}", always_ok, label), cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group), cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge), cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 6999ac9d..abe3596c 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -40,6 +40,8 @@ pub(crate) fn open(s: &mut Session, p: &Value) -> Result { let path = str_p(p, "path").ok_or_else(|| bad("file.open", "missing `path`"))?; let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; let text = String::from_utf8(bytes).map_err(|_| EngineError::Other("not a text project file".into()))?; + // XML copies (File ▸ Save a Copy As XML…) open like the JSON project. + let text = if crate::xml_project::is_xml(&text) { crate::xml_project::from_xml(&text).map_err(EngineError::Other)? } else { text }; let proj = Project::from_json(&text)?; s.replace_project(proj, Some(path.to_string())); s.note_project_path(path); @@ -48,6 +50,8 @@ pub(crate) fn open(s: &mut Session, p: &Value) -> Result { fn save_to(s: &mut Session, path: &str) -> Result { let json = s.project.to_json(); + // A project opened from an XML copy saves as XML again. + let json = if crate::xml_project::is_xml_path(path) { crate::xml_project::to_xml(&json).map_err(EngineError::Other)? } else { json }; s.services.write_file(path, json.as_bytes()).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; s.path = Some(path.to_string()); s.saved_revision = s.revision; @@ -78,14 +82,20 @@ fn revert(s: &mut Session, _: &Value) -> Result { open(s, &json!({"path": path})) } -fn import(s: &mut Session, p: &Value) -> Result { +pub(crate) fn import(s: &mut Session, p: &Value) -> Result { let paths: Vec = match p.get("paths").or(p.get("path")) { Some(Value::Array(a)) => a.iter().filter_map(|v| v.as_str().map(str::to_string)).collect(), Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; // Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes). - let import_as = str_p(p, "importAs").unwrap_or("footage"); + // Files dragged in follow Settings ▸ Import ▸ Default Drag Import As. + let drag = p.get("drag").and_then(Value::as_bool).unwrap_or(false); + let import_as = str_p(p, "importAs").unwrap_or(if drag { s.prefs.drag_import_as() } else { "footage" }); + for path in &paths { + s.prefs.push_recent_footage(path); + } + s.save_prefs(); let retain = match import_as { "footage" => None, "composition" | "comp" => Some(false), @@ -127,9 +137,25 @@ fn import(s: &mut Session, p: &Value) -> Result { let mut probed = vec![]; let psd_layer = p.get("layer").cloned(); let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); + let mut ask_alpha = vec![]; + let mut warnings = vec![]; for path in &paths { match importer.probe(path) { Ok(mut f) => { + // Settings ▸ Import ▸ Report Missing Frames. + if f.kind == FootageKind::Sequence + && s.prefs.import.report_missing_frames + && let Some(gaps) = missing_frames(&f.sequence) + { + warnings.push(format!("{path}: {gaps}")); + } + // Settings ▸ Import ▸ Interpret Unlabeled Alpha As. + if f.alpha != effectcraft_project::AlphaMode::Ignore && !alpha_is_labeled(path, &f.codec) { + match s.prefs.unlabeled_alpha(f.kind == FootageKind::Video) { + Some(a) => f.alpha = a, + None => ask_alpha.push(paths.iter().position(|x| x == path).unwrap_or(0)), + } + } // Settings ▸ Import ▸ Sequence Footage frames per second. if f.kind == FootageKind::Sequence { let r = seq_rate; @@ -154,6 +180,7 @@ fn import(s: &mut Session, p: &Value) -> Result { Err(e) => errors.push(format!("{path}: {e}")), } } + let mut asked: Vec = vec![]; s.edit("Import", None, |proj, st| { for (path, f) in probed { let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); @@ -166,8 +193,12 @@ fn import(s: &mut Session, p: &Value) -> Result { FootageKind::Audio => Label::SeaFoam, FootageKind::Video | FootageKind::Model => Label::Aqua, }; + let ask = ask_alpha.contains(&paths.iter().position(|x| *x == path).unwrap_or(usize::MAX)); let id = proj.add_item(&name, label, None, ItemKind::Footage(f)); ids.push(id.0); + if ask { + asked.push(id.0); + } st.project_selection = vec![id]; } Ok(()) @@ -175,7 +206,44 @@ fn import(s: &mut Session, p: &Value) -> Result { if let Some(c) = out_comps.first() { s.open_comp(effectcraft_project::ItemId(*c)); } - Ok(json!({"items": ids, "comps": out_comps, "errors": errors})) + for w in &warnings { + s.events.push(crate::Event::Toast { message: w.clone(), error: true }); + } + // Ask User: the frontend opens Interpret Footage for footage with unlabeled alpha. + if let Some(first) = asked.first() { + s.events.push(crate::Event::Frontend { command: "file.interpretFootage".into(), params: json!({"item": first, "reason": "unlabeledAlpha"}) }); + } + Ok(json!({"items": ids, "comps": out_comps, "errors": errors, "warnings": warnings, "unlabeledAlpha": asked})) +} + +/// Whether a file format states how its alpha is stored (PNG, GIF, WebP and PSD are straight by +/// definition, OpenEXR premultiplied); TGA, TIFF and movies with alpha don't. +fn alpha_is_labeled(path: &str, codec: &str) -> bool { + let ext = std::path::Path::new(path).extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default(); + matches!(ext.as_str(), "png" | "gif" | "webp" | "psd" | "psb" | "exr" | "svg" | "jpg" | "jpeg" | "ai" | "pdf" | "eps") || codec.eq_ignore_ascii_case("PSD") +} + +/// Gaps in an image sequence's frame numbers ("missing frames 4–6, 9"), from the digit run at the +/// end of each file name. `None` when the numbering is continuous (or there is none). +pub(crate) fn missing_frames(files: &[String]) -> Option { + let num = |f: &String| -> Option { + let stem = std::path::Path::new(f).file_stem()?.to_string_lossy().to_string(); + let digits: String = stem.chars().rev().take_while(char::is_ascii_digit).collect::>().into_iter().rev().collect(); + digits.parse().ok() + }; + let mut ns: Vec = files.iter().filter_map(num).collect(); + ns.sort_unstable(); + ns.dedup(); + let mut gaps = vec![]; + let mut total = 0; + for w in ns.windows(2) { + if w[1] - w[0] > 1 { + let (a, b) = (w[0] + 1, w[1] - 1); + total += b - a + 1; + gaps.push(if a == b { a.to_string() } else { format!("{a}–{b}") }); + } + } + (!gaps.is_empty()).then(|| format!("{total} missing frame{} ({})", if total == 1 { "" } else { "s" }, gaps.join(", "))) } /// A Photoshop layer by index or name (pixel layers only). @@ -189,7 +257,7 @@ fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, Strin } /// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings). -fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { +pub(crate) fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into()); let rate = effectcraft_time::FrameRate::FPS_29_97; @@ -313,7 +381,7 @@ pub fn specs() -> Vec { "File...", ["File", "Import"], Some("Cmd+I"), - "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer), drag?: bool (dropped files: Settings ▸ Import ▸ Default Drag Import As)}", always, import ), diff --git a/crates/engine/src/commands/keys_more.rs b/crates/engine/src/commands/keys_more.rs new file mode 100644 index 00000000..0051d945 --- /dev/null +++ b/crates/engine/src/commands/keys_more.rs @@ -0,0 +1,298 @@ +//! Keyframe features of M14.5: Animation ▸ Keyframe Assistant ▸ Convert Audio to Keyframes and +//! RPF Camera Import, keyframe colour labels (Edit ▸ Label on selected keyframes) and Edit ▸ +//! Select Keyframe Label Group. + +use effectcraft_color::Label; +use effectcraft_keyframe::{Keyframe, Value as KV}; +use effectcraft_project::build::Ids; +use effectcraft_project::{LayerId, Uid}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::app_more::grouped; +use super::{CommandSpec, bad, comp_id, has_comp, has_keys, layer_mut, str_p}; +use crate::{EngineError, KeyRef, Result, Session, cmd}; + +// ---------------------------------------------------------------- keyframe labels + +/// The label index (0 = None, 1–16) of a label name (built-in or renamed in Settings ▸ Labels). +fn label_index(s: &Session, name: &str) -> Option { + let l = if name.eq_ignore_ascii_case("none") { Label::None } else { s.prefs.label_from_name(name)? }; + Label::ALL.iter().position(|x| *x == l).map(|i| i as u8) +} + +fn keys_p(s: &Session, p: &Value) -> Vec { + match p.get("keys") { + Some(Value::Array(a)) => a.iter().filter_map(|v| serde_json::from_value(v.clone()).ok()).collect(), + _ => s.state.selected_keys.clone(), + } +} + +/// Edit ▸ Label on keyframes: colour the selected (or given) keyframes. +pub(crate) fn label_keys(s: &mut Session, p: &Value) -> Result { + let name = str_p(p, "label").unwrap_or("Red"); + let idx = label_index(s, name).ok_or_else(|| bad("keys.label", format!("unknown label `{name}`")))?; + let keys = keys_p(s, p); + if keys.is_empty() { + return Err(bad("keys.label", "select keyframes first")); + } + let cid = comp_id(s, p)?; + let n = s.edit("Keyframe Label", None, |proj, _| { + let mut n = 0; + for k in &keys { + let Ok(l) = layer_mut(proj, cid, k.layer) else { continue }; + let Some(pr) = l.props.find_mut(k.prop) else { continue }; + for key in pr.keys.iter_mut().filter(|x| x.time == k.time) { + key.label = idx; + n += 1; + } + } + Ok(n) + })?; + Ok(json!({"keys": n, "label": idx})) +} + +/// Edit ▸ Select Keyframe Label Group: select every keyframe with the label of a selected +/// keyframe. `scope`: `selected` (selected layers), `all` (all layers), `visibleSelected` / +/// `visibleAll` (only properties the frontend shows, given as `props`: [uid]; agents without a +/// timeline may omit it, then every property counts). +fn select_label_group(s: &mut Session, p: &Value) -> Result { + let scope = str_p(p, "scope").unwrap_or("selected"); + if !matches!(scope, "selected" | "all" | "visibleSelected" | "visibleAll") { + return Err(bad("keys.selectLabelGroup", "scope: selected|all|visibleSelected|visibleAll")); + } + let comp = s.active_comp().ok_or(EngineError::NoComp)?; + let labels: Vec = s + .state + .selected_keys + .iter() + .filter_map(|k| comp.layer(k.layer).and_then(|l| l.props.find(k.prop)).and_then(|pr| pr.keys.iter().find(|x| x.time == k.time)).map(|x| x.label)) + .collect(); + if labels.is_empty() { + return Err(bad("keys.selectLabelGroup", "select a keyframe first")); + } + let visible: Option> = p.get("props").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).collect()); + let layers: Vec = + if scope == "all" || scope == "visibleAll" { comp.layers.iter().map(|l| l.id).collect() } else { s.state.selected_layers.clone() }; + let mut sel = vec![]; + for l in comp.layers.iter().filter(|l| layers.contains(&l.id)) { + l.props.walk("", &mut |_, pr| { + if scope.starts_with("visible") && visible.as_ref().is_some_and(|v| !v.contains(&pr.uid)) { + return; + } + for k in pr.keys.iter().filter(|k| labels.contains(&k.label)) { + sel.push(KeyRef { layer: l.id, prop: pr.uid, time: k.time }); + } + }); + } + s.state.selected_keys = sel; + Ok(json!({"keys": s.state.selected_keys.len()})) +} + +// ---------------------------------------------------------------- Convert Audio to Keyframes + +/// Per-frame amplitude of interleaved stereo: (left, right, both), mean absolute sample × 100. +pub fn amplitudes(buf: &[f32]) -> (f64, f64, f64) { + let n = (buf.len() / 2).max(1) as f64; + let (mut l, mut r) = (0.0f64, 0.0f64); + for c in buf.chunks_exact(2) { + l += c[0].abs() as f64; + r += c[1].abs() as f64; + } + let (l, r) = (l / n * 100.0, r / n * 100.0); + (l, r, (l + r) / 2.0) +} + +/// Animation ▸ Keyframe Assistant ▸ Convert Audio to Keyframes: an "Audio Amplitude" null with +/// Left Channel, Right Channel and Both Channels sliders keyed on every frame of the work area +/// with the amplitude of the comp's audible layers. One undo step. +fn audio_to_keys(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let fr = comp.frame_rate; + let (a, b) = comp.work_area; + let (f0, f1) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(fr.snap_nearest(b)).max(fr.frame_at(fr.snap_nearest(a)) + 1)); + const RATE: u32 = 48_000; + let per = ((RATE as f64 / fr.as_f64()).round() as usize).max(1); + let mut rows = vec![]; + for f in f0..f1 { + let t = fr.tick_of(f); + let buf = effectcraft_render::audio::mix_comp(&s.project, s.footage.as_ref(), s.expr.as_deref(), cid, t, per, RATE); + rows.push((t, amplitudes(&buf))); + } + let spec = effectcraft_effects::lookup("ec.control.slider").ok_or_else(|| EngineError::Other("Slider Control is missing".into()))?; + let peak = rows.iter().map(|(_, (_, _, both))| *both).fold(0.0, f64::max); + let r = grouped(s, "Convert Audio to Keyframes", |s| { + let nl = s.execute("layer.newNull", json!({"comp": cid.0, "name": "Audio Amplitude"}))?["layer"].as_u64().unwrap_or(0); + let lid = LayerId(nl); + s.edit("Convert Audio to Keyframes", None, |proj, st| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let start = l.start_time; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad("keys.audioToKeyframes", "the null has no effects group"))?; + for (i, name) in ["Left Channel", "Right Channel", "Both Channels"].into_iter().enumerate() { + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), name, [comp.width as f64, comp.height as f64]); + if let Some(pr) = g.get_mut("slider") { + pr.keys = rows + .iter() + .map(|(t, amp)| { + let v = [amp.0, amp.1, amp.2][i]; + Keyframe::new(*t - start, KV::Scalar((v * 100.0).round() / 100.0)) + }) + .collect(); + } + fx.children.push(g.into()); + } + proj.next_id = next; + st.selected_layers = vec![lid]; + Ok(()) + })?; + Ok(json!({"layer": nl, "frames": rows.len(), "peak": peak})) + })?; + Ok(r) +} + +// ---------------------------------------------------------------- RPF Camera Import + +/// One camera sample: time, position, orientation (degrees, X/Y/Z) and zoom (px). +#[derive(Clone, Debug, PartialEq)] +pub struct CamSample { + pub time: Tick, + pub position: [f64; 3], + pub orientation: [f64; 3], + pub zoom: Option, +} + +fn zoom_of(fov: Option, zoom: Option, comp_w: f64) -> Option { + zoom.or_else(|| fov.map(|a| comp_w * 0.5 / (a.clamp(0.1, 179.0).to_radians() * 0.5).tan())) +} + +/// Parse camera data: JSON `{frameRate?, frames: [{frame | time, position, orientation | +/// rotation, zoom? | fov?}]}` or CSV with a header naming `frame`/`time`, `px py pz` +/// (or `x y z`), `rx ry rz` and `zoom`/`fov` columns. +pub fn parse_camera(text: &str, rate: effectcraft_time::FrameRate, comp_w: f64) -> std::result::Result, String> { + let t = text.trim_start(); + if t.starts_with('{') || t.starts_with('[') { + let v: Value = serde_json::from_str(t).map_err(|e| format!("camera JSON: {e}"))?; + let rate = v.get("frameRate").and_then(Value::as_f64).map(effectcraft_time::FrameRate::from_f64).unwrap_or(rate); + let frames = v.get("frames").or(Some(&v)).and_then(Value::as_array).ok_or("camera JSON: expected `frames`")?; + let v3 = |x: Option<&Value>| -> Option<[f64; 3]> { + let a = x?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + return frames + .iter() + .enumerate() + .map(|(i, f)| { + let time = match (f.get("time").and_then(Value::as_f64), f.get("frame").and_then(Value::as_i64)) { + (Some(s), _) => Tick::from_seconds_f64(s), + (None, Some(n)) => rate.tick_of(n), + _ => rate.tick_of(i as i64), + }; + Ok(CamSample { + time, + position: v3(f.get("position")).ok_or(format!("camera JSON: frame {i} needs `position` [x,y,z]"))?, + orientation: v3(f.get("orientation").or(f.get("rotation"))).unwrap_or([0.0; 3]), + zoom: zoom_of(f.get("fov").and_then(Value::as_f64), f.get("zoom").and_then(Value::as_f64), comp_w), + }) + }) + .collect(); + } + let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')); + let head: Vec = lines.next().ok_or("camera CSV: empty")?.split([',', ';', '\t']).map(|h| h.trim().to_ascii_lowercase()).collect(); + let col = |names: &[&str]| head.iter().position(|h| names.contains(&h.as_str())); + let (cf, ct) = (col(&["frame", "f"]), col(&["time", "seconds", "t"])); + let pos = [col(&["px", "x", "posx", "position x"]), col(&["py", "y", "posy", "position y"]), col(&["pz", "z", "posz", "position z"])]; + let rot = [col(&["rx", "rotx", "orientation x", "tilt"]), col(&["ry", "roty", "orientation y", "pan"]), col(&["rz", "rotz", "orientation z", "roll"])]; + let (cz, cfov) = (col(&["zoom"]), col(&["fov", "angle", "angleofview"])); + if pos.iter().any(Option::is_none) { + return Err("camera CSV: needs px, py, pz (or x, y, z) columns".into()); + } + lines + .enumerate() + .map(|(i, l)| { + let cells: Vec = l.split([',', ';', '\t']).map(|c| c.trim().parse().unwrap_or(f64::NAN)).collect(); + let get = |c: Option| c.and_then(|c| cells.get(c).copied()).filter(|v| v.is_finite()); + let time = match (get(ct), get(cf)) { + (Some(s), _) => Tick::from_seconds_f64(s), + (None, Some(n)) => rate.tick_of(n as i64), + _ => rate.tick_of(i as i64), + }; + Ok(CamSample { + time, + position: pos.map(|c| get(c).unwrap_or(0.0)), + orientation: rot.map(|c| get(c).unwrap_or(0.0)), + zoom: zoom_of(get(cfov), get(cz), comp_w), + }) + }) + .collect() +} + +/// Animation ▸ Keyframe Assistant ▸ RPF Camera Import: a camera layer keyed from camera data. +/// RPF/RLA files carry their camera in an undocumented block, so EffectCraft reads the same data +/// from JSON or CSV exports (see `docs/preferences.md` ▸ RPF Camera Import). One undo step. +fn rpf_camera_import(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("keys.rpfCameraImport", "missing `path` (.json or .csv camera data)"))?; + let lower = path.to_ascii_lowercase(); + if lower.ends_with(".rpf") || lower.ends_with(".rla") { + return Err(bad( + "keys.rpfCameraImport", + "the camera block of RPF/RLA files is not publicly documented: export the camera from your 3D application as JSON or CSV (frame, px, py, pz, rx, ry, rz, zoom|fov)", + )); + } + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let samples = parse_camera(&String::from_utf8_lossy(&bytes), comp.frame_rate, comp.width as f64).map_err(|e| bad("keys.rpfCameraImport", e))?; + if samples.is_empty() { + return Err(bad("keys.rpfCameraImport", "no camera samples")); + } + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "RPF Camera".into()); + grouped(s, "RPF Camera Import", |s| { + let cam = s.execute("layer.newCamera", json!({"comp": cid.0, "name": name, "type": "oneNode"}))?["layer"].as_u64().unwrap_or(0); + let lid = LayerId(cam); + s.edit("RPF Camera Import", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let start = l.start_time; + let keyed = |vals: Vec<(Tick, KV)>| -> Vec { vals.into_iter().map(|(t, v)| Keyframe::new(t - start, v)).collect() }; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.keys = keyed(samples.iter().map(|c| (c.time, KV::Vec3(c.position))).collect()); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.keys = keyed(samples.iter().map(|c| (c.time, KV::Vec3(c.orientation))).collect()); + } + if samples.iter().any(|c| c.zoom.is_some()) + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys = keyed(samples.iter().filter_map(|c| Some((c.time, KV::Scalar(c.zoom?)))).collect()); + } + Ok(()) + })?; + Ok(json!({"layer": cam, "keyframes": samples.len()})) + }) +} + +pub fn specs() -> Vec { + vec![ + cmd!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], None, "{comp?}", has_comp, audio_to_keys), + cmd!( + "keys.rpfCameraImport", + "RPF Camera Import", + ["Animation", "Keyframe Assistant"], + None, + "{path: .json|.csv camera data, comp?}", + has_comp, + rpf_camera_import + ), + cmd!( + "keys.selectLabelGroup", + "Select Keyframe Label Group", + [], + None, + "{scope: selected|all|visibleSelected|visibleAll, props?: [uid]}", + has_keys, + select_label_group + ), + cmd!("keys.label", "Keyframe Label", [], None, "{label, keys?: [{layer, prop, time}]}", has_keys, label_keys), + ] +} diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index 7b44096e..f34ed2f5 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -822,12 +822,13 @@ fn add_mask(s: &mut Session, p: &Value) -> Result { _ => ShapePath::rect([cx, cy], rw, rh), }; let mode = str_p(p, "mode").and_then(MaskMode::from_name).unwrap_or(MaskMode::Add); + let cycle = s.prefs.appearance.cycle_mask_colors; let uid = s.edit("New Mask", None, |proj, _| { let mut next = proj.next_id; let l = layer_mut(proj, cid, lid)?; let masks = l.props.sub_mut("masks").ok_or_else(|| bad("layer.addMask", "this layer can't have masks"))?; let n = masks.children.len(); - let g = build::mask(&mut Ids(&mut next), &format!("Mask {}", n + 1), path, mode, build::MASK_COLORS[n % build::MASK_COLORS.len()]); + let g = build::mask(&mut Ids(&mut next), &format!("Mask {}", n + 1), path, mode, crate::prefs::mask_color(cycle, n)); let uid = g.uid; masks.children.push(g.into()); proj.next_id = next; @@ -909,6 +910,11 @@ fn add_shape_item(s: &mut Session, p: &Value) -> Result { fn set_text(s: &mut Session, p: &Value) -> Result { let (cid, lid) = layer_p(s, p, "layer.setText")?; + // The Character panel's recent fonts (Settings ▸ Type ▸ Number of Recent Fonts to Display). + if let Some(f) = str_p(p, "font") { + s.prefs.push_recent_font(f); + s.save_prefs(); + } let t = s.time(); let range = text_range_p(p); // Character attributes with only a caret (no selected text) set the insertion style. diff --git a/crates/engine/src/commands/layer_menu.rs b/crates/engine/src/commands/layer_menu.rs index b441c366..263d0d58 100644 --- a/crates/engine/src/commands/layer_menu.rs +++ b/crates/engine/src/commands/layer_menu.rs @@ -652,33 +652,38 @@ fn reveal_comp(s: &mut Session, p: &Value) -> Result { Ok(json!({"item": cid.0})) } -fn reveal_expression_errors(s: &mut Session, p: &Value) -> Result { - let cid = comp_id(s, p)?; - let t = s.time(); - let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; - let mut errors = vec![]; - for l in &comp.layers { - let mut has = vec![]; - l.props.walk("", &mut |_, pr| { - if pr.has_expression() { - has.push(pr); - } - }); - for pr in has { - let text = pr.expr.as_ref().map(|e| e.text.clone()).unwrap_or_default(); - let err = match &s.expr { - Some(host) => { - let mut ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); - ctx.expr = Some(host.as_ref()); - host.eval(&ctx, l, pr, &pr.value_at(l.layer_time(t))).err() +impl Session { + /// Expressions of comp `cid` that fail at its current time: `{layer, prop, name, expression, + /// error}` (Reveal Expression Errors, the viewer's expression error banner). + pub fn expression_errors(&self, cid: effectcraft_project::ItemId) -> Vec { + let t = self.time_of(cid); + let Some(comp) = self.project.comp(cid) else { return vec![] }; + let Some(host) = &self.expr else { return vec![] }; + let mut errors = vec![]; + for l in &comp.layers { + let mut has = vec![]; + l.props.walk("", &mut |_, pr| { + if pr.has_expression() { + has.push(pr); + } + }); + for pr in has { + let text = pr.expr.as_ref().map(|e| e.text.clone()).unwrap_or_default(); + let mut ctx = effectcraft_render::EvalCtx::new(&self.project, cid, comp, t); + ctx.expr = Some(host.as_ref()); + if let Err(e) = host.eval(&ctx, l, pr, &pr.value_at(l.layer_time(t))) { + errors.push(json!({"layer": l.id.0, "prop": pr.uid, "name": pr.name, "expression": text, "error": e})); } - None => None, - }; - if let Some(e) = err { - errors.push(json!({"layer": l.id.0, "prop": pr.uid, "name": pr.name, "expression": text, "error": e})); } } + errors } +} + +fn reveal_expression_errors(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + s.project.comp(cid).ok_or(EngineError::NoComp)?; + let errors = s.expression_errors(cid); let props: Vec = errors.iter().map(|e| json!({"layer": e["layer"], "prop": e["prop"]})).collect(); frontend(s, "timeline.revealProps", &json!({"props": props}))?; if errors.is_empty() { diff --git a/crates/engine/src/commands/link.rs b/crates/engine/src/commands/link.rs index 1e070ab5..3892bd06 100644 --- a/crates/engine/src/commands/link.rs +++ b/crates/engine/src/commands/link.rs @@ -101,7 +101,7 @@ fn quote(s: &str) -> String { } /// The AE expression reference to property `uid` of `target` as seen from a property of `from`. -pub(crate) fn reference(comp: &Comp, from: &Layer, target: &Layer, uid: Uid) -> Option { +pub(crate) fn reference(comp: &Comp, from: &Layer, target: &Layer, uid: Uid, compact: bool) -> Option { let chain = target.props.node_chain(uid)?; let mut parts: Vec = vec![]; let mut pending: Option<&str> = None; @@ -125,6 +125,15 @@ pub(crate) fn reference(comp: &Comp, from: &Layer, target: &Layer, uid: Uid) -> } let mut out = if from.id == target.id { String::new() } else { format!("thisComp.layer({}).", quote(&target.name)) }; let _ = comp; + if !compact { + // Match names: language-independent, like After Effects with Compact English off. + let chain = target.props.node_chain(uid)?; + let mut out = if from.id == target.id { "thisLayer".to_string() } else { format!("thisComp.layer({})", quote(&target.name)) }; + for n in chain { + out.push_str(&format!("({})", quote(n.match_id()))); + } + return Some(out); + } for (i, p) in parts.iter().enumerate() { if i > 0 && !p.starts_with('(') { out.push('.'); @@ -152,7 +161,7 @@ fn pick_whip(s: &mut Session, p: &Value) -> Result { let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; let from = comp.layer(lid).ok_or(EngineError::NoComp)?; let target = comp.layer(tl).ok_or(EngineError::NoComp)?; - let r = reference(comp, from, target, tu).ok_or_else(|| bad("prop.pickWhip", "no such target property"))?; + let r = reference(comp, from, target, tu, s.prefs.general.expression_pick_whip_compact).ok_or_else(|| bad("prop.pickWhip", "no such target property"))?; let (me, them) = (from.props.find(uid).ok_or(EngineError::NoComp)?, target.props.find(tu).ok_or(EngineError::NoComp)?); let (dm, dt) = (dims(from, me), dims(target, them)); let text = if dt == 1 && dm > 1 { diff --git a/crates/engine/src/commands/mask.rs b/crates/engine/src/commands/mask.rs index 39894b58..f34d673b 100644 --- a/crates/engine/src/commands/mask.rs +++ b/crates/engine/src/commands/mask.rs @@ -54,9 +54,76 @@ pub(crate) fn path_group<'a>(props: &'a mut PropGroup, key: &Value) -> Option<&' /// Edit the mask path of (layer, mask) at the current time. fn edit_path(s: &mut Session, p: &Value, label: &str, f: impl FnOnce(&mut ShapePath) -> Result) -> Result<(Uid, T)> { + edit_path_counted(s, p, label, |sp| f(sp).map(|r| (r, None))) +} + +/// A change of a path's vertex count, repeated on the other keyframes of an animated path when +/// Settings ▸ General ▸ Preserve Constant Vertex and Feather Point Count is on. +#[derive(Clone, Debug)] +pub(crate) enum CountOp { + /// A vertex was inserted at this index. + Insert(usize), + /// These vertices were removed. + Remove(Vec), +} + +/// Apply a vertex-count change to another keyframe's path: an inserted vertex splits the +/// matching segment in half (the shape doesn't change), removed vertices go. +pub(crate) fn apply_count_op(sp: &mut ShapePath, op: &CountOp) { + match op { + CountOp::Insert(at) => { + let n = sp.vertices.len(); + if n == 0 { + return; + } + if *at > 0 && split_segment(sp, at - 1, 0.5).is_some() { + return; + } + let at = (*at).min(n); + let v = sp.vertices[at.min(n - 1)]; + sp.in_tangents.resize(n, [0.0; 2]); + sp.out_tangents.resize(n, [0.0; 2]); + sp.vertices.insert(at, v); + sp.in_tangents.insert(at, [0.0; 2]); + sp.out_tangents.insert(at, [0.0; 2]); + } + CountOp::Remove(idx) => { + let mut idx = idx.clone(); + idx.sort_unstable(); + idx.dedup(); + for &i in idx.iter().rev() { + if i < sp.vertices.len() { + sp.vertices.remove(i); + if i < sp.in_tangents.len() { + sp.in_tangents.remove(i); + } + if i < sp.out_tangents.len() { + sp.out_tangents.remove(i); + } + } + } + if sp.vertices.len() < 3 { + sp.closed = false; + } + } + } +} + +/// Repeat `op` on every keyframe of `pr` other than the one at `lt`. +fn sync_keys(pr: &mut effectcraft_project::Property, lt: effectcraft_time::Tick, op: &CountOp) { + for k in pr.keys.iter_mut().filter(|k| k.time != lt) { + if let KV::Path(sp) = &mut k.value { + apply_count_op(sp, op); + } + } +} + +/// [`edit_path`] whose edit may change the vertex count (`f` says how). +fn edit_path_counted(s: &mut Session, p: &Value, label: &str, f: impl FnOnce(&mut ShapePath) -> Result<(T, Option)>) -> Result<(Uid, T)> { let (cid, lid) = layer_p(s, p, label)?; let key = p.get("mask").cloned().ok_or_else(|| bad(label, "missing `mask` (uid, index or name)"))?; let t = s.time(); + let preserve = s.prefs.general.preserve_constant_vertex_count; s.edit(label, merge_p(p), |proj, _| { let l = layer_mut(proj, cid, lid)?; let lt = l.layer_time(t); @@ -65,11 +132,14 @@ fn edit_path(s: &mut Session, p: &Value, label: &str, f: impl FnOnce(&mut Sha let roto = super::paths::is_roto(g); let pr = g.get_mut("path").ok_or_else(|| bad(label, "mask has no path"))?; let KV::Path(mut sp) = pr.value_at(lt) else { return Err(bad(label, "not a path")) }; - let r = f(&mut sp)?; + let (r, op) = f(&mut sp)?; if roto { super::paths::roto_smooth(&mut sp); } pr.set_value_at(lt, KV::Path(sp)); + if let Some(op) = op.filter(|_| preserve) { + sync_keys(pr, lt, &op); + } Ok((uid, r)) }) } @@ -88,12 +158,13 @@ fn new_mask(s: &mut Session, p: &Value) -> Result { outs.resize(n, [0.0; 2]); let path = ShapePath { vertices: v, in_tangents: ins, out_tangents: outs, closed: b_p(p, "closed").unwrap_or(false) }; let mode = str_p(p, "mode").and_then(MaskMode::from_name).unwrap_or(MaskMode::Add); + let cycle = s.prefs.appearance.cycle_mask_colors; let uid = s.edit("New Mask", None, |proj, st| { let mut next = proj.next_id; let l = layer_mut(proj, cid, lid)?; let masks = l.props.sub_mut("masks").ok_or_else(|| bad("mask.new", "this layer can't have masks"))?; let k = masks.children.len(); - let g = build::mask(&mut Ids(&mut next), &format!("Mask {}", k + 1), path, mode, build::MASK_COLORS[k % build::MASK_COLORS.len()]); + let g = build::mask(&mut Ids(&mut next), &format!("Mask {}", k + 1), path, mode, crate::prefs::mask_color(cycle, k)); let uid = g.uid; masks.children.push(g.into()); proj.next_id = next; @@ -109,12 +180,12 @@ fn add_vertex(s: &mut Session, p: &Value) -> Result { let tout = pt(p.get("out")).unwrap_or([0.0; 2]); let at = p.get("index").and_then(Value::as_u64).map(|i| i as usize); let (_, lid) = layer_p(s, p, "mask.addVertex")?; - let (uid, i) = edit_path(s, p, "mask.addVertex", |sp| { + let (uid, i) = edit_path_counted(s, p, "mask.addVertex", |sp| { let i = at.unwrap_or(sp.vertices.len()).min(sp.vertices.len()); sp.vertices.insert(i, point); sp.in_tangents.insert(i, tin); sp.out_tangents.insert(i, tout); - Ok(i) + Ok((i, Some(CountOp::Insert(i)))) })?; s.state.selected_vertices = vec![VertexRef { layer: lid, mask: uid, index: i }]; Ok(json!(i)) @@ -196,9 +267,16 @@ fn has_vertices(s: &Session) -> std::result::Result<(), String> { /// Apply `f` to the selected vertices' paths (each mask once). fn edit_vertices(s: &mut Session, p: &Value, label: &str, f: impl Fn(&mut ShapePath, &[usize])) -> Result { + edit_vertices_counted(s, p, label, false, f) +} + +/// [`edit_vertices`]; `removes`: `f` deletes the given vertices (repeated on the other keyframes +/// when Preserve Constant Vertex Count is on). +fn edit_vertices_counted(s: &mut Session, p: &Value, label: &str, removes: bool, f: impl Fn(&mut ShapePath, &[usize])) -> Result { let cid = s.active_comp_id().ok_or(EngineError::NoComp)?; let groups = vertex_groups(s, p)?; let t = s.time(); + let preserve = removes && s.prefs.general.preserve_constant_vertex_count; s.edit(label, merge_p(p), |proj, _| { let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; for ((l, m), idx) in &groups { @@ -213,6 +291,9 @@ fn edit_vertices(s: &mut Session, p: &Value, label: &str, f: impl Fn(&mut ShapeP super::paths::roto_smooth(&mut sp); } pr.set_value_at(lt, KV::Path(sp)); + if preserve { + sync_keys(pr, lt, &CountOp::Remove(idx.clone())); + } } Ok(()) })?; @@ -232,7 +313,7 @@ fn move_vertices(s: &mut Session, p: &Value) -> Result { } fn delete_vertices(s: &mut Session, p: &Value) -> Result { - let n = edit_vertices(s, p, "Delete Mask Vertices", |sp, idx| { + let n = edit_vertices_counted(s, p, "Delete Mask Vertices", true, |sp, idx| { let mut idx = idx.to_vec(); idx.sort_unstable(); idx.dedup(); @@ -333,7 +414,10 @@ fn insert_vertex(s: &mut Session, p: &Value) -> Result { p.get("segment").and_then(Value::as_u64).ok_or_else(|| bad("mask.insertVertex", "missing `segment` (index of the segment's first vertex)"))? as usize; let t = super::f_p(p, "t").unwrap_or(0.5).clamp(0.0, 1.0); let (_, lid) = layer_p(s, p, "mask.insertVertex")?; - let (uid, i) = edit_path(s, p, "Add Vertex", |sp| split_segment(sp, seg, t).ok_or_else(|| bad("mask.insertVertex", "no such segment")))?; + let (uid, i) = edit_path_counted(s, p, "Add Vertex", |sp| { + let i = split_segment(sp, seg, t).ok_or_else(|| bad("mask.insertVertex", "no such segment"))?; + Ok((i, Some(CountOp::Insert(i)))) + })?; s.state.selected_vertices = vec![VertexRef { layer: lid, mask: uid, index: i }]; Ok(json!(i)) } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 2e2d2782..06d9f14a 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod app_more; mod camera_cmds; mod comp; mod comp_more; @@ -15,6 +16,7 @@ mod file_more; mod frontend; mod help; mod key_transform; +mod keys_more; mod layer; mod layer_menu; mod layer_time; @@ -46,6 +48,13 @@ mod view; pub mod viewer_cmds; mod warp_cmds; #[cfg(test)] +pub(crate) use app_more::report_text as report_text_for_tests; +pub(crate) use app_more::view_command; +#[cfg(test)] +pub(crate) use file::missing_frames as missing_frames_for_tests; +#[cfg(test)] +pub(crate) use keys_more::amplitudes as amplitudes_for_tests; +#[cfg(test)] pub(crate) use mask::split_segment as split_segment_for_tests; use std::sync::OnceLock; @@ -144,6 +153,8 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(puppet::specs()); v.extend(liquify::specs()); v.extend(settings::specs()); + v.extend(app_more::specs()); + v.extend(keys_more::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/prop.rs b/crates/engine/src/commands/prop.rs index 05dc37fa..d10fa232 100644 --- a/crates/engine/src/commands/prop.rs +++ b/crates/engine/src/commands/prop.rs @@ -184,7 +184,7 @@ fn set_expr(s: &mut Session, p: &Value) -> Result { enabled = Some(false); } // Alt-click on a stopwatch starts with the property's own reference (`transform.opacity`). - let def = s.project.comp(cid).and_then(|c| c.layer(lid).and_then(|l| super::link::reference(c, l, l, uid))).unwrap_or_else(|| "value".into()); + let def = s.project.comp(cid).and_then(|c| c.layer(lid).and_then(|l| super::link::reference(c, l, l, uid, true))).unwrap_or_else(|| "value".into()); let r = with_prop(s, "Expression", merge_p(p), cid, lid, uid, |pr, _| { match (&text, enabled) { (Some(t), _) if t.trim().is_empty() => pr.expr = None, diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index be6cebfb..15b5a3a2 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -495,8 +495,15 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { let mut item = RenderQueueItem::new(0, cid); item.output = OutputModule::for_format(format); item.output.output = path.clone(); - let job = - crate::ExportJob { project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), accel: s.accel.as_deref(), item: &item, path: &path }; + let job = crate::ExportJob { + project: &s.project, + footage: s.footage.as_ref(), + expr: s.expr.as_deref(), + accel: s.accel.as_deref(), + item: &item, + path: &path, + nested_switches: s.prefs.general.switches_affect_nested_comps, + }; let r = exporter.export(&job, &mut |_, _| true).map_err(EngineError::Other)?; s.toast(format!("Saved preview to {}", r.path)); Ok(serde_json::to_value(&r).unwrap_or_default()) diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 27cbc5d4..014360ff 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -23,7 +23,6 @@ pub fn specs() -> Vec { stub!("file.setProxy", "File...", ["File", "Set Proxy"], "{path}"), stub!("file.setProxyNone", "None", ["File", "Set Proxy"], "{}"), stub!("file.interpretProxy", "Proxy...", ["File", "Interpret Footage"], "{}"), - stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). @@ -47,8 +46,6 @@ pub fn specs() -> Vec { stub!("view.displayColorManagement", "Use Display Color Management", ["View"], "{}"), stub!("view.simulateOutput", "Simulate Output", [], "{profile}"), // Keyframes / text / tracking. - stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), - stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a6dbe2c4..45f5c0f6 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -14,7 +14,9 @@ pub mod commands; pub mod config; pub mod demo; pub mod links; +pub mod logging; pub mod mask_track; +pub mod media_cache; pub mod menus; pub mod prefs; pub mod psd_import; @@ -22,10 +24,12 @@ pub mod render_queue; pub mod roto; mod session_settings; pub mod shortcuts; +pub mod sysinfo; pub mod tracking; pub mod vector; pub mod viewer; pub mod warp; +pub mod xml_project; use std::sync::Arc; @@ -309,6 +313,11 @@ pub struct Session { /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. pub script: Option, + /// The last physical-memory reading (Settings ▸ Memory & CPU budgets); `None` = unknown + /// (headless sessions don't query it, see [`Session::memory_tick`]). + pub sys_memory: Option, + /// Switches Affect Nested Comps as last applied (a change empties the layer cache). + pub applied_nested_switches: Option, } /// A script to run (see [`Session::script`]). @@ -363,6 +372,8 @@ impl Default for Session { autosave: autosave::AutoSaveState::default(), snapshot: None, script: None, + sys_memory: None, + applied_nested_switches: None, } } } @@ -527,6 +538,47 @@ impl Session { None => t, }; self.state.times.insert(c, t); + if self.prefs.general.sync_time_related_items { + self.sync_related_times(c, t); + } + } + } + + /// Settings ▸ General ▸ Synchronize Time of All Related Items: moving the current time of + /// `comp` moves the current time of the comps nested in it (through their precomp layers' + /// timing) and of the comps it is nested in, so every related viewer shows the same moment. + pub fn sync_related_times(&mut self, comp: ItemId, t: Tick) { + let mut seen = std::collections::BTreeSet::from([comp]); + let mut todo = vec![(comp, t)]; + let mut found = vec![]; + while let Some((cid, ct)) = todo.pop() { + if cid != comp { + found.push((cid, ct)); + } + let Some(c) = self.project.comp(cid) else { continue }; + // Down: the comps this one nests. + for l in &c.layers { + if let effectcraft_project::LayerSource::Comp { item } = &l.source + && seen.insert(*item) + { + todo.push((*item, l.layer_time(ct))); + } + } + // Up: the comps nesting this one. + for iid in self.project.items.keys() { + if let Some(pc) = self.project.comp(*iid) + && let Some(l) = pc.layers.iter().find(|l| matches!(&l.source, effectcraft_project::LayerSource::Comp { item } if *item == cid)) + && seen.insert(*iid) + { + todo.push((*iid, l.comp_time(ct))); + } + } + } + for (cid, ct) in found { + if let Some(c) = self.project.comp(cid) { + let ct = c.frame_rate.snap_nearest(ct.clamp(Tick::ZERO, (c.duration - c.frame_duration()).max(Tick::ZERO))); + self.state.times.insert(cid, ct); + } } } @@ -568,6 +620,7 @@ impl Session { /// Render a comp frame with the session's footage + expression hosts. pub fn render(&self, comp: ItemId, t: Tick, opts: RenderOpts) -> Image { + let opts = RenderOpts { nested_switches: self.prefs.general.switches_affect_nested_comps, draft_shadows: self.prefs.three_d.realtime_shadows, ..opts }; let mut r = Renderer::new(&self.project, self.footage.as_ref(), opts); r.expr = self.expr.as_deref(); r.cache = Some(&self.layer_cache); @@ -649,6 +702,8 @@ mod tests_fidelity; #[cfg(test)] mod tests_lottie; #[cfg(test)] +mod tests_m145; +#[cfg(test)] mod tests_markers; #[cfg(test)] mod tests_mask_warp; @@ -686,6 +741,68 @@ pub fn text_presets() -> Vec<(String, String)> { pub fn text_families() -> Vec { effectcraft_text::families().into_iter().map(|(f, _)| f).collect() } + +/// One row of the Character panel's font menu. +#[derive(Clone, Debug, PartialEq)] +pub struct FontRow { + /// The family to apply (empty for the separator after the recent fonts). + pub family: String, + /// What the menu shows: the English family name, or the font's own-language name when + /// Settings ▸ Type ▸ Show Font Names in English is off. + pub display: String, + pub recent: bool, +} + +/// Outline polylines of "Sample" set in `family` at `size` px (baseline at y = 0, y down), for +/// the font menu's preview (Settings ▸ Type ▸ Show Font Preview). +pub fn font_preview(family: &str, size: f64) -> Vec> { + use kurbo::PathEl; + let st = effectcraft_keyframe::text_doc::CharStyle { font: family.to_string(), size, ..Default::default() }; + let mut out = vec![]; + let mut x = 0.0; + for ch in "Sample".chars() { + let (path, adv) = effectcraft_text::char_glyph_style(&st, ch); + let mut cur: Vec<[f32; 2]> = vec![]; + let pt = |p: kurbo::Point| [(p.x + x) as f32, p.y as f32]; + kurbo::flatten(&path, 0.2, |el| match el { + PathEl::MoveTo(p) => { + if cur.len() > 1 { + out.push(std::mem::take(&mut cur)); + } + cur = vec![pt(p)]; + } + PathEl::LineTo(p) => cur.push(pt(p)), + PathEl::ClosePath => { + if let Some(f) = cur.first().copied() { + cur.push(f); + } + if cur.len() > 1 { + out.push(std::mem::take(&mut cur)); + } + } + _ => {} + }); + if cur.len() > 1 { + out.push(cur); + } + x += adv; + } + out +} + +/// The Character panel's font menu: the recent fonts (Settings ▸ Type ▸ Number of Recent Fonts +/// to Display), a separator, then every family. +pub fn font_menu(prefs: &prefs::Prefs) -> Vec { + let all = text_families(); + let native = if prefs.type_.font_names_in_english { Default::default() } else { effectcraft_text::fonts::native_families() }; + let row = |f: &String, recent: bool| FontRow { family: f.clone(), display: native.get(f).cloned().unwrap_or_else(|| f.clone()), recent }; + let mut out: Vec = prefs.recent_fonts_shown().iter().filter(|f| all.contains(f)).map(|f| row(f, true)).collect(); + if !out.is_empty() { + out.push(FontRow { family: String::new(), display: "-".into(), recent: false }); + } + out.extend(all.iter().map(|f| row(f, false))); + out +} #[cfg(test)] mod tests_paint; #[cfg(test)] diff --git a/crates/engine/src/logging.rs b/crates/engine/src/logging.rs new file mode 100644 index 00000000..23ef28f0 --- /dev/null +++ b/crates/engine/src/logging.rs @@ -0,0 +1,99 @@ +//! Help ▸ Enable Logging / Reveal Logging File: a `log` logger that, while enabled, appends +//! every record to `Logs/EffectCraft Log.txt` in the settings folder (or the temporary folder), +//! and always keeps the last few hundred warnings and errors in memory for the System +//! Compatibility Report. +//! +//! Frontends call [`install`] once at startup; [`set_enabled`] (the `help.enableLogging` +//! command) turns file logging on and off. + +use std::collections::VecDeque; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +static ENABLED: AtomicBool = AtomicBool::new(false); +static STATE: Mutex = Mutex::new(State { file: None, recent: VecDeque::new() }); + +struct State { + file: Option, + recent: VecDeque, +} + +struct Logger; + +const RECENT: usize = 300; + +impl log::Log for Logger { + fn enabled(&self, m: &log::Metadata) -> bool { + m.level() <= log::Level::Warn || (ENABLED.load(Ordering::Relaxed) && m.target().starts_with("effectcraft")) + } + fn log(&self, r: &log::Record) { + if !self.enabled(r.metadata()) { + return; + } + let line = format!("{} {:<5} {}: {}", unix_secs(), r.level(), r.target(), r.args()); + let Ok(mut st) = STATE.lock() else { return }; + if r.level() <= log::Level::Warn { + st.recent.push_back(line.clone()); + if st.recent.len() > RECENT { + st.recent.pop_front(); + } + } + if ENABLED.load(Ordering::Relaxed) + && let Some(f) = &st.file + && let Ok(mut out) = std::fs::OpenOptions::new().create(true).append(true).open(f) + { + let _ = writeln!(out, "{line}"); + } + } + fn flush(&self) {} +} + +fn unix_secs() -> u64 { + web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0) +} + +/// Install the logger (once; later calls and other installed loggers are left alone). +pub fn install() { + static LOGGER: Logger = Logger; + if log::set_logger(&LOGGER).is_ok() { + log::set_max_level(log::LevelFilter::Info); + } +} + +/// The log file inside `config_dir` (the settings folder), or the temporary folder. +pub fn log_path(config_dir: Option<&Path>) -> PathBuf { + config_dir.map(Path::to_path_buf).unwrap_or_else(std::env::temp_dir).join("Logs").join("EffectCraft Log.txt") +} + +/// Turn file logging on (writing to `file`) or off. +pub fn set_enabled(on: bool, file: PathBuf) -> std::io::Result<()> { + install(); + if on { + if let Some(d) = file.parent() { + std::fs::create_dir_all(d)?; + } + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&file)?; + writeln!(f, "{} INFO effectcraft: logging enabled ({} {})", unix_secs(), std::env::consts::OS, env!("CARGO_PKG_VERSION"))?; + } + if let Ok(mut st) = STATE.lock() { + st.file = Some(file); + } + ENABLED.store(on, Ordering::Relaxed); + Ok(()) +} + +pub fn is_enabled() -> bool { + ENABLED.load(Ordering::Relaxed) +} + +/// The current log file (set once logging was enabled). +pub fn current_file() -> Option { + STATE.lock().ok().and_then(|s| s.file.clone()) +} + +/// Recent warnings and errors, oldest first. +pub fn recent() -> Vec { + STATE.lock().map(|s| s.recent.iter().cloned().collect()).unwrap_or_default() +} diff --git a/crates/engine/src/media_cache.rs b/crates/engine/src/media_cache.rs new file mode 100644 index 00000000..1a9b932a --- /dev/null +++ b/crates/engine/src/media_cache.rs @@ -0,0 +1,65 @@ +//! The media cache (Settings ▸ Disk ▸ Database and Cache Folder): audio waveform summaries kept +//! on disk so a footage file is summarised once, not in every session. +//! +//! A summary is stored as `Peaks/_.ecpk` (raw little-endian `f32` quadruples +//! `[min L, max L, min R, max R]` per bin), named by a hash of the footage path, its size and +//! modification time, so an edited file is summarised again. + +use std::path::{Path, PathBuf}; + +/// The peak file of footage `path` at `bins` bins per second in the media cache `folder` +/// (`None` when the file can't be inspected). +pub fn peaks_path(folder: &Path, path: &str, bins: u32) -> Option { + use std::hash::{Hash, Hasher}; + let meta = std::fs::metadata(path).ok()?; + let mut h = std::collections::hash_map::DefaultHasher::new(); + (path, meta.len(), meta.modified().ok()).hash(&mut h); + Some(folder.join("Peaks").join(format!("{:016x}_{bins}.ecpk", h.finish()))) +} + +/// Read a stored summary. +pub fn load_peaks(file: &Path) -> Option> { + let b = std::fs::read(file).ok()?; + if b.len() % 16 != 0 { + return None; + } + let f = |c: &[u8]| f32::from_le_bytes([c[0], c[1], c[2], c[3]]); + Some(b.chunks_exact(16).map(|c| [f(&c[0..4]), f(&c[4..8]), f(&c[8..12]), f(&c[12..16])]).collect()) +} + +/// Store a summary (atomically: written beside, then renamed). +pub fn store_peaks(file: &Path, peaks: &[[f32; 4]]) -> std::io::Result<()> { + if let Some(d) = file.parent() { + std::fs::create_dir_all(d)?; + } + let mut b = Vec::with_capacity(peaks.len() * 16); + for p in peaks { + for v in p { + b.extend_from_slice(&v.to_le_bytes()); + } + } + crate::config::atomic_write(file, &b) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn peaks_round_trip_and_follow_the_file() { + let d = std::env::temp_dir().join(format!("ec-media-cache-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let src = d.join("a.wav"); + std::fs::write(&src, b"one").unwrap(); + let p = peaks_path(&d, src.to_str().unwrap(), 200).unwrap(); + assert!(p.starts_with(d.join("Peaks"))); + let peaks = vec![[-0.5, 0.5, -0.25, 0.25], [0.0, 0.1, -0.1, 0.0]]; + store_peaks(&p, &peaks).unwrap(); + assert_eq!(load_peaks(&p).unwrap(), peaks); + // A changed file gets another name. + std::fs::write(&src, b"longer").unwrap(); + assert_ne!(peaks_path(&d, src.to_str().unwrap(), 200).unwrap(), p); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 0191c6ea..e1ef4e8f 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -12,6 +12,8 @@ //! Next Blending Mode | layer.setBlendMode {"step":1} | Shift+= //! --- separator //! @effects the effect categories (Effect menu) +//! @dynamic:history entries computed from the session when the menu +//! opens ([`dynamic`]: recent files, undo history…) //! [mac] Quit EffectCraft | app.quit only on macOS ([!mac] = everywhere else) //! ``` //! @@ -34,11 +36,120 @@ use crate::commands::{self, CommandSpec}; pub enum MenuNode { Item(MenuEntry), Separator, - Submenu { label: String, children: Vec }, + Submenu { + label: String, + children: Vec, + }, + /// Entries computed when the menu opens (see [`dynamic`]). + Dynamic { + name: String, + }, +} + +/// Frontend state dynamic menus depend on. +#[derive(Clone, Copy, Debug, Default)] +pub struct DynCtx<'a> { + /// The current workspace (Window ▸ Assign Shortcut to "…" Workspace). + pub workspace: Option<&'a str>, +} + +fn file_name(p: &str) -> String { + std::path::Path::new(p).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| p.to_string()) +} + +fn dyn_entry(label: String, command: &str, params: Value) -> MenuEntry { + MenuEntry { label, command: command.into(), params, shortcut: None } +} + +/// The bindable holding shortcut `keys` in the active preset (its label). +fn holder_of(s: &Session, keys: &str) -> Option { + let t = s.shortcuts(); + t.bindables.iter().find(|b| t.keys.get(&b.key).is_some_and(|k| k.iter().any(|k| k == keys))).map(|b| b.label.clone()) +} + +/// The entries of a `@dynamic:` node, and the disabled placeholder shown when there are +/// none: `recentProjects`, `recentFootage`, `recentPresets`, `history` (undo steps, newest +/// first), `view3dShortcuts`, `workspaceShortcuts` and `openViewers` (open compositions other +/// than the active one). +pub fn dynamic(s: &Session, name: &str, cx: &DynCtx) -> (Vec, Option<&'static str>) { + use serde_json::json; + let indexed = |list: &[String], cmd: &str, stem: bool| -> Vec { + list.iter() + .enumerate() + .map(|(i, p)| { + let n = + if stem { std::path::Path::new(p).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| p.clone()) } else { file_name(p) }; + dyn_entry(n, cmd, json!({"index": i})) + }) + .collect() + }; + match name { + "recentProjects" => (indexed(&s.prefs.recent_projects, "file.openRecent", false), Some("No Recent Projects")), + "recentFootage" => (indexed(&s.prefs.recent_footage, "file.importRecent", false), Some("No Recent Footage")), + "recentPresets" => (indexed(&s.prefs.recent_presets, "anim.applyRecentPreset", true), Some("No Recent Presets")), + "history" => ( + s.history.undo.iter().rev().enumerate().map(|(i, (l, _))| dyn_entry(format!("Undo {l}"), "edit.history", json!({"steps": i + 1}))).collect(), + Some("No History"), + ), + "view3dShortcuts" => ( + ["F10", "F11", "F12"] + .iter() + .map(|k| { + let label = match holder_of(s, k) { + Some(h) => format!("{k} (Replace “{h}”)"), + None => k.to_string(), + }; + dyn_entry(label, "view.assign3dShortcut", json!({"slot": k})) + }) + .collect(), + None, + ), + "workspaceShortcuts" => ( + ["Shift+F10", "Shift+F11", "Shift+F12"] + .iter() + .map(|k| { + let label = match holder_of(s, k) { + Some(h) => format!("{k} (Replace “{h}”)"), + None => k.to_string(), + }; + let mut p = json!({"slot": k}); + if let Some(w) = cx.workspace { + p["workspace"] = json!(w); + } + dyn_entry(label, "window.assignWorkspaceShortcut", p) + }) + .collect(), + None, + ), + "openViewers" => ( + s.state + .open_comps + .iter() + .filter(|c| Some(**c) != s.state.active_comp) + .filter_map(|c| s.project.item(*c).map(|i| dyn_entry(format!("Composition: {}", i.name), "comp.open", json!({"comp": c.0})))) + .collect(), + None, + ), + _ => (vec![], None), + } +} + +/// Display label of a submenu (the "Assign Shortcut to …" submenus name the current view or +/// workspace, like After Effects). +pub fn submenu_label(s: &Session, label: &str, cx: &DynCtx) -> String { + match label { + "Assign Shortcut to 3D View" => { + let v = s.active_comp_id().and_then(|c| s.state.views3d.get(&c)).map(|v| v.current).unwrap_or_default(); + let name = crate::commands::find(crate::commands::view_command(v)).map(|c| c.label).unwrap_or("Active Camera"); + format!("Assign Shortcut to “{name}”") + } + "Assign Shortcut to Workspace" => format!("Assign Shortcut to “{}” Workspace", cx.workspace.unwrap_or("Default")), + _ => label.to_string(), + } } /// A clickable menu entry. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, PartialEq, Serialize)] pub struct MenuEntry { pub label: String, pub command: String, @@ -80,7 +191,7 @@ pub fn entries() -> Vec<(Vec, &'static MenuEntry)> { for n in nodes { match n { MenuNode::Item(e) => out.push((path.clone(), e)), - MenuNode::Separator => {} + MenuNode::Separator | MenuNode::Dynamic { .. } => {} MenuNode::Submenu { label, children } => { path.push(label.clone()); rec(children, path, out); @@ -96,7 +207,23 @@ pub fn entries() -> Vec<(Vec, &'static MenuEntry)> { /// Display label of an entry: label names come from Settings ▸ Labels. pub fn entry_label(s: &Session, e: &MenuEntry) -> String { + let comp_name = || s.active_comp_id().and_then(|c| s.project.item(c)).map(|i| i.name.clone()).unwrap_or_else(|| "(none)".into()); + let layer_name = + || s.active_comp().and_then(|c| s.state.selected_layers.first().and_then(|l| c.layer(*l))).map(|l| l.name.clone()).unwrap_or_else(|| "(none)".into()); match e.command.as_str() { + // Window menu: the panels' current viewer instances. + "window.panel" => match e.params.get("panel").and_then(Value::as_str) { + Some("composition") => format!("Composition: {}", comp_name()), + Some("timeline") => format!("Timeline: {}", comp_name()), + Some("flowchart") => format!("Flowchart: {}", comp_name()), + Some("layer") => format!("Layer: {}", layer_name()), + Some("effectControls") if e.label == "Effect Controls" => format!("Effect Controls: {}", layer_name()), + _ => e.label.clone(), + }, + "window.unavailablePanel" if e.params.get("panel").and_then(Value::as_str) == Some("footage") => { + let f = s.state.project_selection.iter().filter_map(|i| s.project.item(*i)).find(|i| matches!(i.kind, effectcraft_project::ItemKind::Footage(_))); + format!("Footage: {}", f.map(|i| i.name.clone()).unwrap_or_else(|| "(none)".into())) + } "edit.label" => match e.params.get("label").and_then(Value::as_str).and_then(effectcraft_color::Label::from_name) { Some(l) if l != effectcraft_color::Label::None => s.prefs.label_name(l), _ => e.label.clone(), @@ -108,6 +235,9 @@ pub fn entry_label(s: &Session, e: &MenuEntry) -> String { /// Check-mark state of an entry (`None` = not a toggle / radio entry). Frontend-only toggles /// (rulers, grid…) are answered by the frontend. pub fn checked(s: &Session, command: &str, params: &Value) -> Option { + if command == "help.enableLogging" { + return Some(crate::logging::is_enabled()); + } let comp = s.active_comp()?; let layer = s.state.selected_layers.first().and_then(|l| comp.layer(*l)); let pstr = |k: &str| params.get(k).and_then(Value::as_str); @@ -236,6 +366,10 @@ pub fn parse(text: &str, mac: bool) -> Result, String> { out.extend(effect_categories()); continue; } + if let Some(name) = l.body.strip_prefix("@dynamic:") { + out.push(MenuNode::Dynamic { name: name.trim().to_string() }); + continue; + } let mut parts = l.body.split(" | "); let label = parts.next().unwrap_or_default().trim().to_string(); match parts.next() { @@ -321,6 +455,8 @@ File New Folder | project.newFolder Open Project... | file.open Open Recent + @dynamic:recentProjects + --- Clear Recent Projects | file.clearRecent --- Close | file.close @@ -329,6 +465,7 @@ File Save As Save As... | file.saveAs Save a Copy... | file.saveCopy + Save a Copy As XML... | file.saveCopyAsXml Increment and Save | file.incrementAndSave Revert | file.revert --- @@ -338,6 +475,10 @@ File Placeholder... | file.importPlaceholder Solid... | file.importSolid Lottie... | file.importLottie + Import Recent Footage + @dynamic:recentFootage + --- + Clear Recent Footage | file.clearRecentFootage Export Add to Render Queue | renderQueue.add Lottie JSON... | file.exportLottie @@ -374,6 +515,7 @@ File File... | file.replaceFootage Placeholder... | file.replaceWithPlaceholder Solid... | file.replaceWithSolid + With Layered Comp | file.replaceWithLayeredComp Reload Footage | file.reloadFootage Reveal in Finder | file.revealInFinder --- @@ -383,6 +525,8 @@ File Edit Undo | edit.undo Redo | edit.redo + History + @dynamic:history --- Quick Apply... | app.commandPalette --- @@ -494,6 +638,10 @@ Composition --- Composition Flowchart | comp.flowchart Composition Mini-Flowchart | comp.miniFlowchart + --- + VR + Create VR Environment... | comp.vr.createEnvironment + Extract Cubemap... | comp.vr.extractCubemap Layer New Text | layer.newText @@ -744,6 +892,10 @@ Effect Animation Save Animation Preset... | anim.savePreset Apply Animation Preset... | anim.applyPreset + Recent Animation Presets + @dynamic:recentPresets + --- + Clear Recent Presets | anim.clearRecentPresets Browse Presets... | anim.browsePresets Text Animation Presets Typewriter | layer.applyTextPreset {"preset":"typewriter"} @@ -902,6 +1054,8 @@ View Custom View 3 | view.3d.custom3 --- Reset 3D View | view.reset3DView + Assign Shortcut to 3D View + @dynamic:view3dShortcuts Switch to Last 3D View | view.3d.last | Escape Look at Selected Layers | view.lookAtSelected Look at All Layers | view.lookAtAll @@ -931,6 +1085,8 @@ Window Save Changes to this Workspace | window.saveWorkspace Save as New Workspace... | window.saveWorkspaceAs Edit Workspaces... | window.editWorkspaces + Assign Shortcut to Workspace + @dynamic:workspaceShortcuts --- Align | window.panel {"panel":"align"} Audio | window.panel {"panel":"audio"} | Cmd+4 @@ -963,6 +1119,7 @@ Window Project | window.panel {"panel":"project"} | Cmd+0 Render Queue | window.panel {"panel":"renderQueue"} | Cmd+Alt+0 Timeline | window.panel {"panel":"timeline"} + @dynamic:openViewers Help EffectCraft Help... | help.docs {"page":"help"} | F1 Scripting Help... | help.docs {"page":"scripting"} @@ -971,6 +1128,10 @@ Help Animation Presets... | anim.browsePresets Keyboard Shortcuts... | app.keyboardShortcuts --- + System Compatibility Report... | help.systemReport + Enable Logging | help.enableLogging + Reveal Logging File | help.revealLogFile + --- Join the ArtCraft Discord... | help.discord Provide Feedback... | help.reportIssue --- diff --git a/crates/engine/src/menus/tests.rs b/crates/engine/src/menus/tests.rs index ee8790c7..6fdbd2a7 100644 --- a/crates/engine/src/menus/tests.rs +++ b/crates/engine/src/menus/tests.rs @@ -6,7 +6,7 @@ fn all_entries(nodes: &[MenuNode], path: &mut Vec, out: &mut Vec<(Vec out.push((path.clone(), e.clone())), - MenuNode::Separator => {} + MenuNode::Separator | MenuNode::Dynamic { .. } => {} MenuNode::Submenu { label, children } => { path.push(label.clone()); all_entries(children, path, out); diff --git a/crates/engine/src/prefs.rs b/crates/engine/src/prefs.rs index 5a2afab8..edefe370 100644 --- a/crates/engine/src/prefs.rs +++ b/crates/engine/src/prefs.rs @@ -13,6 +13,7 @@ use std::collections::BTreeMap; use effectcraft_color::Label; +use effectcraft_time::Tick; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; @@ -271,6 +272,13 @@ pub struct Prefs { pub scripting: Scripting, /// File ▸ Open Recent, newest first. pub recent_projects: Vec, + /// File ▸ Import Recent Footage, newest first. + pub recent_footage: Vec, + /// Animation ▸ Recent Animation Presets (`.ecpreset` paths or built-in preset ids), newest + /// first. + pub recent_presets: Vec, + /// Character panel: recently used font families, newest first. + pub recent_fonts: Vec, /// Top-level keys this version doesn't know (kept for newer versions). #[serde(flatten)] pub extra: Extra, @@ -299,6 +307,9 @@ impl Default for Prefs { three_d: ThreeD::default(), scripting: Scripting::default(), recent_projects: vec![], + recent_footage: vec![], + recent_presets: vec![], + recent_fonts: vec![], extra: Extra::new(), } } @@ -428,7 +439,13 @@ impl Prefs { pub fn reset(&mut self, page: Option<&str>) -> Result<(), String> { let d = Prefs::default(); let Some(page) = page else { - *self = Prefs { recent_projects: std::mem::take(&mut self.recent_projects), ..d }; + *self = Prefs { + recent_projects: std::mem::take(&mut self.recent_projects), + recent_footage: std::mem::take(&mut self.recent_footage), + recent_presets: std::mem::take(&mut self.recent_presets), + recent_fonts: std::mem::take(&mut self.recent_fonts), + ..d + }; return Ok(()); }; let keys: Vec<&str> = match page { @@ -488,6 +505,174 @@ impl Prefs { _ => 0.125, } } + + /// The Memory & CPU cache budgets in effect: the configured layer / footage / preview + /// budgets, scaled down so together they leave **RAM Reserved for Other Applications** free, + /// and halved while the system is low on memory when **Reduce Cache Size When System Is Low + /// on Memory** is on. `mem` = the system's memory (`None` = unknown: configured budgets). + pub fn cache_budgets(&self, mem: Option) -> CacheBudgets { + let (mut l, mut m, mut p) = (self.layer_cache_bytes() as u64, self.media_cache_bytes() as u64, self.preview_cache_bytes() as u64); + let floor = 64u64 << 20; + let mut capped = false; + let mut reduced = false; + if let Some(mem) = mem.filter(|m| m.total > 0) { + let reserved = (self.memory.ram_reserved_gb as u64) << 30; + let cap = mem.total.saturating_sub(reserved).max(3 * floor); + let sum = l + m + p; + if sum > cap { + let k = cap as f64 / sum as f64; + (l, m, p) = ((l as f64 * k) as u64, (m as f64 * k) as u64, (p as f64 * k) as u64); + capped = true; + } + if self.memory.reduce_cache_when_low && mem.is_low() { + (l, m, p) = (l / 2, m / 2, p / 2); + reduced = true; + } + } + CacheBudgets { layer: l.max(floor) as usize, media: m.max(floor) as usize, preview: p.max(floor) as usize, capped, reduced } + } + + /// Composition ▸ Motion Path: the layer-time span of a motion path to draw around `now` + /// (layer time), for keyframes at `keys` (sorted). `None` = No Motion Path. + pub fn motion_path_span(&self, keys: &[Tick], now: Tick) -> Option<(Tick, Tick)> { + let (first, last) = (*keys.first()?, *keys.last()?); + let c = &self.composition; + match c.motion_path.as_str() { + "none" => None, + "seconds" => { + let half = Tick::from_seconds_f64(c.motion_path_seconds.max(0.0) / 2.0); + let (a, b) = ((now - half).max(first), (now + half).min(last)); + (a < b).then_some((a, b)) + } + "keyframes" => { + let n = (c.motion_path_keyframes.max(1) as usize).min(keys.len()); + if n < 2 { + return None; + } + // The n keyframes centred on the current time. + let at = keys.partition_point(|k| *k < now); + let start = at.saturating_sub(n / 2).min(keys.len() - n); + Some((keys[start], keys[start + n - 1])) + } + _ => Some((first, last)), + } + } + + /// Previews ▸ Viewer Zoom Quality: smooth (bilinear) scaling of the viewer image, or nearest + /// neighbour ("Faster"). + pub fn viewer_zoom_smooth(&self) -> bool { + self.previews.zoom_quality != "faster" + } + + /// Import ▸ Interpret Unlabeled Alpha As, for footage whose file has an alpha channel that + /// doesn't state how it is stored. `None` = Ask User (keep straight and open Interpret + /// Footage); `guess` takes premultiplied for movies (codecs with alpha are usually written + /// premultiplied) and straight for stills. + pub fn unlabeled_alpha(&self, movie: bool) -> Option { + use effectcraft_project::AlphaMode::*; + Some(match self.import.unlabeled_alpha.as_str() { + "ignore" => Ignore, + "straight" => Straight, + "premultiplied" => Premultiplied, + "guess" if movie => Premultiplied, + "guess" => Straight, + _ => return None, + }) + } + + /// Import ▸ Default Drag Import As, as `file.import`'s `importAs`. + pub fn drag_import_as(&self) -> &'static str { + match self.import.drag_import_as.as_str() { + "comp" => "composition", + "compLayerSizes" => "compositionLayerSizes", + _ => "footage", + } + } + + /// Export ▸ Append Bit Depth to File Name: `Comp 1.png` → `Comp 1_16bpc.png` (sequence frame + /// runs and folders are kept). + pub fn output_name(&self, path: &str, bits: u32) -> String { + if !self.export.append_bits_to_name { + return path.to_string(); + } + let p = std::path::Path::new(path); + let (Some(stem), dir) = (p.file_stem().map(|s| s.to_string_lossy().to_string()), p.parent()) else { return path.to_string() }; + let tag = format!("_{bits}bpc"); + if stem.ends_with(&tag) { + return path.to_string(); + } + let name = match p.extension() { + Some(e) => format!("{stem}{tag}.{}", e.to_string_lossy()), + None => format!("{stem}{tag}"), + }; + match dir.filter(|d| !d.as_os_str().is_empty()) { + Some(d) => d.join(name).to_string_lossy().to_string(), + None => name, + } + } + + /// Export ▸ Segment Sequences / Segment Movie Files: how many frames go in each segment of an + /// output of `frames` frames (`None` = one file / folder). Sequences split every Files per + /// Segment frames; movies every Segment Size MB at the output's data rate + /// (`bytes_per_frame`, estimated from the format's bitrate). + pub fn segment_frames(&self, frames: u64, sequence: bool, bytes_per_frame: f64) -> Option { + let e = &self.export; + let per = if sequence { + e.segment_sequences.then_some(e.segment_sequence_files.max(1) as u64)? + } else { + if !e.segment_movies || bytes_per_frame <= 0.0 { + return None; + } + (((e.segment_movie_mb.max(1) as f64) * 1_048_576.0 / bytes_per_frame).floor() as u64).max(1) + }; + (per < frames).then_some(per) + } + + /// Remember a font in the Character panel's recent fonts (Type ▸ Number of Recent Fonts). + pub fn push_recent_font(&mut self, family: &str) { + push_mru(&mut self.recent_fonts, family, 30); + } + + /// The recent fonts the Character panel shows (at most Number of Recent Fonts to Display). + pub fn recent_fonts_shown(&self) -> &[String] { + let n = (self.type_.recent_fonts as usize).min(self.recent_fonts.len()); + &self.recent_fonts[..n] + } + + /// Remember imported footage for File ▸ Import Recent Footage. + pub fn push_recent_footage(&mut self, path: &str) { + push_mru(&mut self.recent_footage, path, self.general.recent_items as usize); + } + + /// Remember an applied animation preset for Animation ▸ Recent Animation Presets. + pub fn push_recent_preset(&mut self, path: &str) { + push_mru(&mut self.recent_presets, path, self.general.recent_items as usize); + } +} + +fn push_mru(list: &mut Vec, v: &str, max: usize) { + list.retain(|x| x != v); + list.insert(0, v.to_string()); + list.truncate(max.max(1)); +} + +/// Effective cache budgets in bytes (see [`Prefs::cache_budgets`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +pub struct CacheBudgets { + pub layer: usize, + pub media: usize, + pub preview: usize, + /// Scaled down to leave the reserved RAM free. + pub capped: bool, + /// Halved because the system is low on memory. + pub reduced: bool, +} + +/// Colour of the `k`-th mask of a layer: Appearance ▸ Cycle Mask Colors cycles through the mask +/// colours, otherwise every new mask takes the first one. +pub fn mask_color(cycle: bool, k: usize) -> [u8; 3] { + use effectcraft_project::build::MASK_COLORS; + if cycle { MASK_COLORS[k % MASK_COLORS.len()] } else { MASK_COLORS[0] } } fn label_index(l: Label) -> Option { @@ -657,13 +842,14 @@ pub fn pages() -> Vec { Section("Options"), s("general.showToolTips", "Show Tool Tips", B, true), s("general.createLayersAtCompStart", "Create Layers at Composition Start Time", B, true), - s("general.switchesAffectNestedComps", "Switches Affect Nested Comps", B, false), + s("general.switchesAffectNestedComps", "Switches Affect Nested Comps", B, true), s("general.defaultSpatialLinear", "Default Spatial Interpolation to Linear", B, true), - s("general.preserveConstantVertexCount", "Preserve Constant Vertex and Feather Point Count when Editing Masks", B, false), - s("general.syncTimeRelatedItems", "Synchronize Time of All Related Items", B, false), - s("general.expressionPickWhipCompact", "Expression Pick Whip Writes Compact English", B, false), - s("general.createSplitLayersAbove", "Create Split Layers Above Original Layer", B, false), + s("general.preserveConstantVertexCount", "Preserve Constant Vertex and Feather Point Count when Editing Masks", B, true), + s("general.syncTimeRelatedItems", "Synchronize Time of All Related Items", B, true), + s("general.expressionPickWhipCompact", "Expression Pick Whip Writes Compact English", B, true), + s("general.createSplitLayersAbove", "Create Split Layers Above Original Layer", B, true), s("general.useSystemColorPicker", "Use System Color Picker", B, false), + Note("EffectCraft always uses its own colour picker (the same on every platform and the web)."), ], }, Page { @@ -671,7 +857,7 @@ pub fn pages() -> Vec { title: "Startup & Repair", items: vec![ s("startup.showHomeOnLaunch", "Show Home Screen When Launching", B, true), - s("startup.showHomeOnOpenProject", "Show Home Screen When Opening a Project", B, false), + s("startup.showHomeOnOpenProject", "Show Home Screen When Opening a Project", B, true), s("startup.offerCrashRecovery", "Offer to Open the Latest Auto-Save After a Crash", B, true), Section("Repair"), Button { label: "Reset Settings", command: "prefs.reset", params: "{}" }, @@ -710,13 +896,14 @@ pub fn pages() -> Vec { ("No More Than (Seconds)", "seconds"), ("No More Than (Keyframes)", "keyframes"), ]), - false, + true, ), - s("composition.motionPathSeconds", "Seconds", Kind::Float(0.0, 3600.0, "s"), false), - s("composition.motionPathKeyframes", "Keyframes", Kind::Int(1, 10_000, ""), false), + s("composition.motionPathSeconds", "Seconds", Kind::Float(0.0, 3600.0, "s"), true), + s("composition.motionPathKeyframes", "Keyframes", Kind::Int(1, 10_000, ""), true), Section("Display"), s("composition.showRenderingProgress", "Show Rendering Progress in Info Panel and Flowchart", B, true), s("composition.hardwareAcceleratePanels", "Hardware Accelerate Composition, Layer and Footage Panels", B, false), + Note("Panels are always drawn by the GPU (wgpu)."), ], }, Page { @@ -724,10 +911,10 @@ pub fn pages() -> Vec { title: "Previews", items: vec![ s("previews.adaptiveResolutionLimit", "Adaptive Resolution Limit", Kind::Choice(ON_OFF_RES), true), - s("previews.showInternalWireframes", "Show Internal Wireframes", B, false), + s("previews.showInternalWireframes", "Show Internal Wireframes", B, true), s("previews.cacheFramesWhenIdle", "Cache Frames When Idle", B, true), s("previews.fastPreviews", "Fast Previews (Draft 3D, Faster Effects)", B, true), - s("previews.zoomQuality", "Viewer Zoom Quality", Kind::Choice(&[("Faster", "faster"), ("More Accurate", "moreAccurate")]), false), + s("previews.zoomQuality", "Viewer Zoom Quality", Kind::Choice(&[("Faster", "faster"), ("More Accurate", "moreAccurate")]), true), Button { label: "GPU Information...", command: "app.gpuInfo", params: "{}" }, ], }, @@ -739,9 +926,9 @@ pub fn pages() -> Vec { s("appearance.brightness", "Brightness", Kind::Slider(-1.0, 1.0), true), Section("Labels and Colors"), s("appearance.useLabelColorForHandles", "Use Label Color for Layer Handles and Paths", B, true), - s("appearance.useLabelColorForTabs", "Use Label Color for Related Tabs", B, false), - s("appearance.cycleMaskColors", "Cycle Mask Colors", B, false), - s("appearance.useGradients", "Use Gradients", B, false), + s("appearance.useLabelColorForTabs", "Use Label Color for Related Tabs", B, true), + s("appearance.cycleMaskColors", "Cycle Mask Colors", B, true), + s("appearance.useGradients", "Use Gradients", B, true), Section("Menu Bar"), s("appearance.inWindowMenuBarMac", "Use In-Window Menu Bar on macOS", B, true), ], @@ -754,13 +941,13 @@ pub fn pages() -> Vec { s("grids.gridColor", "Color", Kind::Color, true), s("grids.gridSpacing", "Gridline Every", Kind::Float(1.0, 10_000.0, "px"), true), s("grids.gridSubdivisions", "Subdivisions", Kind::Int(1, 100, ""), true), - s("grids.gridStyle", "Style", Kind::Choice(STYLES), false), + s("grids.gridStyle", "Style", Kind::Choice(STYLES), true), Section("Proportional Grid"), - s("grids.proportionalHorizontal", "Horizontal", Kind::Int(1, 100, ""), false), - s("grids.proportionalVertical", "Vertical", Kind::Int(1, 100, ""), false), + s("grids.proportionalHorizontal", "Horizontal", Kind::Int(1, 100, ""), true), + s("grids.proportionalVertical", "Vertical", Kind::Int(1, 100, ""), true), Section("Guides"), s("grids.guideColor", "Color", Kind::Color, true), - s("grids.guideStyle", "Style", Kind::Choice(STYLES), false), + s("grids.guideStyle", "Style", Kind::Choice(STYLES), true), Section("Safe Margins"), s("grids.actionSafe", "Action-safe", Kind::Float(0.0, 50.0, "%"), true), s("grids.titleSafe", "Title-safe", Kind::Float(0.0, 50.0, "%"), true), @@ -771,10 +958,10 @@ pub fn pages() -> Vec { id: "type", title: "Type", items: vec![ - s("type.textEngine", "Text Engine", Kind::Choice(&[("Latin", "latin"), ("South Asian and Middle Eastern", "southAsian")]), false), - s("type.fontPreview", "Show Font Preview", B, false), - s("type.recentFonts", "Number of Recent Fonts to Display", Kind::Int(0, 30, ""), false), - s("type.fontNamesInEnglish", "Show Font Names in English", B, false), + s("type.textEngine", "Text Engine", Kind::Choice(&[("Latin", "latin"), ("South Asian and Middle Eastern", "southAsian")]), true), + s("type.fontPreview", "Show Font Preview", B, true), + s("type.recentFonts", "Number of Recent Fonts to Display", Kind::Int(0, 30, ""), true), + s("type.fontNamesInEnglish", "Show Font Names in English", B, true), ], }, Page { @@ -784,7 +971,7 @@ pub fn pages() -> Vec { s("import.stillFootage", "Still Footage", Kind::Choice(&[("Length of Composition", "compLength"), ("Duration (Seconds)", "seconds")]), true), s("import.stillSeconds", "Still Duration", Kind::Float(0.04, 86_400.0, "s"), true), s("import.sequenceFps", "Sequence Footage", Kind::Float(1.0, 999.0, "frames per second"), true), - s("import.reportMissingFrames", "Report Missing Frames", B, false), + s("import.reportMissingFrames", "Report Missing Frames", B, true), s( "import.unlabeledAlpha", "Interpret Unlabeled Alpha As", @@ -795,13 +982,13 @@ pub fn pages() -> Vec { ("Straight", "straight"), ("Premultiplied", "premultiplied"), ]), - false, + true, ), s( "import.dragImportAs", "Default Drag Import As", Kind::Choice(&[("Footage", "footage"), ("Composition", "comp"), ("Composition - Retain Layer Sizes", "compLayerSizes")]), - false, + true, ), ], }, @@ -809,12 +996,12 @@ pub fn pages() -> Vec { id: "export", title: "Export", items: vec![ - s("export.defaultOutputFolder", "Default Output Folder", Kind::Path, false), - s("export.segmentSequences", "Segment Sequences", B, false), - s("export.segmentSequenceFiles", "Files per Segment", Kind::Int(1, 100_000, "files"), false), - s("export.segmentMovies", "Segment Movie Files", B, false), - s("export.segmentMovieMb", "Segment Size", Kind::Int(1, 1_000_000, "MB"), false), - s("export.appendBitsToName", "Append Bit Depth to File Name", B, false), + s("export.defaultOutputFolder", "Default Output Folder", Kind::Path, true), + s("export.segmentSequences", "Segment Sequences", B, true), + s("export.segmentSequenceFiles", "Files per Segment", Kind::Int(1, 100_000, "files"), true), + s("export.segmentMovies", "Segment Movie Files", B, true), + s("export.segmentMovieMb", "Segment Size", Kind::Int(1, 1_000_000, "MB"), true), + s("export.appendBitsToName", "Append Bit Depth to File Name", B, true), ], }, Page { @@ -823,7 +1010,7 @@ pub fn pages() -> Vec { items: vec![ Section("Audio Hardware"), AudioDevices { key: "audio.outputDevice" }, - s("audio.previewSampleRate", "Preview Sample Rate", Kind::Int(8_000, 192_000, "Hz"), false), + s("audio.previewSampleRate", "Preview Sample Rate", Kind::Int(8_000, 192_000, "Hz"), true), Section("Audio Output Mapping"), s("audio.outputLeft", "Left", Kind::Int(1, 64, "channel"), true), s("audio.outputRight", "Right", Kind::Int(1, 64, "channel"), true), @@ -834,36 +1021,36 @@ pub fn pages() -> Vec { title: "Disk", items: vec![ Section("Disk Cache"), - s("disk.diskCacheEnabled", "Enable Disk Cache", B, false), - s("disk.diskCacheMaxGb", "Maximum Disk Cache Size", Kind::Int(1, 100_000, "GB"), false), - s("disk.diskCacheFolder", "Disk Cache Folder", Kind::Path, false), + s("disk.diskCacheEnabled", "Enable Disk Cache", B, true), + s("disk.diskCacheMaxGb", "Maximum Disk Cache Size", Kind::Int(1, 100_000, "GB"), true), + s("disk.diskCacheFolder", "Disk Cache Folder", Kind::Path, true), Button { label: "Empty Disk Cache", command: "edit.purge", params: r#"{"what":"disk"}"# }, Section("Media Cache"), - s("disk.mediaCacheFolder", "Database and Cache Folder", Kind::Path, false), - s("disk.conformedMediaFolder", "Conformed Audio Folder", Kind::Path, false), + s("disk.mediaCacheFolder", "Database and Cache Folder", Kind::Path, true), + s("disk.conformedMediaFolder", "Conformed Audio Folder", Kind::Path, true), ], }, Page { id: "memory", title: "Memory & CPU", items: vec![ - s("memory.ramReservedGb", "RAM Reserved for Other Applications", Kind::Int(0, 1024, "GB"), false), + s("memory.ramReservedGb", "RAM Reserved for Other Applications", Kind::Int(0, 1024, "GB"), true), Section("Cache Budgets"), s("memory.layerCacheMb", "Layer Cache", Kind::Int(64, 1 << 20, "MB"), true), s("memory.mediaCacheMb", "Footage Frame Cache", Kind::Int(64, 1 << 20, "MB"), true), s("memory.previewCacheMb", "Preview (RAM) Cache", Kind::Int(64, 1 << 20, "MB"), true), - s("memory.reduceCacheWhenLow", "Reduce Cache Size When System Is Low on Memory", B, false), + s("memory.reduceCacheWhenLow", "Reduce Cache Size When System Is Low on Memory", B, true), ], }, Page { id: "video", title: "Video", items: vec![ - s("video.enableOutput", "Enable Video Preview Output", B, false), - s("video.device", "Video Device", Kind::Text, false), - s("video.outputDuringPlayback", "Video Output During Playback", B, false), - s("video.mirrorOnMonitor", "Mirror on Computer Monitor", B, false), - s("video.disableWhenBackground", "Disable Video Output When in Background", B, false), + s("video.enableOutput", "Enable Video Preview Output", B, true), + s("video.device", "Video Device", Kind::Choice(&[("Floating Window", ""), ("Full Screen", "fullscreen")]), true), + s("video.outputDuringPlayback", "Video Output During Playback", B, true), + s("video.mirrorOnMonitor", "Mirror on Computer Monitor", B, true), + s("video.disableWhenBackground", "Disable Video Output When in Background", B, true), ], }, Page { @@ -871,9 +1058,9 @@ pub fn pages() -> Vec { title: "3D", items: vec![ s("threeD.defaultRenderer", "Default 3D Renderer", Kind::Choice(&[("Classic 3D", "classic"), ("Advanced 3D", "advanced")]), true), - s("threeD.showReferenceAxes", "Show 3D Reference Axes", B, false), - s("threeD.extendedViewer", "Extended Viewer", B, false), - s("threeD.realtimeShadows", "Realtime Shadows in Draft", B, false), + s("threeD.showReferenceAxes", "Show 3D Reference Axes", B, true), + s("threeD.extendedViewer", "Extended Viewer", B, true), + s("threeD.realtimeShadows", "Realtime Shadows in Draft", B, true), ], }, Page { @@ -881,18 +1068,18 @@ pub fn pages() -> Vec { title: "Scripting & Expressions", items: vec![ Section("Application Scripting"), - s("scripting.allowScriptsWriteFiles", "Allow Scripts to Write Files and Access Network", B, false), - s("scripting.warnExecutingFiles", "Warn User When Executing Files", B, false), + s("scripting.allowScriptsWriteFiles", "Allow Scripts to Write Files and Access Network", B, true), + s("scripting.warnExecutingFiles", "Warn User When Executing Files", B, true), s("scripting.enableJsDebugger", "Enable JavaScript Debugger", B, false), Section("Expressions Editor"), - s("scripting.editorFontSize", "Font Size", Kind::Int(8, 40, "pt"), false), - s("scripting.syntaxHighlighting", "Syntax Highlighting", B, false), - s("scripting.lineNumbers", "Line Numbers", B, false), - s("scripting.autoComplete", "Auto-complete", B, false), - s("scripting.bracketMatching", "Bracket Matching", B, false), - s("scripting.wordWrap", "Word Wrap", B, false), - s("scripting.errorBanner", "Show Expression Error Banner", B, false), - Note("Expressions use the JavaScript engine."), + s("scripting.editorFontSize", "Font Size", Kind::Int(8, 40, "pt"), true), + s("scripting.syntaxHighlighting", "Syntax Highlighting", B, true), + s("scripting.lineNumbers", "Line Numbers", B, true), + s("scripting.autoComplete", "Auto-complete", B, true), + s("scripting.bracketMatching", "Bracket Matching", B, true), + s("scripting.wordWrap", "Word Wrap", B, true), + s("scripting.errorBanner", "Show Expression Error Banner", B, true), + Note("Expressions use the JavaScript engine. Script errors report their file and line in the Script Console; there is no step debugger."), ], }, ] diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index bc490f47..31325f4a 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -10,7 +10,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use effectcraft_project::render_queue::{OutputFormat, PostRenderAction, RenderQueueItem, RenderStatus, TemplateVars, expand_template}; +use effectcraft_project::render_queue::{OutputFormat, OutputModule, PostRenderAction, RenderQueueItem, RenderStatus, TemplateVars, expand_template}; use effectcraft_project::{ItemId, ItemKind, LayerSource, Project}; use effectcraft_render::{ExprHost, FootageSource}; use serde::Serialize; @@ -27,6 +27,8 @@ pub struct ExportJob<'a> { pub accel: Option<&'a dyn effectcraft_render::Accelerator>, /// Resolved output path (templates expanded except the `#` frame-number run). pub path: &'a str, + /// Settings ▸ General ▸ Switches Affect Nested Comps. + pub nested_switches: bool, } /// What an export wrote. @@ -124,8 +126,56 @@ struct Work { expr: Option>, accel: Option>, exporter: Arc, - /// Per queue item: one (item with that output module, resolved path) per output module. + /// Per queue item: one (item with that output module, resolved path) per output module + /// (and per segment, see [`Session::segments`]). items: Vec>, + /// Settings ▸ General ▸ Switches Affect Nested Comps. + nested_switches: bool, +} + +/// Bits per channel an output format writes for a project bit depth (named in the file when +/// Settings ▸ Export ▸ Append Bit Depth to File Name is on). +pub fn output_bits(format: OutputFormat, depth: effectcraft_project::BitDepth) -> u32 { + use effectcraft_project::BitDepth; + let project = match depth { + BitDepth::Bpc8 => 8, + BitDepth::Bpc16 => 16, + BitDepth::Bpc32 => 32, + }; + match format { + OutputFormat::ExrSequence => 32, + OutputFormat::PngSequence | OutputFormat::TiffSequence => project.min(16), + OutputFormat::ProRes => 10, + OutputFormat::Wav | OutputFormat::Aiff => 16, + _ => 8, + } +} + +/// The bytes an output writes per frame, estimated from its data rate (H.264 bitrate, Apple's +/// published ProRes target rates at 1080p29.97 scaled by frame size, typical rates otherwise). +pub fn bytes_per_frame(om: &OutputModule, w: u32, h: u32, fps: f64) -> f64 { + use effectcraft_project::render_queue::ProResProfile::*; + let fps = fps.max(1.0); + let px = (w as f64 * h as f64) / (1920.0 * 1080.0); + let mbps = |m: f64| m * 1_000_000.0 / 8.0; + match om.format { + OutputFormat::H264 => om.bitrate_kbps as f64 * 1000.0 / 8.0 / fps, + OutputFormat::ProRes => { + let rate = match om.prores_profile { + Proxy => 45.0, + Lt => 102.0, + Standard => 147.0, + Hq => 220.0, + P4444 => 330.0, + P4444Xq => 500.0, + }; + mbps(rate) * px / 29.97 + } + OutputFormat::WebM => mbps(8.0) * px / fps, + OutputFormat::Gif => w as f64 * h as f64 * 0.5, + OutputFormat::Wav | OutputFormat::Aiff => om.audio_sample_rate as f64 * 4.0 / fps, + _ => w as f64 * h as f64 * 3.0, + } } fn unix_now() -> u64 { @@ -158,7 +208,15 @@ fn run_work(w: Work, shared: &JobShared) { // first module's file is the item's output. let mut r: Result = Err("no output module".into()); for (mi, (mitem, path)) in modules.iter().enumerate() { - let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), accel: w.accel.as_deref(), item: mitem, path }; + let job = ExportJob { + project: &w.project, + footage: w.footage.as_ref(), + expr: w.expr.as_deref(), + accel: w.accel.as_deref(), + item: mitem, + path, + nested_switches: w.nested_switches, + }; let rr = w.exporter.export(&job, &mut |done, total| { let mut s = lock(&shared.state); s.done = done; @@ -228,12 +286,58 @@ impl Session { if path.is_absolute() { return Some(p); } - let base = self.path.as_deref().and_then(|s| std::path::Path::new(s).parent().map(|d| d.to_path_buf())).filter(|d| !d.as_os_str().is_empty()); + // Settings ▸ Export ▸ Default Output Folder, else the project's folder. + let folder = self.prefs.export.default_output_folder.trim(); + let base = (!folder.is_empty()).then(|| std::path::PathBuf::from(folder)); + let base = + base.or_else(|| self.path.as_deref().and_then(|s| std::path::Path::new(s).parent().map(|d| d.to_path_buf())).filter(|d| !d.as_os_str().is_empty())); // (the web has no working directory: relative outputs land in its virtual root) let base = base.or_else(|| std::env::current_dir().ok()).or_else(|| cfg!(target_arch = "wasm32").then(|| "/".into()))?; Some(base.join(path).to_string_lossy().to_string()) } + /// Settings ▸ Export ▸ Segment Sequences / Segment Movie Files: the (item, path) outputs one + /// output module writes. Sequences go into numbered folders of Files per Segment frames each + /// (`out/Comp 1_001/Comp 1_00000.png`…); movies into numbered files of about Segment Size MB + /// each (`Comp 1_001.mp4`…), cut at the frame count the output's data rate gives. + pub fn segments(&self, item: &RenderQueueItem, path: &str) -> Vec<(RenderQueueItem, String)> { + let Some(comp) = self.project.comp(item.comp) else { return vec![(item.clone(), path.to_string())] }; + let fmt = item.output.format; + let frames = item.settings.frame_count(comp); + let (w, h) = item.settings.output_size(comp); + let bpf = bytes_per_frame(&item.output, w, h, item.settings.rate(comp).as_f64()); + let Some(per) = self.prefs.segment_frames(frames, fmt.is_sequence(), bpf) else { return vec![(item.clone(), path.to_string())] }; + let p = std::path::Path::new(path); + let dir = p.parent().map(|d| d.to_path_buf()).unwrap_or_default(); + let stem = p.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + let ext = p.extension().map(|s| s.to_string_lossy().to_string()); + let rate = item.settings.rate(comp); + let f0 = item.settings.first_frame(comp); + let mut out = vec![]; + let mut i = 0; + let mut k = 1; + while i < frames { + let n = per.min(frames - i); + let mut m = item.clone(); + let (start, end) = (rate.tick_of(f0 + i as i64), rate.tick_of(f0 + (i + n) as i64)); + m.settings.time_span = effectcraft_project::render_queue::TimeSpan::Custom { start, end }; + let seg = if fmt.is_sequence() { + let base = stem.trim_end_matches(['#', '[', ']', '_']).to_string(); + let folder = dir.join(format!("{base}_{k:03}")); + folder.join(p.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default()) + } else { + dir.join(match &ext { + Some(e) => format!("{stem}_{k:03}.{e}"), + None => format!("{stem}_{k:03}"), + }) + }; + out.push((m, seg.to_string_lossy().to_string())); + i += n; + k += 1; + } + out + } + /// Whether a render job is running. pub fn is_rendering(&self) -> bool { self.render_job.as_ref().is_some_and(|j| !j.is_finished()) @@ -264,7 +368,13 @@ impl Session { let mut m = it.clone(); m.output = om.clone(); match self.resolve_output(&m) { - Some(p) if formats.contains(&om.format) => modules.push((m, p)), + Some(p) if formats.contains(&om.format) => { + // Settings ▸ Export: Append Bit Depth to File Name, Segment Sequences / + // Movie Files. + let bits = output_bits(om.format, self.project.settings.bit_depth); + let p = self.prefs.output_name(&p, bits); + modules.extend(self.segments(&m, &p)); + } Some(_) => problem = Some(RenderStatus::Failed(format!("{} export is not available", om.format.label()))), None if self.project.comp(it.comp).is_none() => problem = Some(RenderStatus::Failed("composition missing".into())), None => problem = Some(RenderStatus::NeedsOutput), @@ -291,7 +401,15 @@ impl Session { return Err("nothing is queued: add a composition (Composition ▸ Add to Render Queue) and tick Render".into()); } let ids: Vec = items.iter().filter_map(|m| m.first().map(|(i, _)| i.id)).collect(); - let work = Work { project: self.project.clone(), footage: self.footage.clone(), expr: self.expr.clone(), accel: self.accel.clone(), exporter, items }; + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + accel: self.accel.clone(), + exporter, + items, + nested_switches: self.prefs.general.switches_affect_nested_comps, + }; let shared = Arc::new(JobShared::default()); if wait { run_work(work, &shared); diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index e54eb93b..1d433100 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -25,9 +25,15 @@ impl Session { /// Settings changed: apply what the engine owns (cache budgets) and tell frontends. pub fn prefs_changed(&mut self) { self.prefs.normalize(); - self.layer_cache.set_budget(self.prefs.layer_cache_bytes()); - self.footage.set_cache_budget(self.prefs.media_cache_bytes()); + self.apply_cache_budgets(); self.configure_disk_cache(); + self.footage.set_conform_folder(self.conformed_audio_folder()); + // Switches Affect Nested Comps changes what precomp layers render: drop cached pixels. + let nested = self.prefs.general.switches_affect_nested_comps; + if self.applied_nested_switches.replace(nested).is_some_and(|was| was != nested) { + self.layer_cache.clear(); + self.events.push(crate::Event::PurgeCaches); + } // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; if self.history.undo.len() > levels { @@ -37,6 +43,53 @@ impl Session { self.prefs_revision += 1; } + /// A Disk settings folder: the setting, or (for frontends with a settings store) `name` next + /// to the platform disk cache folder. Headless sessions without the setting stay off disk. + fn cache_folder(&self, setting: &str, name: &str) -> Option { + let s = setting.trim(); + if !s.is_empty() { + return Some(std::path::PathBuf::from(s)); + } + let base = effectcraft_render::disk_cache::default_folder(); + self.config.is_some().then(|| base.parent().map(|p| p.join(name)).unwrap_or_else(|| base.join(name))) + } + + /// Settings ▸ Disk ▸ Database and Cache Folder (media cache: audio waveform summaries). + pub fn media_cache_folder(&self) -> Option { + self.cache_folder(&self.prefs.disk.media_cache_folder, "Media Cache") + } + + /// Settings ▸ Disk ▸ Conformed Audio Folder (decoded footage audio, see + /// `MediaPool::set_conform_folder`). + pub fn conformed_audio_folder(&self) -> Option { + self.cache_folder(&self.prefs.disk.conformed_media_folder, "Conformed Audio") + } + + /// Apply the Memory & CPU cache budgets, limited by RAM Reserved for Other Applications and + /// reduced while the system is low on memory (see [`crate::prefs::Prefs::cache_budgets`]). + pub fn apply_cache_budgets(&mut self) { + let b = self.prefs.cache_budgets(self.sys_memory); + self.layer_cache.set_budget(b.layer); + self.footage.set_cache_budget(b.media); + } + + /// Re-read the system's memory (desktop frontends call this every few seconds) and re-apply + /// the cache budgets when the reading changes them. Returns the effective budgets. + pub fn memory_tick(&mut self) -> crate::prefs::CacheBudgets { + let before = self.prefs.cache_budgets(self.sys_memory); + self.sys_memory = crate::sysinfo::memory().or(self.sys_memory); + let after = self.prefs.cache_budgets(self.sys_memory); + if after != before { + self.apply_cache_budgets(); + // Frontends re-apply their preview cache budget. + self.prefs_revision += 1; + if after.reduced && !before.reduced { + self.toast("The system is low on memory: cache sizes were reduced"); + } + } + after + } + /// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache /// and attach it to the layer cache. Without a folder setting the platform cache folder is /// used, but only by frontends with a settings store (headless sessions stay off disk). diff --git a/crates/engine/src/sysinfo.rs b/crates/engine/src/sysinfo.rs new file mode 100644 index 00000000..0c79c2a1 --- /dev/null +++ b/crates/engine/src/sysinfo.rs @@ -0,0 +1,140 @@ +//! Host information for Settings ▸ Memory & CPU (RAM reserved for other applications, reducing +//! caches when the system is low on memory) and Help ▸ System Compatibility Report. +//! +//! Pure Rust and dependency-free: the values come from the operating system's own reporting +//! (`/proc/meminfo` on Linux, `sysctl`/`vm_stat` on macOS, `wmic`/PowerShell on Windows). Every +//! query may fail (sandboxes, the web): callers treat `None` as "unknown" and keep their +//! configured budgets. + +use serde::Serialize; + +/// Physical memory in bytes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SysMemory { + pub total: u64, + /// Memory the system can hand out without swapping (free + reclaimable caches). + pub available: u64, +} + +impl SysMemory { + /// The system is low on memory: less than 10% (or less than 1 GB) is available. + pub fn is_low(&self) -> bool { + self.total > 0 && (self.available < self.total / 10 || self.available < 1 << 30) + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[allow(dead_code)] +fn run(cmd: &str, args: &[&str]) -> Option { + let out = std::process::Command::new(cmd).args(args).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).to_string()) +} + +/// Parse `/proc/meminfo` (kB values). +pub fn parse_meminfo(text: &str) -> Option { + let field = |name: &str| -> Option { + let line = text.lines().find(|l| l.starts_with(name))?; + let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + Some(kb * 1024) + }; + let total = field("MemTotal:")?; + let available = field("MemAvailable:").or_else(|| Some(field("MemFree:")? + field("Cached:").unwrap_or(0)))?; + Some(SysMemory { total, available }) +} + +/// Parse macOS `vm_stat` output: free + inactive + speculative + purgeable pages. +pub fn parse_vm_stat(text: &str, total: u64) -> Option { + let page: u64 = text.split("page size of ").nth(1)?.split_whitespace().next()?.parse().ok()?; + let pages = |name: &str| -> u64 { + text.lines() + .find(|l| l.starts_with(name)) + .and_then(|l| l.rsplit(':').next()) + .and_then(|v| v.trim().trim_end_matches('.').parse::().ok()) + .unwrap_or(0) + }; + let free = pages("Pages free") + pages("Pages inactive") + pages("Pages speculative") + pages("Pages purgeable"); + Some(SysMemory { total, available: (free * page).min(total) }) +} + +/// Total and available physical memory, when the system reports them. +pub fn memory() -> Option { + #[cfg(target_os = "linux")] + { + return parse_meminfo(&std::fs::read_to_string("/proc/meminfo").ok()?); + } + #[cfg(target_os = "macos")] + { + let total: u64 = run("sysctl", &["-n", "hw.memsize"])?.trim().parse().ok()?; + return parse_vm_stat(&run("vm_stat", &[])?, total).or(Some(SysMemory { total, available: total })); + } + #[cfg(target_os = "windows")] + { + let text = run( + "powershell", + &["-NoProfile", "-Command", "$o=Get-CimInstance Win32_OperatingSystem; \"$($o.TotalVisibleMemorySize) $($o.FreePhysicalMemory)\""], + )?; + let mut it = text.split_whitespace().filter_map(|v| v.parse::().ok()); + return Some(SysMemory { total: it.next()? * 1024, available: it.next()? * 1024 }); + } + #[allow(unreachable_code)] + None +} + +/// CPU model name. +pub fn cpu_name() -> Option { + #[cfg(target_os = "linux")] + { + let t = std::fs::read_to_string("/proc/cpuinfo").ok()?; + return t.lines().find(|l| l.starts_with("model name")).and_then(|l| l.split(':').nth(1)).map(|s| s.trim().to_string()); + } + #[cfg(target_os = "macos")] + { + return run("sysctl", &["-n", "machdep.cpu.brand_string"]).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + } + #[cfg(target_os = "windows")] + { + return std::env::var("PROCESSOR_IDENTIFIER").ok(); + } + #[allow(unreachable_code)] + None +} + +/// Operating system name and version. +pub fn os_version() -> String { + #[cfg(target_os = "macos")] + if let Some(v) = run("sw_vers", &["-productVersion"]) { + return format!("macOS {}", v.trim()); + } + #[cfg(target_os = "linux")] + if let Ok(t) = std::fs::read_to_string("/etc/os-release") + && let Some(n) = t.lines().find_map(|l| l.strip_prefix("PRETTY_NAME=")) + { + return n.trim_matches('"').to_string(); + } + #[cfg(target_os = "windows")] + if let Some(v) = run("cmd", &["/C", "ver"]) { + return v.trim().to_string(); + } + format!("{} ({})", std::env::consts::OS, std::env::consts::ARCH) +} + +/// Logical CPU cores. +pub fn cpu_cores() -> usize { + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_proc_meminfo_and_vm_stat() { + let m = parse_meminfo("MemTotal: 16000000 kB\nMemFree: 1000 kB\nMemAvailable: 8000000 kB\n").unwrap(); + assert_eq!(m, SysMemory { total: 16_000_000 * 1024, available: 8_000_000 * 1024 }); + assert!(!m.is_low()); + let vm = "Mach Virtual Memory Statistics: (page size of 16384 bytes)\nPages free: 1000.\nPages active: 5.\nPages inactive: 3000.\nPages speculative: 100.\n"; + let m = parse_vm_stat(vm, 8 << 30).unwrap(); + assert_eq!(m.available, 4100 * 16384); + assert!(m.is_low()); + } +} diff --git a/crates/engine/src/tests_m145.rs b/crates/engine/src/tests_m145.rs new file mode 100644 index 00000000..cc6a081c --- /dev/null +++ b/crates/engine/src/tests_m145.rs @@ -0,0 +1,666 @@ +//! M14.5: settings that change behaviour, the keyframe assistants and labels, and the menu +//! entries added for After Effects parity (History, recent footage / presets, XML copies, VR, +//! shortcut slots, logging and the compatibility report). + +use std::path::PathBuf; + +use serde_json::{Value, json}; + +use crate::render::RenderOpts; +use crate::{KeyRef, Session}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{ItemId, LayerId}; +use effectcraft_time::Tick; + +fn tmp(name: &str) -> PathBuf { + use std::sync::atomic::{AtomicU32, Ordering}; + static N: AtomicU32 = AtomicU32::new(0); + let d = std::env::temp_dir().join(format!("ec-m145-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d.join(name) +} + +fn session() -> Session { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Main", "width": 64, "height": 48, "duration": 2, "frameRate": 10})).unwrap(); + s +} + +fn solid(s: &mut Session, color: &str) -> u64 { + s.execute("layer.newSolid", json!({"color": color, "width": 32, "height": 24})).unwrap()["layer"].as_u64().unwrap() +} + +fn layer(s: &Session, id: u64) -> effectcraft_project::Layer { + s.active_comp().unwrap().layer(LayerId(id)).unwrap().clone() +} + +// ---------------------------------------------------------------- General + +#[test] +fn split_layers_go_above_or_below_the_original() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + s.execute("time.set", json!({"time": 1.0})).unwrap(); + let below = s.execute("edit.splitLayer", json!({"layers": [a]})).unwrap()[0].as_u64().unwrap(); + let order: Vec = s.active_comp().unwrap().layers.iter().map(|l| l.id.0).collect(); + assert_eq!(order, vec![a, below], "off (default): the new part goes below"); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("prefs.set", json!({"key": "general.createSplitLayersAbove", "value": true})).unwrap(); + let above = s.execute("edit.splitLayer", json!({"layers": [a]})).unwrap()[0].as_u64().unwrap(); + let order: Vec = s.active_comp().unwrap().layers.iter().map(|l| l.id.0).collect(); + assert_eq!(order, vec![above, a]); +} + +#[test] +fn new_masks_cycle_colours_only_when_asked() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + let colors = |s: &Session| -> Vec<[u8; 3]> { + layer(s, a) + .props + .sub("masks") + .unwrap() + .groups() + .map(|g| match g.kind { + effectcraft_project::GroupKind::Mask { color, .. } => color, + _ => [0; 3], + }) + .collect() + }; + s.execute("layer.addMask", json!({"layer": a})).unwrap(); + s.execute("layer.addMask", json!({"layer": a})).unwrap(); + let c = colors(&s); + assert_ne!(c[0], c[1], "cycling by default"); + s.execute("prefs.set", json!({"key": "appearance.cycleMaskColors", "value": false})).unwrap(); + s.execute("layer.addMask", json!({"layer": a})).unwrap(); + s.execute("mask.new", json!({"layer": a, "vertices": [[0, 0], [5, 5]]})).unwrap(); + let c = colors(&s); + assert_eq!(c[2], c[0]); + assert_eq!(c[3], c[0]); +} + +#[test] +fn preserve_constant_vertex_count_edits_every_mask_key() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + let m = s.execute("layer.addMask", json!({"layer": a})).unwrap()["mask"].as_u64().unwrap(); + s.execute("prop.select", json!({"layer": a, "prop": layer(&s, a).props.sub("masks").unwrap().groups().next().unwrap().get("path").unwrap().uid})).unwrap(); + let path_uid = layer(&s, a).props.sub("masks").unwrap().groups().next().unwrap().get("path").unwrap().uid; + s.execute("prop.toggleAnimation", json!({"layer": a, "prop": path_uid})).unwrap(); + s.execute("time.set", json!({"time": 1.0})).unwrap(); + s.execute("mask.setVertex", json!({"layer": a, "mask": m, "index": 0, "point": [1, 1]})).unwrap(); + let counts = |s: &Session| -> Vec { + layer(s, a) + .props + .find(path_uid) + .unwrap() + .keys + .iter() + .map(|k| match &k.value { + KV::Path(p) => p.vertices.len(), + _ => 0, + }) + .collect() + }; + assert_eq!(counts(&s), vec![4, 4]); + // On (the default): a vertex added at one key is added to the other. + s.execute("mask.insertVertex", json!({"layer": a, "mask": m, "segment": 0})).unwrap(); + assert_eq!(counts(&s), vec![5, 5]); + s.state.selected_vertices = vec![crate::VertexRef { layer: LayerId(a), mask: m, index: 1 }]; + s.execute("mask.deleteVertices", json!({})).unwrap(); + assert_eq!(counts(&s), vec![4, 4]); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(counts(&s), vec![5, 5], "one undo step"); + // Off: only the current key changes. + s.execute("prefs.set", json!({"key": "general.preserveConstantVertexCount", "value": false})).unwrap(); + s.execute("mask.addVertex", json!({"layer": a, "mask": m, "point": [3, 3]})).unwrap(); + assert_eq!(counts(&s), vec![5, 6]); +} + +#[test] +fn switches_affect_nested_comps_passes_quality_down() { + let mut s = Session::default(); + let inner = s.execute("comp.new", json!({"name": "Inner", "width": 40, "height": 40, "duration": 1})).unwrap()["comp"].as_u64().unwrap(); + let a = solid(&mut s, "#ff8040"); + // A rotated, scaled layer: bilinear (Best) and nearest (Draft) sampling differ. + s.execute("prop.set", json!({"layer": a, "path": "transform/rotation", "value": 17})).unwrap(); + s.execute("prop.set", json!({"layer": a, "path": "transform/scale", "value": [131, 131]})).unwrap(); + let outer = s.execute("comp.new", json!({"name": "Outer", "width": 40, "height": 40, "duration": 1})).unwrap()["comp"].as_u64().unwrap(); + let pre = s.execute("layer.addItem", json!({"item": inner})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.quality", json!({"layers": [pre], "quality": "draft"})).unwrap(); + let draw = |s: &Session| s.render(ItemId(outer), Tick::ZERO, RenderOpts::default()); + let on = draw(&s); + s.execute("prefs.set", json!({"key": "general.switchesAffectNestedComps", "value": false})).unwrap(); + let off = draw(&s); + assert_ne!(on.data, off.data, "the precomp's Draft switch reaches the nested layer only when on"); + // On = the nested layer itself set to Draft. + s.execute("prefs.set", json!({"key": "general.switchesAffectNestedComps", "value": true})).unwrap(); + s.open_comp(ItemId(inner)); + s.execute("layer.quality", json!({"layers": [a], "quality": "draft"})).unwrap(); + s.execute("prefs.set", json!({"key": "general.switchesAffectNestedComps", "value": false})).unwrap(); + assert_eq!(draw(&s).data, on.data); +} + +#[test] +fn related_comps_share_the_current_time() { + let mut s = Session::default(); + let inner = s.execute("comp.new", json!({"name": "Inner", "duration": 10, "frameRate": 10})).unwrap()["comp"].as_u64().unwrap(); + let outer = s.execute("comp.new", json!({"name": "Outer", "duration": 10, "frameRate": 10})).unwrap()["comp"].as_u64().unwrap(); + let l = s.execute("layer.addItem", json!({"item": inner})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.setTiming", json!({"layer": l, "startTime": 2.0})).ok(); + s.edit("t", None, |p, _| { + p.comp_mut(ItemId(outer)).unwrap().layer_mut(LayerId(l)).unwrap().start_time = Tick::from_seconds_f64(2.0); + Ok(()) + }) + .unwrap(); + s.set_time(Tick::from_seconds_f64(5.0)); + assert!((s.time_of(ItemId(inner)).seconds() - 3.0).abs() < 1e-9, "nested comp follows through the layer's start"); + s.open_comp(ItemId(inner)); + s.set_time(Tick::from_seconds_f64(1.0)); + assert!((s.time_of(ItemId(outer)).seconds() - 3.0).abs() < 1e-9, "and the other way"); + s.execute("prefs.set", json!({"key": "general.syncTimeRelatedItems", "value": false})).unwrap(); + s.set_time(Tick::from_seconds_f64(7.0)); + assert!((s.time_of(ItemId(outer)).seconds() - 3.0).abs() < 1e-9); +} + +#[test] +fn pick_whip_writes_match_names_without_compact_english() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + let r = s.execute("layer.newSolid", json!({"name": "Other", "color": "#00ff00"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": r, "effect": "Gaussian Blur"})).unwrap(); + s.execute("prefs.set", json!({"key": "general.expressionPickWhipCompact", "value": false})).unwrap(); + let e = s.execute("prop.pickWhip", json!({"layer": a, "path": "transform/opacity", "target": {"layer": r, "path": "transform/opacity"}})).unwrap(); + assert_eq!(e, json!("thisComp.layer(\"Other\")(\"transform\")(\"opacity\")")); + let e = s.execute("prop.pickWhip", json!({"layer": a, "path": "transform/rotation", "target": {"layer": a, "path": "transform/opacity"}})).unwrap(); + assert_eq!(e, json!("thisLayer(\"transform\")(\"opacity\")")); +} + +// ---------------------------------------------------------------- pure setting helpers + +#[test] +fn cache_budgets_leave_reserved_ram_and_shrink_when_low() { + use crate::sysinfo::SysMemory; + let mut p = crate::prefs::Prefs::default(); + p.memory.layer_cache_mb = 4096; + p.memory.media_cache_mb = 4096; + p.memory.preview_cache_mb = 8192; + p.memory.ram_reserved_gb = 6; + let b = p.cache_budgets(None); + assert_eq!((b.layer >> 20, b.preview >> 20, b.capped), (4096, 8192, false)); + // 16 GB machine, 6 GB reserved: 16 GB of caches scaled into 10 GB. + let m = SysMemory { total: 16 << 30, available: 12 << 30 }; + let b = p.cache_budgets(Some(m)); + assert!(b.capped && !b.reduced); + assert!(((b.layer + b.media + b.preview) as i64 - (10i64 << 30)).abs() < 4, "{b:?}"); + // Low on memory: halved; off: not. + let low = SysMemory { total: 16 << 30, available: 512 << 20 }; + let r = p.cache_budgets(Some(low)); + assert!(r.reduced && r.preview < b.preview / 2 + 2); + p.memory.reduce_cache_when_low = false; + assert!(!p.cache_budgets(Some(low)).reduced); +} + +#[test] +fn motion_path_span_follows_the_composition_setting() { + let mut p = crate::prefs::Prefs::default(); + let s = |x: f64| Tick::from_seconds_f64(x); + let keys: Vec = (0..40).map(|i| s(i as f64)).collect(); + // Default: no more than 15 keyframes, centred on the current time. + assert_eq!(p.motion_path_span(&keys, s(20.0)), Some((s(13.0), s(27.0)))); + assert_eq!(p.motion_path_span(&keys, s(0.0)), Some((s(0.0), s(14.0)))); + p.composition.motion_path = "all".into(); + assert_eq!(p.motion_path_span(&keys, s(5.0)), Some((s(0.0), s(39.0)))); + p.composition.motion_path = "none".into(); + assert_eq!(p.motion_path_span(&keys, s(5.0)), None); + p.composition.motion_path = "seconds".into(); + p.composition.motion_path_seconds = 4.0; + assert_eq!(p.motion_path_span(&keys, s(10.0)), Some((s(8.0), s(12.0)))); +} + +#[test] +fn import_export_helpers() { + use effectcraft_project::AlphaMode; + let mut p = crate::prefs::Prefs::default(); + assert_eq!(p.unlabeled_alpha(false), None, "Ask User"); + for (v, movie, want) in [ + ("ignore", false, AlphaMode::Ignore), + ("premultiplied", false, AlphaMode::Premultiplied), + ("guess", true, AlphaMode::Premultiplied), + ("guess", false, AlphaMode::Straight), + ] { + p.import.unlabeled_alpha = v.into(); + assert_eq!(p.unlabeled_alpha(movie), Some(want)); + } + assert_eq!(p.drag_import_as(), "footage"); + p.import.drag_import_as = "compLayerSizes".into(); + assert_eq!(p.drag_import_as(), "compositionLayerSizes"); + assert_eq!( + super::commands::missing_frames_for_tests(&["a_001.png".into(), "a_002.png".into(), "a_005.png".into(), "a_007.png".into()]).unwrap(), + "3 missing frames (3–4, 6)" + ); + assert!(super::commands::missing_frames_for_tests(&["a1.png".into(), "a2.png".into()]).is_none()); + // Append Bit Depth to File Name. + assert_eq!(p.output_name("/out/Comp 1.png", 16), "/out/Comp 1.png"); + p.export.append_bits_to_name = true; + assert_eq!(p.output_name("/out/Comp 1.png", 16), "/out/Comp 1_16bpc.png"); + assert_eq!(p.output_name("Comp_[#####].exr", 32), "Comp_[#####]_32bpc.exr"); + // Segments: sequences by file count, movies by size at the data rate. + assert_eq!(p.segment_frames(100, true, 0.0), None); + p.export.segment_sequences = true; + p.export.segment_sequence_files = 30; + assert_eq!(p.segment_frames(100, true, 0.0), Some(30)); + p.export.segment_movies = true; + p.export.segment_movie_mb = 1; + assert_eq!(p.segment_frames(100, false, 1_048_576.0 / 10.0), Some(10)); + assert_eq!(p.segment_frames(5, false, 1_048_576.0 / 10.0), None); + // Recent fonts. + for f in ["A", "B", "C", "A"] { + p.push_recent_font(f); + } + p.type_.recent_fonts = 2; + assert_eq!(p.recent_fonts_shown(), ["A".to_string(), "C".to_string()]); +} + +#[test] +fn render_queue_uses_default_folder_bit_depth_names_and_segments() { + use effectcraft_project::render_queue::{OutputFormat, RenderQueueItem}; + let mut s = session(); + solid(&mut s, "#ff0000"); + let dir = tmp("out"); + s.execute("prefs.set", json!({"values": {"export.defaultOutputFolder": dir.to_string_lossy(), "export.appendBitsToName": true}})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let mut item = RenderQueueItem::new(1, cid); + item.output.output = "Main.png".into(); + item.output.format = OutputFormat::PngSequence; + let out = s.resolve_output(&item).unwrap(); + assert_eq!(PathBuf::from(&out), dir.join("Main.png")); + assert_eq!( + s.prefs.output_name(&out, crate::render_queue::output_bits(OutputFormat::PngSequence, s.project.settings.bit_depth)), + dir.join("Main_8bpc.png").to_string_lossy() + ); + // 20 frames in segments of 8 files: three folders, spans tiling the work area. + s.prefs.export.segment_sequences = true; + s.prefs.export.segment_sequence_files = 8; + let segs = s.segments(&item, &out); + assert_eq!(segs.len(), 3); + let comp = s.active_comp().unwrap().clone(); + let n: Vec = segs.iter().map(|(i, _)| i.settings.frame_count(&comp)).collect(); + assert_eq!(n, vec![8, 8, 4]); + assert!(segs[1].1.contains("Main_002"), "{}", segs[1].1); + assert_eq!(segs[2].0.settings.first_frame(&comp), 16); + // Movies: a ~1 MB file per segment at the H.264 bitrate. + s.prefs.export.segment_movies = true; + s.prefs.export.segment_movie_mb = 1; + item.output.format = OutputFormat::H264; + item.output.bitrate_kbps = 4000; // 50 kB per frame at 10 fps + let segs = s.segments(&item, "/o/Main.mp4"); + assert_eq!(segs.len(), 1, "20 frames × 50 kB fit in 1 MB"); + item.output.bitrate_kbps = 8000; // 100 kB per frame: 10 frames per MB + let segs = s.segments(&item, "/o/Main.mp4"); + assert_eq!(segs.iter().map(|(_, p)| p.as_str()).collect::>(), ["/o/Main_001.mp4", "/o/Main_002.mp4"]); +} + +#[test] +fn every_wired_setting_is_live_and_display_only_ones_are_documented() { + let todo = crate::prefs::todo_keys(); + assert_eq!(todo, ["general.useSystemColorPicker", "composition.hardwareAcceleratePanels", "scripting.enableJsDebugger"]); + let docs = include_str!("../../../docs/preferences.md"); + assert!(docs.contains("### Display-only settings")); + assert!(!docs.contains("Not wired yet")); +} + +// ---------------------------------------------------------------- scripting gate + +#[test] +fn execute_file_needs_the_gate_and_asks_first() { + let mut s = session(); + let f = tmp("run.txt"); + std::fs::write(&f, "x").unwrap(); + let p = json!({"path": f.to_string_lossy()}); + assert!(s.execute("file.executeFile", p.clone()).is_err(), "off by default"); + s.execute("prefs.set", json!({"key": "scripting.allowScriptsWriteFiles", "value": true})).unwrap(); + s.drain_events(); + let r = s.execute("file.executeFile", p.clone()).unwrap(); + assert!(r.get("needsConfirmation").is_some()); + assert!(s.drain_events().iter().any(|e| matches!(e, crate::Event::Frontend { command, .. } if command == "app.confirmExecute"))); + let r = s.execute("file.executeFile", json!({"path": f.to_string_lossy(), "confirmed": true})).unwrap(); + assert!(r.get("opened").is_some()); + assert!(s.drain_events().iter().any(|e| matches!(e, crate::Event::OpenUrl(u) if u.starts_with("file://")))); + s.execute("prefs.set", json!({"key": "scripting.warnExecutingFiles", "value": false})).unwrap(); + assert!(s.execute("file.executeFile", p).unwrap().get("opened").is_some()); +} + +// ---------------------------------------------------------------- Edit ▸ History, recent lists + +#[test] +fn history_submenu_jumps_back_several_steps() { + let mut s = session(); + solid(&mut s, "#ff0000"); + solid(&mut s, "#00ff00"); + solid(&mut s, "#0000ff"); + let list = s.execute("edit.history", json!({})).unwrap(); + assert_eq!(list["undo"][0], json!("New Solid")); + let n = s.history.undo.len(); + let (entries, _) = crate::menus::dynamic(&s, "history", &Default::default()); + assert_eq!(entries.len(), n); + assert_eq!(entries[1].params, json!({"steps": 2})); + s.execute("edit.history", json!({"steps": 2})).unwrap(); + assert_eq!(s.active_comp().unwrap().layers.len(), 1); + assert_eq!(s.history.redo.len(), 2); + s.execute("edit.history", json!({"redo": 2})).unwrap(); + assert_eq!(s.active_comp().unwrap().layers.len(), 3); + assert!(s.execute("edit.history", json!({"steps": 999})).is_err()); +} + +#[test] +fn recent_footage_and_presets_are_remembered() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + // Imports are remembered even when the build has no decoder (the error comes after). + let _ = s.execute("file.import", json!({"paths": ["/x/clip.mov", "/x/still.png"]})); + assert_eq!(s.prefs.recent_footage, ["/x/still.png", "/x/clip.mov"]); + let (e, _) = crate::menus::dynamic(&s, "recentFootage", &Default::default()); + assert_eq!(e.iter().map(|e| e.label.as_str()).collect::>(), ["still.png", "clip.mov"]); + s.execute("file.clearRecentFootage", json!({})).unwrap(); + assert_eq!(crate::menus::dynamic(&s, "recentFootage", &Default::default()), (vec![], Some("No Recent Footage"))); + // Presets. + let path = tmp("fade.ecpreset"); + s.execute("prop.select", json!({"layer": a, "path": "transform/opacity"})).unwrap(); + s.execute("anim.savePreset", json!({"path": path})).unwrap(); + s.execute("anim.applyPreset", json!({"path": path, "layers": [a]})).unwrap(); + assert_eq!(s.prefs.recent_presets, [path.to_string_lossy().to_string()]); + let (e, _) = crate::menus::dynamic(&s, "recentPresets", &Default::default()); + assert_eq!(e[0].label, "fade"); + s.state.selected_layers = vec![LayerId(a)]; + s.execute("anim.applyRecentPreset", json!({"index": 0})).unwrap(); + s.execute("anim.clearRecentPresets", json!({})).unwrap(); + assert!(s.prefs.recent_presets.is_empty()); + // Recent lists survive a settings round trip and a reset. + s.prefs.push_recent_footage("/y/a.wav"); + let back = crate::prefs::Prefs::from_json(&s.prefs.to_json()); + assert_eq!(back.recent_footage, ["/y/a.wav"]); + s.execute("prefs.reset", json!({})).unwrap(); + assert_eq!(s.prefs.recent_footage, ["/y/a.wav"]); +} + +#[test] +fn save_a_copy_as_xml_opens_again() { + let mut s = session(); + solid(&mut s, "#ff0000"); + let path = tmp("copy.ecprojx"); + s.execute("file.saveCopyAsXml", json!({"path": path.to_string_lossy()})).unwrap(); + let text = std::fs::read_to_string(&path).unwrap(); + assert!(text.starts_with(" = r["faces"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect(); + assert_eq!(faces.len(), 6); + for f in &faces { + let c = s.project.comp(ItemId(*f)).unwrap(); + assert_eq!((c.width, c.height), (64, 64)); + assert!(c.layers.iter().any(|l| l.is_camera())); + assert!(c.layers.iter().any(|l| l.name == aname && l.is_3d()), "the scene's layers"); + let _ = main; + } + let cube = s.project.comp(ItemId(r["cubeMap"].as_u64().unwrap())).unwrap(); + assert_eq!((cube.width, cube.height, cube.layers.len()), (192, 128, 6)); + let out = s.project.comp(ItemId(r["output"].as_u64().unwrap())).unwrap(); + assert_eq!((out.width, out.height), (128, 64)); + let fx = out.layers[0].props.sub("effects").unwrap().groups().next().unwrap().clone(); + assert_eq!(fx.match_id, "ec.vr.converter"); + // Every face camera looks a different way. + let mut dirs = vec![]; + for f in &faces { + let c = s.project.comp(ItemId(*f)).unwrap(); + let cam = c.layers.iter().find(|l| l.is_camera()).unwrap(); + dirs.push(format!("{:?}", cam.props.prop("transform/orientation").unwrap().value)); + } + dirs.sort(); + dirs.dedup(); + assert_eq!(dirs.len(), 6, "{dirs:?}"); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.comp(ItemId(faces[0])).is_none()); + // Extract Cubemap of a 2:1 comp. + let eq = s.execute("comp.new", json!({"name": "Equi", "width": 128, "height": 64, "duration": 1})).unwrap()["comp"].as_u64().unwrap(); + let r = s.execute("comp.vr.extractCubemap", json!({"comp": eq})).unwrap(); + let c = s.project.comp(ItemId(r["cubeMap"].as_u64().unwrap())).unwrap(); + assert_eq!((c.width, c.height), (96, 64)); +} + +/// The equirectangular output sees each wall of a box around the rig in the right direction. +#[test] +fn vr_environment_faces_look_the_right_way() { + // One wall at a time (centre 100,100,0; AE's −Y is up), and where the equirectangular output + // must show it: front +Z, back −Z, right +X, left −X, up, down. + let walls = [ + ("front", [100.0, 100.0, 150.0], [0.0, 0.0], (0.5, 0.5)), + ("back", [100.0, 100.0, -150.0], [0.0, 180.0], (0.01, 0.5)), + ("right", [250.0, 100.0, 0.0], [0.0, 90.0], (0.75, 0.5)), + ("left", [-50.0, 100.0, 0.0], [0.0, 90.0], (0.25, 0.5)), + ("up", [100.0, -50.0, 0.0], [90.0, 0.0], (0.5, 0.03)), + ("down", [100.0, 250.0, 0.0], [90.0, 0.0], (0.5, 0.97)), + ]; + for (name, pos, rot, (fx, fy)) in walls { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Room", "width": 200, "height": 200, "duration": 1, "frameRate": 10})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 200, "height": 200})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.setSwitch", json!({"layers": [l], "switch": "threeD", "value": true})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "transform/position", "value": pos})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "transform/rotationX", "value": rot[0]})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "transform/rotationY", "value": rot[1]})).unwrap(); + let r = s.execute("comp.vr.createEnvironment", json!({"size": 64})).unwrap(); + let img = s.render(ItemId(r["output"].as_u64().unwrap()), Tick::ZERO, RenderOpts::default()); + let (w, h) = (img.width as f64, img.height as f64); + let alpha = + |x: f64, y: f64| img.data[((y * h) as usize).min(img.height as usize - 1) * img.width as usize + ((x * w) as usize).min(img.width as usize - 1)][3]; + assert!(alpha(fx, fy) > 0.5, "{name}: the wall shows in its direction"); + // The opposite direction is empty. + let (ox, oy) = if fy != 0.5 { (fx, 1.0 - fy) } else { ((fx + 0.5) % 1.0, fy) }; + assert!(alpha(ox, oy) < 0.5, "{name}: nothing in the opposite direction"); + } +} + +// ---------------------------------------------------------------- keyframes + +#[test] +fn convert_audio_to_keyframes_makes_the_amplitude_null() { + assert_eq!(crate::commands::amplitudes_for_tests(&[0.5, -0.25, -0.5, 0.25]), (50.0, 25.0, 37.5)); + let mut s = session(); + let undo = s.history.undo.len(); + let r = s.execute("keys.audioToKeyframes", json!({})).unwrap(); + assert_eq!(s.history.undo.len(), undo + 1); + let l = layer(&s, r["layer"].as_u64().unwrap()); + assert_eq!(l.name, "Audio Amplitude"); + let fx: Vec<_> = l.props.sub("effects").unwrap().groups().map(|g| (g.name.clone(), g.get("slider").unwrap().keys.len())).collect(); + assert_eq!(fx, [("Left Channel".to_string(), 20), ("Right Channel".to_string(), 20), ("Both Channels".to_string(), 20)]); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.active_comp().unwrap().layers.is_empty()); +} + +#[test] +fn keyframe_labels_and_label_groups() { + let mut s = session(); + let a = solid(&mut s, "#ff0000"); + for t in [0.0, 0.5, 1.0] { + s.execute("prop.addKey", json!({"layer": a, "path": "transform/opacity", "time": t, "value": 50})).unwrap(); + } + let uid = layer(&s, a).props.prop("transform/opacity").unwrap().uid; + let k = |t: f64| KeyRef { layer: LayerId(a), prop: uid, time: Tick::from_seconds_f64(t) }; + s.state.selected_keys = vec![k(0.0), k(1.0)]; + // Edit ▸ Label colours the selected keyframes. + s.execute("edit.label", json!({"label": "Blue"})).unwrap(); + let labels: Vec = layer(&s, a).props.prop("transform/opacity").unwrap().keys.iter().map(|k| k.label).collect(); + let blue = effectcraft_color::Label::ALL.iter().position(|l| *l == effectcraft_color::Label::Blue).unwrap() as u8; + assert_eq!(labels, [blue, 0, blue]); + assert_eq!(layer(&s, a).label, effectcraft_color::Label::Red, "the layer keeps its label"); + // Serde: labels round-trip; unlabeled keys don't write the field. + let json = s.project.to_json(); + assert!(json.contains(&format!("\"label\": {blue}")) || json.contains(&format!("\"label\":{blue}"))); + let back = effectcraft_project::Project::from_json(&json).unwrap(); + assert_eq!(back.comp(s.active_comp_id().unwrap()).unwrap().layer(LayerId(a)).unwrap().props.prop("transform/opacity").unwrap().keys[2].label, blue); + // Moving a labelled key's value keeps the label. + s.execute("time.set", json!({"time": 1.0})).unwrap(); + s.execute("prop.set", json!({"layer": a, "path": "transform/opacity", "value": 80})).unwrap(); + assert_eq!(layer(&s, a).props.prop("transform/opacity").unwrap().keys[2].label, blue); + // Select Keyframe Label Group. + s.state.selected_keys = vec![k(0.0)]; + let r = s.execute("keys.selectLabelGroup", json!({"scope": "all"})).unwrap(); + assert_eq!(r["keys"], 2); + s.state.selected_keys = vec![k(0.5)]; + s.execute("keys.selectLabelGroup", json!({"scope": "selected"})).unwrap(); + assert_eq!(s.state.selected_keys, vec![k(0.5)]); + s.state.selected_keys = vec![k(0.0)]; + let r = s.execute("keys.selectLabelGroup", json!({"scope": "visibleAll", "props": [12345]})).unwrap(); + assert_eq!(r["keys"], 0, "only the given (visible) properties"); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert!(layer(&s, a).props.prop("transform/opacity").unwrap().keys.iter().all(|k| k.label == 0)); +} + +#[test] +fn rpf_camera_import_reads_json_and_csv_camera_data() { + let mut s = session(); + let json_path = tmp("cam.json"); + std::fs::write( + &json_path, + r#"{"frameRate": 10, "frames": [{"frame": 0, "position": [0, 0, -500], "rotation": [0, 0, 0], "fov": 90}, {"frame": 10, "position": [100, 0, -500], "orientation": [0, 30, 0], "zoom": 400}]}"#, + ) + .unwrap(); + let undo = s.history.undo.len(); + let r = s.execute("keys.rpfCameraImport", json!({"path": json_path.to_string_lossy()})).unwrap(); + assert_eq!(s.history.undo.len(), undo + 1); + let cam = layer(&s, r["layer"].as_u64().unwrap()); + assert!(cam.is_camera()); + let pos = &cam.props.prop("transform/position").unwrap().keys; + assert_eq!(pos.len(), 2); + assert_eq!(pos[1].value, KV::Vec3([100.0, 0.0, -500.0])); + assert!((pos[1].time.seconds() - 1.0).abs() < 1e-9); + let zoom = &cam.props.prop("cameraOptions/zoom").unwrap().keys; + assert!((zoom[0].value.as_f64() - 32.0).abs() < 1e-6, "90° across 64 px = zoom 32: {:?}", zoom[0].value); + let csv = tmp("cam.csv"); + std::fs::write(&csv, "frame,px,py,pz,rx,ry,rz,zoom\n0,1,2,3,0,0,0,100\n5,4,5,6,10,20,30,200\n").unwrap(); + let r = s.execute("keys.rpfCameraImport", json!({"path": csv.to_string_lossy()})).unwrap(); + let cam = layer(&s, r["layer"].as_u64().unwrap()); + let o = &cam.props.prop("transform/orientation").unwrap().keys; + assert_eq!(o[1].value, KV::Vec3([10.0, 20.0, 30.0])); + assert!(s.execute("keys.rpfCameraImport", json!({"path": "/x/scene.rpf"})).unwrap_err().to_string().contains("JSON or CSV")); +} + +// ---------------------------------------------------------------- menus + +#[test] +fn dynamic_menus_and_window_viewer_labels() { + let mut s = session(); + s.prefs.push_recent("/p/One.ecproj"); + let (e, _) = crate::menus::dynamic(&s, "recentProjects", &Default::default()); + assert_eq!((e[0].label.as_str(), e[0].command.as_str()), ("One.ecproj", "file.openRecent")); + assert_eq!(crate::menus::dynamic(&s, "history", &Default::default()).0.len(), s.history.undo.len()); + let a = solid(&mut s, "#ff0000"); + s.state.selected_layers = vec![LayerId(a)]; + let entry = |cmd: &str, panel: &str| crate::menus::MenuEntry { label: "x".into(), command: cmd.into(), params: json!({"panel": panel}), shortcut: None }; + assert_eq!(crate::menus::entry_label(&s, &entry("window.panel", "composition")), "Composition: Main"); + assert_eq!(crate::menus::entry_label(&s, &entry("window.panel", "timeline")), "Timeline: Main"); + assert!(crate::menus::entry_label(&s, &entry("window.panel", "layer")).starts_with("Layer: ")); + assert_eq!(crate::menus::entry_label(&s, &entry("window.unavailablePanel", "footage")), "Footage: (none)"); + // Other open comps are listed as viewers. + let other = s.execute("comp.new", json!({"name": "Second"})).unwrap()["comp"].as_u64().unwrap(); + let (e, _) = crate::menus::dynamic(&s, "openViewers", &Default::default()); + assert_eq!(e.iter().map(|e| e.label.as_str()).collect::>(), ["Composition: Main"]); + assert_eq!(e[0].command, "comp.open"); + let _ = other; + // Every tree entry of the new menus resolves. + for (path, e) in crate::menus::entries() { + if ["History", "VR", "Recent Animation Presets", "Import Recent Footage"].iter().any(|p| path.contains(&p.to_string())) { + assert!(crate::commands::find(&e.command).is_some()); + } + } + let labels: Vec = crate::menus::entries().iter().map(|(_, e)| e.label.clone()).collect(); + for l in [ + "Save a Copy As XML...", + "With Layered Comp", + "Create VR Environment...", + "Extract Cubemap...", + "System Compatibility Report...", + "Enable Logging", + "Reveal Logging File", + ] { + assert!(labels.iter().any(|x| x == l), "{l}"); + } +} diff --git a/crates/engine/src/xml_project.rs b/crates/engine/src/xml_project.rs new file mode 100644 index 00000000..ae899d46 --- /dev/null +++ b/crates/engine/src/xml_project.rs @@ -0,0 +1,265 @@ +//! File ▸ Save a Copy As XML…: the project as an XML document (`.ecprojx`), the counterpart of +//! After Effects' `.aepx`. It is a lossless, element-per-value encoding of the `.ecproj` JSON, so +//! File ▸ Open reads it back (and saving to a `.ecprojx` path writes XML again): +//! +//! ```xml +//! +//! +//! +//! 3 +//! … +//! +//! +//! ``` +//! +//! `obj`/`m n="key"` (object members), `arr`, `str`, `num`, `bool`, `null`. + +use serde_json::{Map, Value}; + +const ROOT: &str = "EffectCraftProject"; + +/// Whether `text` looks like an XML project. +pub fn is_xml(text: &str) -> bool { + let t = text.trim_start_matches('\u{feff}').trim_start(); + t.starts_with(" bool { + let p = path.to_ascii_lowercase(); + p.ends_with(".ecprojx") || p.ends_with(".xml") +} + +fn esc(s: &str, out: &mut String) { + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\r' => out.push_str(" "), + c if (c as u32) < 0x20 && c != '\n' && c != '\t' => out.push_str(&format!("&#{};", c as u32)), + c => out.push(c), + } + } +} + +fn write(v: &Value, depth: usize, out: &mut String) { + let pad = " ".repeat(depth); + out.push_str(&pad); + match v { + Value::Null => out.push_str(""), + Value::Bool(b) => out.push_str(&format!("{b}")), + Value::Number(n) => out.push_str(&format!("{n}")), + Value::String(s) => { + out.push_str(""); + esc(s, out); + out.push_str(""); + } + Value::Array(a) if a.is_empty() => out.push_str(""), + Value::Array(a) => { + out.push_str("\n"); + for x in a { + write(x, depth + 1, out); + } + out.push_str(&pad); + out.push_str(""); + } + Value::Object(m) if m.is_empty() => out.push_str(""), + Value::Object(m) => { + out.push_str("\n"); + for (k, x) in m { + out.push_str(&" ".repeat(depth + 1)); + out.push_str("\n"); + write(x, depth + 2, out); + out.push_str(&" ".repeat(depth + 1)); + out.push_str("\n"); + } + out.push_str(&pad); + out.push_str(""); + } + } + out.push('\n'); +} + +/// The project JSON as an XML document. +pub fn to_xml(json: &str) -> Result { + let v: Value = serde_json::from_str(json).map_err(|e| e.to_string())?; + let mut out = format!("\n<{ROOT} encoding=\"json\" version=\"1\">\n"); + write(&v, 1, &mut out); + out.push_str(&format!("\n")); + Ok(out) +} + +struct P<'a> { + s: &'a str, + i: usize, +} + +impl<'a> P<'a> { + fn err(&self, m: &str) -> Result { + Err(format!("XML project: {m} at byte {}", self.i)) + } + fn rest(&self) -> &'a str { + &self.s[self.i..] + } + /// Skip whitespace, comments and processing instructions. + fn skip(&mut self) { + loop { + let t = self.rest(); + let trimmed = t.trim_start(); + self.i += t.len() - trimmed.len(); + if trimmed.starts_with("").map_or(trimmed.len(), |e| e + 2); + } else if trimmed.starts_with("").map_or(trimmed.len(), |e| e + 3); + } else { + return; + } + } + } + /// An opening tag: (name, attribute `n`, self-closing). + fn open(&mut self) -> Result<(String, Option, bool), String> { + self.skip(); + if !self.rest().starts_with('<') || self.rest().starts_with("').ok_or("XML project: unterminated tag")?; + let inner = &self.rest()[1..end]; + self.i += end + 1; + let (inner, closed) = match inner.strip_suffix('/') { + Some(x) => (x, true), + None => (inner, false), + }; + let mut parts = inner.splitn(2, char::is_whitespace); + let name = parts.next().unwrap_or_default().to_string(); + let attrs = parts.next().unwrap_or(""); + let n = attrs.find("n=\"").map(|a| { + let v = &attrs[a + 3..]; + unesc(&v[..v.find('"').unwrap_or(v.len())]) + }); + Ok((name, n, closed)) + } + fn close(&mut self, name: &str) -> Result<(), String> { + self.skip(); + let want = format!(""); + if self.rest().starts_with(&want) { + self.i += want.len(); + Ok(()) + } else { + self.err(&format!("expected {want}")) + } + } + fn text(&mut self) -> String { + let end = self.rest().find('<').unwrap_or(self.rest().len()); + let t = unesc(&self.rest()[..end]); + self.i += end; + t + } + fn value(&mut self) -> Result { + let (name, _, closed) = self.open()?; + let v = match (name.as_str(), closed) { + ("null", _) => Value::Null, + ("arr", true) => Value::Array(vec![]), + ("obj", true) => Value::Object(Map::new()), + ("str", true) => Value::String(String::new()), + ("bool", false) => Value::Bool(self.text().trim() == "true"), + ("num", false) => serde_json::from_str(self.text().trim()).map_err(|e| format!("XML project: bad number: {e}"))?, + ("str", false) => Value::String(self.text()), + ("arr", false) => { + let mut a = vec![]; + loop { + self.skip(); + if self.rest().starts_with(" { + let mut m = Map::new(); + loop { + self.skip(); + if self.rest().starts_with(""); + } + let v = self.value()?; + self.close("m")?; + m.insert(key.unwrap_or_default(), v); + } + Value::Object(m) + } + (other, _) => return self.err(&format!("unexpected <{other}>")), + }; + if !closed && name != "null" { + self.close(&name)?; + } + Ok(v) + } +} + +fn unesc(s: &str) -> String { + if !s.contains('&') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(a) = rest.find('&') { + out.push_str(&rest[..a]); + rest = &rest[a..]; + let Some(e) = rest.find(';') else { break }; + let ent = &rest[1..e]; + match ent { + "amp" => out.push('&'), + "lt" => out.push('<'), + "gt" => out.push('>'), + "quot" => out.push('"'), + "apos" => out.push('\''), + n if n.starts_with("#x") => out.extend(u32::from_str_radix(&n[2..], 16).ok().and_then(char::from_u32)), + n if n.starts_with('#') => out.extend(n[1..].parse::().ok().and_then(char::from_u32)), + other => { + out.push('&'); + out.push_str(other); + out.push(';'); + } + } + rest = &rest[e + 1..]; + } + out.push_str(rest); + out +} + +/// The project JSON of an XML project document. +pub fn from_xml(text: &str) -> Result { + let mut p = P { s: text.trim_start_matches('\u{feff}'), i: 0 }; + let (root, _, _) = p.open()?; + if root != ROOT { + return Err(format!("not an EffectCraft XML project (root <{root}>)")); + } + let v = p.value()?; + p.close(ROOT)?; + serde_json::to_string(&v).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_round_trips_through_xml() { + let j = serde_json::json!({"a": [1, 2.5, -3e-7, true, null, "x")); + let back: Value = serde_json::from_str(&from_xml(&xml).unwrap()).unwrap(); + assert_eq!(back, j); + assert!(from_xml("").is_err()); + } +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index a2af6a4f..30da500a 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -69,6 +69,8 @@ pub struct Job<'a> { pub path: &'a str, /// Hand finished files to this instead of writing them to disk (web downloads, tests). pub sink: Option<&'a Sink>, + /// Settings ▸ General ▸ Switches Affect Nested Comps (see `RenderOpts::nested_switches`). + pub nested_switches: bool, } /// Progress after a batch of frames. @@ -182,6 +184,8 @@ pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { view: None, backend: effectcraft_render::Backend::Auto, roi: None, + nested_switches: job.nested_switches, + draft_shadows: true, }; let mut r = Renderer::new(job.project, job.footage, opts); r.expr = job.expr; diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 1ade594f..f8bef9f2 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -55,7 +55,7 @@ fn settings() -> RenderSettings { fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); - let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None }; + let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None, nested_switches: true }; let mut last = Progress::default(); let mut calls = 0; let r = export(&job, &mut |pr| { @@ -167,7 +167,8 @@ fn h264_odd_size_is_cropped_even() { let comp = p.comp(cid).expect("comp"); assert_eq!(s.output_size(comp).0 % 2, 1); let path = d.join("odd.mp4").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = + Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None, nested_switches: true }; let r = export(&job, &mut |_| true).expect("export"); assert_eq!((r.width % 2, r.height % 2), (0, 0)); } @@ -287,7 +288,8 @@ fn cancel_stops_early() { let om = OutputModule::for_format(OutputFormat::PngSequence); let s = settings(); let path = d.join("c_[#####].png").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = + Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None, nested_switches: true }; let r = export(&job, &mut |_| false); assert!(matches!(r, Err(effectcraft_export::ExportError::Cancelled))); let written = std::fs::read_dir(&d).map(|r| r.count()).unwrap_or(0); diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index aa94b558..20a76cb3 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -40,6 +40,7 @@ impl Exporter for FileExporter { output: &job.item.output, path: job.path, sink: self.sink.as_deref(), + nested_switches: job.nested_switches, }; match effectcraft_export::export(&j, &mut |p| progress(p.done, p.total)) { Ok(r) => Ok(ExportResult { path: r.path, frames: r.frames, width: r.width, height: r.height, bytes: r.bytes, seconds: r.seconds, audio: r.audio }), diff --git a/crates/keyframe/src/lib.rs b/crates/keyframe/src/lib.rs index 731e167c..7a3c6c89 100644 --- a/crates/keyframe/src/lib.rs +++ b/crates/keyframe/src/lib.rs @@ -84,6 +84,14 @@ pub struct Keyframe { pub spatial_continuous: bool, #[serde(default)] pub roving: bool, + /// Keyframe colour label (Edit ▸ Label on selected keyframes): 0 = None, 1–16 = the 16 + /// label colours (Red … Dark Green). + #[serde(default, skip_serializing_if = "is_zero")] + pub label: u8, +} + +fn is_zero(v: &u8) -> bool { + *v == 0 } fn yes() -> bool { @@ -106,6 +114,7 @@ impl Keyframe { spatial_auto: true, spatial_continuous: false, roving: false, + label: 0, } } pub fn hold(mut self) -> Keyframe { @@ -512,6 +521,7 @@ pub fn set_key(keys: &mut Vec, key: Keyframe) -> usize { k.spatial_in = old.spatial_in; k.spatial_out = old.spatial_out; k.spatial_auto = old.spatial_auto; + k.label = old.label; keys[i] = k; i } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index f604a69a..b34463a0 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -131,6 +131,11 @@ struct Inner { hits: AtomicU64, misses: AtomicU64, prefetched: AtomicU64, + /// Settings ▸ Disk ▸ Conformed Audio Folder: decoded audio tracks written once per file and + /// rate, read back instead of decoding again (`None` = off). + conform: Mutex>, + /// Conformed files being written right now. + conforming: Mutex>, } /// Decodes footage for the renderer: stills, image sequences and movies (FilmCraft codecs), plus @@ -193,6 +198,8 @@ impl MediaPool { hits: AtomicU64::new(0), misses: AtomicU64::new(0), prefetched: AtomicU64::new(0), + conform: Mutex::default(), + conforming: Mutex::default(), }), } } @@ -257,14 +264,108 @@ impl MediaPool { Inner::frame_at(&self.inner, footage, t, true) } + /// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there + /// once per sample rate (raw interleaved stereo `f32`) and read it back instead of decoding + /// again. `None` turns it off. + pub fn set_conform_folder(&self, folder: Option) { + *lock(&self.inner.conform) = folder; + } + + /// The conformed-audio file of `path` at `rate` (named by a hash of the path, size and + /// modification time, so an edited file conforms again). + pub fn conformed_path(&self, path: &str, rate: u32) -> Option { + use std::hash::{Hash, Hasher}; + let folder = lock(&self.inner.conform).clone()?; + let meta = std::fs::metadata(path).ok()?; + let mut h = std::collections::hash_map::DefaultHasher::new(); + (path, meta.len(), meta.modified().ok()).hash(&mut h); + Some(folder.join(format!("{:016x}_{rate}.ecaf", h.finish()))) + } + + /// Samples from a conformed file: `Some` when it exists and covers the request. + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + fn read_conformed(file: &std::path::Path, s0: usize, frames: usize) -> Option> { + use std::io::{Read, Seek, SeekFrom}; + let mut f = std::fs::File::open(file).ok()?; + let len = f.metadata().ok()?.len() as usize / 8; + let mut out = vec![0.0f32; frames * 2]; + if s0 >= len { + return Some(out); + } + let n = frames.min(len - s0); + f.seek(SeekFrom::Start(s0 as u64 * 8)).ok()?; + let mut bytes = vec![0u8; n * 8]; + f.read_exact(&mut bytes).ok()?; + for (o, c) in out.iter_mut().zip(bytes.chunks_exact(4)) { + *o = f32::from_le_bytes([c[0], c[1], c[2], c[3]]); + } + Some(out) + } + + /// Conform `footage`'s audio at `rate` into `file` on a worker thread (once). + #[cfg(not(target_arch = "wasm32"))] + fn conform(&self, footage: &Footage, file: std::path::PathBuf, rate: u32) { + if !lock(&self.inner.conforming).insert(file.clone()) { + return; + } + let (pool, f) = (self.clone(), footage.clone()); + let spawned = std::thread::Builder::new().name("ec-conform-audio".into()).spawn(move || { + let frames = f.duration.to_units_floor(rate as i64).max(0) as usize; + let mut bytes = Vec::with_capacity(frames * 8); + let mut at = 0usize; + while at < frames { + let n = (frames - at).min(1 << 16); + let t = Tick(at as i64 * TICKS_PER_SECOND / rate as i64); + for v in pool.decode_audio(&f, t, n, rate) { + bytes.extend_from_slice(&v.to_le_bytes()); + } + at += n; + } + let ok = file.parent().is_some_and(|d| std::fs::create_dir_all(d).is_ok()) && { + let tmp = file.with_extension("tmp"); + std::fs::write(&tmp, &bytes).is_ok() && std::fs::rename(&tmp, &file).is_ok() + }; + if !ok { + log::warn!("media: could not write conformed audio {}", file.display()); + } + lock(&pool.inner.conforming).remove(&file); + }); + if spawned.is_err() { + lock(&self.inner.conforming).clear(); + } + } + /// `frames` stereo sample frames of `footage`'s audio starting at source time `start`, at /// `rate` Hz, interleaved (L R L R …). Mono is duplicated to both channels; channels beyond /// the first two are dropped. Silence where there is no audio (or before the media starts). + /// With a conformed audio folder, reads come from the conformed file once it is written. pub fn audio_samples(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec { - let mut out = vec![0.0; frames * 2]; if !footage.has_audio || footage.missing || rate == 0 || frames == 0 { - return out; + return vec![0.0; frames * 2]; } + #[cfg(not(target_arch = "wasm32"))] + if let Some(file) = self.conformed_path(&footage.path, rate) { + let s0 = start.to_units_floor(rate as i64); + if file.exists() && !lock(&self.inner.conforming).contains(&file) { + let skip = (-s0).max(0) as usize; + if skip >= frames { + return vec![0.0; frames * 2]; + } + if let Some(v) = Self::read_conformed(&file, s0.max(0) as usize, frames - skip) { + let mut out = vec![0.0; skip * 2]; + out.extend(v); + return out; + } + } else { + self.conform(footage, file, rate); + } + } + self.decode_audio(footage, start, frames, rate) + } + + /// [`MediaPool::audio_samples`] straight from the decoder. + fn decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> Vec { + let mut out = vec![0.0; frames * 2]; let path: Arc = footage.path.as_str().into(); let Some(src) = self.inner.source(&path) else { return out }; let s0 = start.to_units_floor(rate as i64); @@ -486,6 +587,9 @@ impl Inner { } impl FootageSource for MediaPool { + fn set_conform_folder(&self, folder: Option) { + MediaPool::set_conform_folder(self, folder); + } fn model(&self, _item: ItemId, footage: &Footage) -> Option> { if footage.missing || footage.kind != FootageKind::Model { return None; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index db1ace7d..bff830a7 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -53,6 +53,9 @@ pub trait FootageSource: Send + Sync { /// Set the decoded-frame cache budget in bytes (Settings ▸ Memory & CPU); sources without /// a cache ignore it. fn set_cache_budget(&self, _bytes: usize) {} + /// Settings ▸ Disk ▸ Conformed Audio Folder: where decoded audio is kept between reads + /// (`None` = off); sources that don't decode audio ignore it. + fn set_conform_folder(&self, _folder: Option) {} /// The decoded-frame cache budget in bytes, if the source has a cache. fn cache_budget(&self) -> Option { None @@ -242,11 +245,34 @@ pub struct RenderOpts { /// Region of interest `[x, y, w, h]` in comp pixels (viewer only): the top-level frame covers /// just this rectangle, and only its pixels are composited. pub roi: Option<[f64; 4]>, + /// Switches Affect Nested Comps (Settings ▸ General): a precomp layer's Quality and Motion + /// Blur switches also limit the layers of the nested comp (Draft/Wireframe or motion blur off + /// propagate down; they never raise a nested layer's own setting). + pub nested_switches: bool, + /// Realtime Shadows in Draft (Settings ▸ 3D): draft renders (`draft`) still cast shadows. + pub draft_shadows: bool, +} + +impl RenderOpts { + /// Whether 3D layers cast shadows in this render. + pub fn shadows(&self) -> bool { + !self.draft || self.draft_shadows + } } impl Default for RenderOpts { fn default() -> Self { - RenderOpts { scale: 1.0, motion_blur: true, guides: false, draft: false, view: None, roi: None, backend: Backend::Cpu } + RenderOpts { + scale: 1.0, + motion_blur: true, + guides: false, + draft: false, + view: None, + roi: None, + backend: Backend::Cpu, + nested_switches: true, + draft_shadows: true, + } } } @@ -272,6 +298,9 @@ pub struct Renderer<'a> { opacity_mul: f32, /// Collapsed precomp: nested 3D layers join the parent's 3D space. pub(crate) collapse3d: Option>, + /// Switches inherited from the precomp layers this comp is rendered through (see + /// [`RenderOpts::nested_switches`]): the worst quality and whether motion blur is allowed. + pub(crate) inherited: Option<(Quality, bool)>, } /// How a collapsed precomp's 3D layers are placed in the parent (see `Renderer::collapse_into`). @@ -315,9 +344,39 @@ impl<'a> Renderer<'a> { outer: None, opacity_mul: 1.0, collapse3d: None, + inherited: None, } } + /// A layer's Quality switch, limited by the precomp layers above it when + /// [`RenderOpts::nested_switches`] is on. + pub fn quality(&self, layer: &Layer) -> Quality { + let rank = |q: Quality| match q { + Quality::Best => 0, + Quality::Draft => 1, + Quality::Wireframe => 2, + }; + match self.inherited { + Some((q, _)) if rank(q) > rank(layer.switches.quality) => q, + _ => layer.switches.quality, + } + } + + /// A layer's Motion Blur switch, limited by the precomp layers above it. + pub(crate) fn layer_motion_blur(&self, layer: &Layer) -> bool { + layer.switches.motion_blur && self.inherited.is_none_or(|(_, mb)| mb) + } + + /// The renderer for the nested comp of precomp layer `layer`: its switches pass down when + /// [`RenderOpts::nested_switches`] is on. + fn nested_through(&self, layer: &Layer) -> Renderer<'a> { + let mut sub = self.nested(); + if self.opts.nested_switches { + sub.inherited = Some((self.quality(layer), self.layer_motion_blur(layer))); + } + sub + } + /// A renderer for nested work (precomps, layers read by effects): one level deeper. fn nested(&self) -> Renderer<'a> { Renderer { depth: self.depth + 1, outer: None, opacity_mul: 1.0, collapse3d: None, ..*self } @@ -671,7 +730,7 @@ impl<'a> Renderer<'a> { if self.project.comp_contains(*item, ctx.comp_id) { return None; } - let sub = self.nested(); + let sub = self.nested_through(layer); let lt = ctx.source_time(layer); // Frame blending between the nested comp's frames (time-stretched/remapped). let mode = frame_blend_mode(ctx, layer); @@ -925,7 +984,7 @@ impl<'a> Renderer<'a> { } fn sampling(&self, layer: &Layer) -> effectcraft_raster::Sampling { - if layer.switches.quality == Quality::Draft { + if self.quality(layer) == Quality::Draft { // Draft quality: no interpolation (nearest neighbour), as After Effects' Draft. effectcraft_raster::Sampling::Nearest } else if self.opts.draft { @@ -970,7 +1029,7 @@ impl<'a> Renderer<'a> { /// Motion-blur sub-samples for a layer (1 = no motion blur). fn mb_samples(&self, ctx: &EvalCtx, layer: &Layer) -> usize { - if self.opts.motion_blur && ctx.comp.enable_motion_blur && layer.switches.motion_blur { + if self.opts.motion_blur && ctx.comp.enable_motion_blur && self.layer_motion_blur(layer) { if self.opts.draft { 4 } else { ctx.comp.motion_blur_samples.clamp(2, 64) as usize } } else { 1 @@ -1056,7 +1115,7 @@ impl<'a> Renderer<'a> { /// Draw a 2D layer into `canvas`; returns the region it may have changed. fn draw_layer(&self, ctx: &EvalCtx, layer: &Layer, canvas: &mut Image, blank: bool) -> Region { let opacity = ctx.opacity(layer) as f32 * self.opacity_mul; - if layer.switches.quality == Quality::Wireframe && !layer.switches.adjustment { + if self.quality(layer) == Quality::Wireframe && !layer.switches.adjustment { self.draw_wireframe(ctx, layer, canvas); return Region::Full; } diff --git a/crates/render/src/three_d/adv/scene.rs b/crates/render/src/three_d/adv/scene.rs index 14251745..a28e3e3f 100644 --- a/crates/render/src/three_d/adv/scene.rs +++ b/crates/render/src/three_d/adv/scene.rs @@ -840,7 +840,14 @@ pub(crate) fn build(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32) { add_environment(&mut b, w, h, d, k, rot); } - super::raster::shadow_maps(&mut b.s, r.opts.draft); + if r.opts.shadows() { + super::raster::shadow_maps(&mut b.s, r.opts.draft); + } else { + // Settings ▸ 3D ▸ Realtime Shadows in Draft off: draft renders skip shadows. + for l in &mut b.s.lights { + l.shadow = -1; + } + } } // Camera matrices. let s = r.opts.scale; diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 2647fadd..747b04f9 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -162,7 +162,7 @@ pub(crate) fn camera_for(r: &Renderer, ctx: &EvalCtx) -> CameraState { /// Number of motion-blur sub-samples for a layer. fn mb_samples(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> usize { - if r.opts.motion_blur && ctx.comp.enable_motion_blur && layer.switches.motion_blur { + if r.opts.motion_blur && ctx.comp.enable_motion_blur && r.layer_motion_blur(layer) { if r.opts.draft { 4 } else { ctx.comp.motion_blur_samples.clamp(2, 64) as usize } } else { 1 @@ -335,7 +335,7 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut let mut items: Vec = run .par_iter() .enumerate() - .filter(|(_, l)| ctx.opacity(l) > 0.0 && l.switches.quality != effectcraft_project::Quality::Wireframe) + .filter(|(_, l)| ctx.opacity(l) > 0.0 && r.quality(l) != effectcraft_project::Quality::Wireframe) .flat_map_iter(|(i, l)| { // Stack order with room for a collapsed precomp's layers in between. let i = i * 1024; @@ -366,9 +366,9 @@ pub(crate) fn draw_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], canvas: &mut .collect(); } items.sort_by_key(|i| i.order); - let casters: Vec<&Item> = items.iter().chain(extra.iter()).filter(|i| i.mat.casts_shadows != 0 && !i.geos.is_empty()).collect(); + let casters: Vec<&Item> = items.iter().chain(extra.iter()).filter(|i| r.opts.shadows() && i.mat.casts_shadows != 0 && !i.geos.is_empty()).collect(); composite(canvas, &items, &lights, &casters, run[0].id.0 as u32); - for l in run.iter().filter(|l| l.switches.quality == effectcraft_project::Quality::Wireframe) { + for l in run.iter().filter(|l| r.quality(l) == effectcraft_project::Quality::Wireframe) { r.draw_layer(ctx, l, canvas, false); } } diff --git a/crates/script/src/prelude.js b/crates/script/src/prelude.js index c4f256f4..fec9d9e3 100644 --- a/crates/script/src/prelude.js +++ b/crates/script/src/prelude.js @@ -204,7 +204,7 @@ File.prototype = { return true; }, rename: function () { throw __err("File.rename is not supported"); }, - execute: function () { throw __err("scripts can't execute files"); }, + execute: function () { return JSON.parse(__file("execute", this.__path)); }, openDlg: function () { return null; }, saveDlg: function () { return null; }, }; diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs index 61b49595..c494a6bf 100644 --- a/crates/script/src/runtime.rs +++ b/crates/script/src/runtime.rs @@ -294,6 +294,11 @@ fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult 2 { arg_string(args, 2, ctx)? } else { String::new() }; let r: Result = with_active(|a| { + // File.execute(): opened with its default application through `file.executeFile` + // (Allow Scripts to Write Files gate, Warn User When Executing Files confirmation). + if op == "execute" { + return exec(&mut a.session, "file.executeFile", json!({"path": path})).map(|v| json!(v.get("opened").is_some())); + } let s = &a.session; match op.as_str() { "access" => Ok(json!(check_access(s, &path, false)?.exists())), diff --git a/crates/text/src/fonts.rs b/crates/text/src/fonts.rs index c3a60f5d..009f2522 100644 --- a/crates/text/src/fonts.rs +++ b/crates/text/src/fonts.rs @@ -48,6 +48,8 @@ pub struct FaceInfo { pub data: FaceData, /// "bundled", "system", … pub origin: &'static str, + /// The family name in the font's own (non-English) language, when it has one. + pub native_family: Option, } /// Something that can list fonts (system folders, a web font picker, a project's font folder…). @@ -121,7 +123,7 @@ impl FontSource for DirectorySource { } fn info_from(n: FaceNames, data: FaceData, origin: &'static str) -> FaceInfo { - FaceInfo { family: n.family, style: n.style, weight: n.weight, italic: n.italic, index: n.index, data, origin } + FaceInfo { family: n.family, style: n.style, weight: n.weight, italic: n.italic, index: n.index, data, origin, native_family: n.native_family } } /// A loaded (or loadable) face. @@ -300,6 +302,12 @@ pub fn all_faces() -> Vec> { db().read().unwrap_or_else(|e| e.into_inner()).faces.clone() } +/// Family names in the fonts' own languages: English family → native family (only families +/// that have one). +pub fn native_families() -> std::collections::BTreeMap { + all_faces().into_iter().filter_map(|f| Some((f.info.family.clone(), f.info.native_family.clone()?))).collect() +} + /// Families with their style names, sorted by family; styles in weight order. pub fn families() -> Vec<(String, Vec)> { let mut m: std::collections::BTreeMap> = Default::default(); diff --git a/crates/text/src/sfnt.rs b/crates/text/src/sfnt.rs index bb00d515..72e34506 100644 --- a/crates/text/src/sfnt.rs +++ b/crates/text/src/sfnt.rs @@ -13,6 +13,9 @@ pub struct FaceNames { pub full_name: String, pub weight: u16, pub italic: bool, + /// The family name in another language than English (CJK, Arabic… fonts), when the font + /// has one (Settings ▸ Type ▸ Show Font Names in English off shows it). + pub native_family: Option, } fn be16(b: &[u8], o: usize) -> Option { @@ -75,6 +78,7 @@ fn read_face(r: &mut R, off: u64, index: u32) -> Option(r: &mut R, off: u64, index: u32) -> Option Option { + let count = be16(b, 2)? as usize; + let storage = be16(b, 4)? as usize; + for i in 0..count { + let r = 6 + i * 12; + let (plat, enc, lang, nid, len, off) = + (be16(b, r)?, be16(b, r + 2)?, be16(b, r + 4)?, be16(b, r + 6)?, be16(b, r + 8)? as usize, be16(b, r + 10)? as usize); + if nid != id || plat != 3 || !(enc == 1 || enc == 10) || lang == 0x409 { + continue; + } + let raw = b.get(storage + off..storage + off + len)?; + let u: Vec = raw.chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + let s = String::from_utf16_lossy(&u); + if !s.trim().is_empty() { + return Some(s); + } + } + None } /// A name-table string by id, preferring Windows Unicode English, then any Unicode, then Mac Roman. diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index 6659c77d..483b3e0f 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -47,11 +47,13 @@ pub struct AudioOutput { pub device: String, pub left: u32, pub right: u32, + /// Preview Sample Rate: devices that support it are opened at this rate. + pub rate: u32, } impl AudioOutput { pub fn from_prefs(p: &effectcraft_engine::prefs::Prefs) -> AudioOutput { - AudioOutput { device: p.audio.output_device.clone(), left: p.audio.output_left, right: p.audio.output_right } + AudioOutput { device: p.audio.output_device.clone(), left: p.audio.output_left, right: p.audio.output_right, rate: p.audio.preview_sample_rate } } } @@ -166,6 +168,9 @@ pub struct AudioPlayback { impl AudioPlayback { /// Start mixing `comp` over `span` into `device`. + /// `mix_rate`: Settings ▸ Audio ▸ Preview Sample Rate, the rate the comp's audio is mixed + /// at (resampled to the device's rate when they differ; 0 = the device's rate). + #[allow(clippy::too_many_arguments)] pub fn start( mut device: Box, src: RenderSource, @@ -174,8 +179,10 @@ impl AudioPlayback { wa: Tick, wb: Tick, looping: bool, + mix_rate: u32, ) -> Result { let rate = device.sample_rate().max(8000); + let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) }; let span = PlaySpan::new(start, wa, wb, looping, rate); let feed = Arc::new(AudioFeed::default()); let stop = Arc::new(AtomicBool::new(false)); @@ -183,7 +190,7 @@ impl AudioPlayback { let mut cursor = span.start; let prime = (rate as usize / 20).div_ceil(BLOCK); for _ in 0..prime { - cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); } let thread = { let (feed, stop) = (feed.clone(), stop.clone()); @@ -194,7 +201,7 @@ impl AudioPlayback { let target = rate as usize / 5; while !stop.load(Ordering::Relaxed) { if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { - cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); } else { std::thread::sleep(std::time::Duration::from_millis(4)); } @@ -232,7 +239,7 @@ impl Drop for AudioPlayback { /// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at /// the work-area end). -fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32) -> i64 { +fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32, mix: u32) -> i64 { let mut cursor = cursor; if cursor >= span.wb { if !span.looping { @@ -242,11 +249,39 @@ fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpa cursor = span.wa; } let n = ((span.wb - cursor) as usize).min(BLOCK); - let buf = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, sample_time(cursor, rate), n, rate); + let t = sample_time(cursor, rate); + let buf = if mix == rate { + effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, n, rate) + } else { + let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1; + let b = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, m, mix); + resample_stereo(&b, mix, rate, n) + }; feed.push(&buf); cursor + n as i64 } +/// Linear resampling of interleaved stereo from `from` Hz to `n` frames at `to` Hz. +pub fn resample_stereo(src: &[f32], from: u32, to: u32, n: usize) -> Vec { + let frames = src.len() / 2; + let mut out = Vec::with_capacity(n * 2); + if frames == 0 { + out.resize(n * 2, 0.0); + return out; + } + let step = from as f64 / to as f64; + for i in 0..n { + let x = i as f64 * step; + let a = (x.floor() as usize).min(frames - 1); + let b = (a + 1).min(frames - 1); + let f = (x - a as f64) as f32; + for c in 0..2 { + out.push(src[a * 2 + c] * (1.0 - f) + src[b * 2 + c] * f); + } + } + out +} + /// VU meter state for the Audio panel: smoothed level with a falling decay and peak hold, in /// dBFS per channel. #[derive(Clone, Copy, Debug)] diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index cd052c3e..e2373c9a 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -776,6 +776,10 @@ pub fn draw_group_chrome( } } else { let strip = Rect::from_min_size(g.rect.min, vec2(g.rect.width(), t.tab_h)); + if t.gradients { + // Settings ▸ Appearance ▸ Use Gradients. + crate::theme::gradient_rect(&painter, strip.shrink2(vec2(t.radius, 0.0)), t.grad_top(t.panel_bg), t.panel_bg); + } let mut x = strip.min.x + 12.0; let text_y = strip.min.y + 16.0; for (i, p) in g.panels.iter().enumerate() { diff --git a/crates/ui-egui/src/dock_ui.rs b/crates/ui-egui/src/dock_ui.rs index 44900918..3b222570 100644 --- a/crates/ui-egui/src/dock_ui.rs +++ b/crates/ui-egui/src/dock_ui.rs @@ -29,6 +29,30 @@ impl EffectcraftApp { out } + /// Settings ▸ Appearance ▸ Use Label Color for Related Tabs: the Composition and Timeline + /// tabs carry their comp's label colour, Effect Controls and Properties the layer's. + fn label_tab_marks(&self, ui: &egui::Ui) { + if !self.session.prefs.appearance.use_label_color_for_tabs { + return; + } + let Some(cid) = self.session.active_comp_id() else { return }; + let comp_label = self.session.project.item(cid).map(|i| i.label); + let layer_label = self.session.active_comp().and_then(|c| self.session.state.selected_layers.first().and_then(|l| c.layer(*l))).map(|l| l.label); + for (p, label) in [ + (PanelKind::Composition, comp_label), + (PanelKind::Timeline, comp_label), + (PanelKind::EffectControls, layer_label), + (PanelKind::Properties, layer_label), + ] { + let (Some(label), Some(e)) = (label, self.auto.find(&format!("panel.tab.{}", p.id()))) else { continue }; + if label == effectcraft_engine::color::Label::None { + continue; + } + let r = Rect::from_min_size(pos2(e.rect[0] + 2.0, e.rect[1] + 9.0), vec2(4.0, (e.rect[3] - 16.0).max(6.0))); + ui.painter().rect_filled(r, 1.0, self.tokens.label(label)); + } + } + /// Edit the full layout (docked tree + floating groups) with a [`Layout`] operation. pub fn edit_layout(&mut self, f: impl FnOnce(&mut Layout) -> bool) -> bool { let mut l = Layout { root: self.ui.dock.clone(), floating: self.ui.floating.clone() }; @@ -98,6 +122,7 @@ impl EffectcraftApp { for g in &groups { actions.extend(dock::draw_group_chrome(ui, g, self.ui.focused, &t, &mut self.auto, &title)); } + self.label_tab_marks(ui); if maximized.is_none() { self.ui.dock = dock; } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 0976c47a..e4ec95bb 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -16,6 +16,7 @@ pub mod icons; pub mod menus; pub mod native_menu; pub mod panels; +pub mod prefs_live; pub mod state; pub mod theme; pub mod widgets; @@ -250,7 +251,7 @@ impl EffectcraftApp { ctx.all_styles_mut(|s| s.interaction.tooltip_delay = if tips { 0.5 } else { f32::INFINITY }); self.ui.cache_when_idle = p.previews.cache_frames_when_idle; self.ui.viewer.fast_preview = p.previews.fast_previews; - self.frames.set_budget(p.preview_cache_bytes()); + self.frames.set_budget(p.cache_budgets(self.session.sys_memory).preview); } /// Change settings from the UI (applied next frame and saved). @@ -434,6 +435,8 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, + nested_switches: self.session.prefs.general.switches_affect_nested_comps, + draft_shadows: self.session.prefs.three_d.realtime_shadows, } } @@ -486,7 +489,8 @@ impl EffectcraftApp { } let out = audio::AudioOutput::from_prefs(&self.session.prefs); let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f(&out)) else { return }; - match audio::AudioPlayback::start(dev, self.render_source(), cid, t, c.work_area.0, c.work_area.1, self.ui.preview_loop) { + let mix = self.session.prefs.audio.preview_sample_rate; + match audio::AudioPlayback::start(dev, self.render_source(), cid, t, c.work_area.0, c.work_area.1, self.ui.preview_loop, mix) { Ok(a) => self.audio = Some(a), Err(e) => log::warn!("audio preview: {e}"), } @@ -758,6 +762,7 @@ impl EffectcraftApp { self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); + prefs_live::frame(self, &ctx); if let Some(rx) = self.command_inbox.take() { while let Ok(id) = rx.try_recv() { if let Err(e) = menus::invoke(self, &ctx, &id, json!({})) { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index e48bb514..89a2353b 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -746,6 +746,37 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu app.show_panel(PanelKind::Layer); Value::Null } + "app.confirmExecute" => { + // Settings ▸ Scripting ▸ Warn User When Executing Files. + let path = p.get("path").and_then(Value::as_str).unwrap_or_default().to_string(); + crate::panels::forms::form( + app, + "A script wants to open a file", + "file.executeFile", + json!({"path": path, "confirmed": true}), + vec![crate::panels::forms::Field::text("path", "Open", &path)], + ); + Value::Null + } + "app.showReport" => { + let title = p.get("title").and_then(Value::as_str).unwrap_or("Report").to_string(); + let text = p.get("text").and_then(Value::as_str).unwrap_or_default().to_string(); + crate::panels::dialogs::info(app, &title, &text); + Value::Null + } + "window.assignWorkspaceShortcut" => { + let mut q = p.clone(); + q["workspace"] = json!(app.ui.workspace); + return run_engine(app, ctx, "window.assignWorkspaceShortcut", q); + } + "file.interpretFootage" => { + // Import ▸ Interpret Unlabeled Alpha As: Ask User. + if let Some(i) = p.get("item").and_then(Value::as_u64) { + app.session.state.project_selection = vec![effectcraft_engine::project::ItemId(i)]; + } + crate::panels::dialogs::open_form(app, "file.interpretFootage", &json!({})); + Value::Null + } "effect.manage" | "anim.browsePresets" => { app.show_panel(PanelKind::EffectsPresets); Value::Null @@ -786,6 +817,8 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), + "file.saveCopyAsXml" => ("path", Ask::Save("Untitled Project.ecprojx")), + "keys.rpfCameraImport" => ("path", Ask::Open(&["json", "csv", "txt"])), _ => return None, }; if params.get(key).is_some() @@ -862,6 +895,15 @@ pub(crate) fn entry_label(app: &EffectcraftApp, e: &MenuEntry) -> String { match e.command.as_str() { "edit.undo" => app.session.history.undo.last().map(|u| format!("Undo {}", u.0)).unwrap_or_else(|| "Can't Undo".into()), "edit.redo" => app.session.history.redo.last().map(|u| format!("Redo {}", u.0)).unwrap_or_else(|| "Can't Redo".into()), + // Window ▸ Layer: the layer open in the Layer panel. + "window.panel" if e.params.get("panel").and_then(Value::as_str) == Some("layer") => { + let name = app + .ui + .layer_panel + .and_then(|id| app.session.active_comp().and_then(|c| c.layer(effectcraft_engine::project::LayerId(id)))) + .map(|l| l.name.clone()); + format!("Layer: {}", name.unwrap_or_else(|| "(none)".into())) + } _ => effectcraft_engine::menus::entry_label(&app.session, e), } } @@ -1126,29 +1168,38 @@ fn menu_nodes(app: &mut EffectcraftApp, ui: &mut egui::Ui, nodes: &[MenuNode], c MenuNode::Separator => { ui.separator(); } - MenuNode::Submenu { label, children } if label == "Open Recent" => { - // File ▸ Open Recent: the recent projects (Settings), then the static entries. - ui.menu_button((gutter(false), label.as_str()), |ui| { - ui.set_min_width(320.0); - let recent = app.session.prefs.recent_projects.clone(); - if recent.is_empty() { - ui.add_enabled(false, egui::Button::new((gutter(false), "No Recent Projects"))); + MenuNode::Dynamic { name } => { + // Recent projects / footage / presets, undo history, shortcut slots, viewers. + let ws = app.ui.workspace.clone(); + let (entries, empty) = effectcraft_engine::menus::dynamic(&app.session, name, &dyn_ctx(&ws)); + if entries.is_empty() + && let Some(e) = empty + { + ui.add_enabled(false, egui::Button::new((gutter(false), e))); + } + for (i, e) in entries.iter().enumerate() { + let tip = match name.as_str() { + "recentProjects" => app.session.prefs.recent_projects.get(i).cloned(), + "recentFootage" => app.session.prefs.recent_footage.get(i).cloned(), + "recentPresets" => app.session.prefs.recent_presets.get(i).cloned(), + _ => None, + }; + let r = ui.add_enabled(entry_enabled(app, e), egui::Button::new((gutter(false), e.label.as_str()))); + let r = match &tip { + Some(t) => r.on_hover_text(t), + None => r, + }; + app.auto.add(&format!("menu.{name}.{i}"), r.rect, &e.label); + if r.clicked() { + *clicked = Some((e.command.clone(), e.params.clone())); + ui.close(); } - for (i, path) in recent.iter().enumerate() { - let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); - let r = ui.add(egui::Button::new((gutter(false), name))).on_hover_text(path); - app.auto.add(&format!("menu.openRecent.{i}"), r.rect, path); - if r.clicked() { - *clicked = Some(("file.openRecent".into(), json!({"index": i}))); - ui.close(); - } - } - ui.separator(); - menu_nodes(app, ui, children, clicked); - }); + } } MenuNode::Submenu { label, children } => { - ui.menu_button((gutter(false), label.as_str()), |ui| { + let ws = app.ui.workspace.clone(); + let shown = effectcraft_engine::menus::submenu_label(&app.session, label, &dyn_ctx(&ws)); + ui.menu_button((gutter(false), shown.as_str()), |ui| { ui.set_min_width(if children.len() > 30 { 200.0 } else { 240.0 }); // Long submenus (Blending Mode, effect categories) scroll instead of running // off the screen. @@ -1166,6 +1217,11 @@ fn menu_nodes(app: &mut EffectcraftApp, ui: &mut egui::Ui, nodes: &[MenuNode], c } } +/// Frontend state for dynamic menus (the current workspace). +pub(crate) fn dyn_ctx(workspace: &str) -> effectcraft_engine::menus::DynCtx<'_> { + effectcraft_engine::menus::DynCtx { workspace: Some(workspace) } +} + /// The check-mark column every menu row reserves (like macOS / After Effects menus). fn gutter(checked: bool) -> egui::Atom<'static> { use egui::AtomExt; diff --git a/crates/ui-egui/src/native_menu.rs b/crates/ui-egui/src/native_menu.rs index ef5070b3..42c3aacf 100644 --- a/crates/ui-egui/src/native_menu.rs +++ b/crates/ui-egui/src/native_menu.rs @@ -134,28 +134,41 @@ pub fn build(app: &EffectcraftApp) -> NativeMenu { let id = format!("{path}.{i}"); match n { MenuNode::Separator => out.push(NativeNode::Separator), + MenuNode::Dynamic { name } => { + // Recent projects / footage / presets, undo history, shortcut slots, viewers. + let (entries, empty) = effectcraft_engine::menus::dynamic(&app.session, name, &crate::menus::dyn_ctx(&app.ui.workspace)); + if entries.is_empty() + && let Some(e) = empty + { + out.push(NativeNode::Item(NativeItem { + id: format!("{id}.none"), + label: e.to_string(), + command: String::new(), + params: Value::Null, + shortcut: None, + accelerator: None, + enabled: false, + checked: None, + })); + } + for (k, e) in entries.iter().enumerate() { + out.push(NativeNode::Item(NativeItem { + id: format!("{id}.d{k}"), + label: e.label.clone(), + command: e.command.clone(), + params: e.params.clone(), + shortcut: None, + accelerator: None, + enabled: crate::menus::entry_enabled(app, e), + checked: None, + })); + } + } MenuNode::Submenu { label, children } => { let mut kids = vec![]; - if label == "Open Recent" { - for (k, p) in app.session.prefs.recent_projects.iter().enumerate() { - let name = std::path::Path::new(p).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(p.clone()); - kids.push(NativeNode::Item(NativeItem { - id: format!("{id}.r{k}"), - label: name, - command: "file.openRecent".into(), - params: json!({"index": k}), - shortcut: None, - accelerator: None, - enabled: true, - checked: None, - })); - } - if !kids.is_empty() { - kids.push(NativeNode::Separator); - } - } rec(app, children, &id, &mut kids); - out.push(NativeNode::Submenu { label: label.clone(), children: kids }); + let shown = effectcraft_engine::menus::submenu_label(&app.session, label, &crate::menus::dyn_ctx(&app.ui.workspace)); + out.push(NativeNode::Submenu { label: shown, children: kids }); } MenuNode::Item(e) => { let role = match e.command.as_str() { @@ -272,6 +285,11 @@ pub fn state_key(app: &EffectcraftApp) -> u64 { s.journal.len().hash(&mut h); s.journal.last().map(|j| &j.0).hash(&mut h); s.prefs.recent_projects.hash(&mut h); + (&s.prefs.recent_footage, &s.prefs.recent_presets).hash(&mut h); + s.history.undo.iter().for_each(|(l, _)| l.hash(&mut h)); + s.state.open_comps.hash(&mut h); + s.active_comp_id().and_then(|c| s.state.views3d.get(&c)).map(|v| format!("{:?}", v.current)).hash(&mut h); + s.shortcuts().keys.len().hash(&mut h); s.keymaps.active.hash(&mut h); let v = &app.ui.viewer; (v.rulers, v.guides, v.snap_guides, v.lock_guides, v.grid, v.snap_grid, v.show_layer_controls, v.pasteboard).hash(&mut h); @@ -340,7 +358,10 @@ mod tests { fn commands(n: &[NativeNode], out: &mut Vec<(String, String)>) { for x in n { match x { - NativeNode::Item(i) if i.command != "file.openRecent" => out.push((i.command.clone(), i.params.to_string())), + // Dynamic entries (recent files, History, viewers…) are not tree entries. + NativeNode::Item(i) if !i.id.rsplit('.').next().is_some_and(|l| l.starts_with('d') || l == "none") => { + out.push((i.command.clone(), i.params.to_string())) + } NativeNode::Predefined { command, .. } if !command.is_empty() => out.push((command.clone(), "null".into())), NativeNode::Submenu { children, .. } => commands(children, out), _ => {} @@ -431,7 +452,9 @@ mod tests { app.session.execute("layer.setSwitch", json!({"switch": "lock"})).unwrap(); assert_ne!(state_key(&app), k0); let spec1 = build(&app); - assert_eq!(spec0.structure(), spec1.structure(), "state changes don't rebuild the menu"); + // The new undo step joins Edit ▸ History (a dynamic submenu); nothing else is rebuilt. + let history = |s: &NativeMenu| s.items().into_iter().filter(|i| i.command == "edit.history").count(); + assert_eq!(history(&spec1), history(&spec0) + 1); let lock = spec1.items().into_iter().find(|i| i.command == "layer.setSwitch" && i.params["switch"] == "lock").unwrap().checked; assert_eq!(lock, Some(true)); assert_ne!(undo(&spec0), undo(&spec1)); diff --git a/crates/ui-egui/src/panels/expr_editor.rs b/crates/ui-egui/src/panels/expr_editor.rs new file mode 100644 index 00000000..1efd6876 --- /dev/null +++ b/crates/ui-egui/src/panels/expr_editor.rs @@ -0,0 +1,394 @@ +//! The expression editor of the Timeline, following Settings ▸ Scripting & Expressions: +//! Font Size, Syntax Highlighting, Line Numbers, Auto-complete (Tab accepts), Bracket Matching +//! and Word Wrap; and the Composition panel's expression error banner (Show Expression Error +//! Banner). + +use effectcraft_engine::prefs::Scripting; +use egui::text::{LayoutJob, TextFormat}; +use egui::{Color32, FontId, Rect, Stroke, pos2, vec2}; + +use crate::theme::Tokens; + +/// Syntax colours. +const KEYWORD: Color32 = Color32::from_rgb(0xc6, 0x8a, 0xe6); +const NUMBER: Color32 = Color32::from_rgb(0xe0, 0xb4, 0x6c); +const STRING: Color32 = Color32::from_rgb(0x9c, 0xd0, 0x7a); +const COMMENT: Color32 = Color32::from_rgb(0x80, 0x88, 0x90); +const API: Color32 = Color32::from_rgb(0x6c, 0xb8, 0xf0); +const BRACKET_BG: Color32 = Color32::from_rgb(0x4a, 0x55, 0x6a); + +const KEYWORDS: &[&str] = &[ + "var", + "let", + "const", + "function", + "return", + "if", + "else", + "for", + "while", + "do", + "break", + "continue", + "new", + "true", + "false", + "null", + "undefined", + "this", + "typeof", + "in", + "of", + "switch", + "case", + "default", + "try", + "catch", + "throw", +]; + +/// Expression language names offered by auto-complete (public Expression Language Reference +/// vocabulary). +pub const COMPLETIONS: &[&str] = &[ + "thisComp", + "thisLayer", + "thisProperty", + "time", + "value", + "index", + "layer", + "effect", + "mask", + "content", + "transform", + "position", + "anchorPoint", + "scale", + "rotation", + "opacity", + "orientation", + "xRotation", + "yRotation", + "zRotation", + "wiggle", + "loopOut", + "loopIn", + "loopOutDuration", + "loopInDuration", + "valueAtTime", + "velocityAtTime", + "speedAtTime", + "linear", + "ease", + "easeIn", + "easeOut", + "clamp", + "random", + "gaussRandom", + "seedRandom", + "noise", + "degreesToRadians", + "radiansToDegrees", + "timeToFrames", + "framesToTime", + "timeToTimecode", + "length", + "lookAt", + "toComp", + "fromComp", + "toWorld", + "fromWorld", + "sourceRectAtTime", + "sourceText", + "numKeys", + "key", + "nearestKey", + "inPoint", + "outPoint", + "startTime", + "width", + "height", + "frameDuration", + "marker", + "Math", + "comp", + "footage", + "posterizeTime", + "add", + "sub", + "mul", + "div", + "normalize", + "cross", + "dot", + "hslToRgb", + "rgbToHsl", + "createPath", + "points", + "inTangents", + "outTangents", + "isClosed", +]; + +/// The bracket matching the one at or just before char index `cursor`: (open, close) char +/// indices. +pub fn matching_bracket(text: &str, cursor: usize) -> Option<(usize, usize)> { + let chars: Vec = text.chars().collect(); + let pairs = [('(', ')'), ('[', ']'), ('{', '}')]; + for at in [cursor, cursor.wrapping_sub(1)] { + let Some(&c) = chars.get(at) else { continue }; + if let Some(&(o, cl)) = pairs.iter().find(|(o, _)| *o == c) { + let mut depth = 0; + for (i, &x) in chars.iter().enumerate().skip(at) { + if x == o { + depth += 1; + } else if x == cl { + depth -= 1; + if depth == 0 { + return Some((at, i)); + } + } + } + } + if let Some(&(o, cl)) = pairs.iter().find(|(_, cl)| *cl == c) { + let mut depth = 0; + for i in (0..=at).rev() { + if chars[i] == cl { + depth += 1; + } else if chars[i] == o { + depth -= 1; + if depth == 0 { + return Some((i, at)); + } + } + } + } + } + None +} + +/// The identifier being typed before char index `cursor`, and auto-complete candidates for it. +pub fn completions(text: &str, cursor: usize) -> (String, Vec<&'static str>) { + let chars: Vec = text.chars().collect(); + let end = cursor.min(chars.len()); + let mut start = end; + while start > 0 && (chars[start - 1].is_alphanumeric() || chars[start - 1] == '_') { + start -= 1; + } + let word: String = chars[start..end].iter().collect(); + if word.chars().count() < 2 || word.chars().next().is_some_and(|c| c.is_ascii_digit()) { + return (word, vec![]); + } + let c: Vec<&str> = COMPLETIONS.iter().copied().filter(|w| w.starts_with(word.as_str()) && *w != word).take(6).collect(); + (word, c) +} + +/// Colour runs for JavaScript: (byte start, byte end, colour). +pub fn highlight(text: &str, base: Color32) -> Vec<(usize, usize, Color32)> { + let b = text.as_bytes(); + let mut out = vec![]; + let mut i = 0; + while i < b.len() { + let c = b[i]; + let start = i; + let col = if c == b'/' && b.get(i + 1) == Some(&b'/') { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + COMMENT + } else if c == b'/' && b.get(i + 1) == Some(&b'*') { + i += 2; + while i < b.len() && !(b[i] == b'*' && b.get(i + 1) == Some(&b'/')) { + i += 1; + } + i = (i + 2).min(b.len()); + COMMENT + } else if c == b'"' || c == b'\'' || c == b'`' { + i += 1; + while i < b.len() && b[i] != c { + i += if b[i] == b'\\' { 2 } else { 1 }; + } + i = (i + 1).min(b.len()); + STRING + } else if c.is_ascii_digit() { + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'.') { + i += 1; + } + NUMBER + } else if c.is_ascii_alphabetic() || c == b'_' || c == b'$' { + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_' || b[i] == b'$') { + i += 1; + } + let w = &text[start..i]; + if KEYWORDS.contains(&w) { + KEYWORD + } else if COMPLETIONS.contains(&w) { + API + } else { + base + } + } else { + // Advance one whole character. + i += text[i..].chars().next().map_or(1, char::len_utf8); + base + }; + match out.last_mut() { + Some((_, e, c)) if *c == col && *e == start => *e = i, + _ => out.push((start, i, col)), + } + } + out +} + +/// Byte index of char index `ci`. +fn byte_of(text: &str, ci: usize) -> usize { + text.char_indices().nth(ci).map(|(b, _)| b).unwrap_or(text.len()) +} + +/// The layout job of the editor text: syntax colours, the matched brackets' background, wrap. +fn layout(text: &str, font: FontId, base: Color32, p: &Scripting, brackets: Option<(usize, usize)>, wrap: f32) -> LayoutJob { + let mut job = LayoutJob::default(); + let runs = if p.syntax_highlighting { highlight(text, base) } else { vec![(0, text.len(), base)] }; + let marks: Vec = brackets.filter(|_| p.bracket_matching).map(|(a, b)| vec![byte_of(text, a), byte_of(text, b)]).unwrap_or_default(); + for (s, e, col) in runs { + // Split runs at the marked brackets. + let mut cuts = vec![s]; + for m in &marks { + if *m >= s && *m < e { + cuts.push(*m); + cuts.push((*m + 1).min(e)); + } + } + cuts.push(e); + cuts.sort_unstable(); + cuts.dedup(); + for w in cuts.windows(2) { + let bg = if marks.contains(&w[0]) { BRACKET_BG } else { Color32::TRANSPARENT }; + job.append(&text[w[0]..w[1]], 0.0, TextFormat { font_id: font.clone(), color: col, background: bg, ..Default::default() }); + } + } + job.wrap.max_width = if p.word_wrap { wrap } else { f32::INFINITY }; + job +} + +/// The editor: returns the response; `buf` is edited in place. +pub fn editor(ui: &mut egui::Ui, id: egui::Id, buf: &mut String, rect: Rect, p: &Scripting, color: Color32, rows: usize, t: &Tokens) -> egui::Response { + let size = (p.editor_font_size.clamp(8, 40) as f32) * 0.9; + let font = Tokens::mono(size); + // Line numbers in a gutter on the left. + let gutter = if p.line_numbers { (size * 0.62) * (buf.lines().count().max(1).to_string().len() as f32 + 1.0) + 6.0 } else { 0.0 }; + let er = Rect::from_min_max(pos2(rect.min.x + gutter, rect.min.y), rect.max); + let cursor = egui::TextEdit::load_state(ui.ctx(), id).and_then(|s| s.cursor.char_range()).map(|r| r.primary.index.0); + let brackets = cursor.and_then(|c| matching_bracket(buf, c)); + let pc = p.clone(); + let mut layouter = move |ui: &egui::Ui, text: &dyn egui::TextBuffer, wrap: f32| { + let job = layout(text.as_str(), font.clone(), color, &pc, brackets, wrap); + ui.fonts_mut(|f| f.layout_job(job)) + }; + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(er)); + // Auto-complete: Tab accepts the first candidate (consumed before the editor sees it). + let mut accepted = None; + if p.auto_complete + && child.memory(|m| m.has_focus(id)) + && let Some(c) = cursor + { + let (word, cands) = completions(buf, c); + if let Some(first) = cands.first() + && child.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Tab)) + { + let at = byte_of(buf, c); + buf.insert_str(at, &first[word.len()..]); + accepted = Some(c + first.chars().count() - word.chars().count()); + } + } + let out = egui::TextEdit::multiline(buf) + .id(id) + .font(Tokens::mono(size)) + .desired_width(er.width()) + .desired_rows(rows) + .frame(egui::Frame::NONE) + .lock_focus(true) + .layouter(&mut layouter) + .show(&mut child); + if let Some(c) = accepted { + let mut st = out.state.clone(); + st.cursor.set_char_range(Some(egui::text::CCursorRange::one(egui::text::CCursor::new(c)))); + st.store(ui.ctx(), id); + } + let painter = ui.painter(); + if p.line_numbers { + let g = out.galley.clone(); + let mut line = 1; + let mut new_line = true; + for row in &g.rows { + if new_line { + let y = out.galley_pos.y + row.rect().center().y; + painter.text(pos2(rect.min.x + gutter - 6.0, y), egui::Align2::RIGHT_CENTER, line.to_string(), Tokens::mono(size * 0.85), t.text_faint); + line += 1; + } + new_line = row.ends_with_newline; + } + painter.line_segment([pos2(rect.min.x + gutter - 3.0, rect.min.y), pos2(rect.min.x + gutter - 3.0, rect.max.y)], Stroke::new(1.0, t.separator)); + } + // Candidate list under the editor. + if p.auto_complete + && out.response.has_focus() + && let Some(c) = out.cursor_range.map(|r| r.primary.index.0) + { + let (_, cands) = completions(buf, c); + if !cands.is_empty() { + let pos = out.galley_pos + out.galley.pos_from_cursor(egui::text::CCursor::new(c)).left_bottom().to_vec2() + vec2(0.0, 2.0); + egui::Area::new(id.with("complete")).order(egui::Order::Tooltip).fixed_pos(pos).show(ui.ctx(), |ui| { + egui::Frame::popup(ui.style()).show(ui, |ui| { + for (i, w) in cands.iter().enumerate() { + let txt = if i == 0 { format!("{w} Tab") } else { w.to_string() }; + ui.label(egui::RichText::new(txt).font(Tokens::mono(size * 0.9)).color(if i == 0 { t.text } else { t.text_dim })); + } + }); + }); + } + } + out.response.response +} + +/// Settings ▸ Scripting & Expressions ▸ Show Expression Error Banner: the first failing +/// expression of the comp at the current time, in a banner along the bottom of the viewer +/// (click it to reveal the property). +pub fn error_banner(app: &mut crate::EffectcraftApp, ui: &egui::Ui, area: Rect) { + if !app.session.prefs.scripting.error_banner { + return; + } + let Some(cid) = app.session.active_comp_id() else { return }; + let key = (app.session.revision, cid.0, app.session.time().0); + let id = egui::Id::new("expr-error-banner"); + let cached: Option<((u64, u64, i64), Vec)> = ui.ctx().data(|d| d.get_temp(id)); + let errors = match cached { + Some((k, e)) if k == key => e, + _ => { + let e = app.session.expression_errors(cid); + ui.ctx().data_mut(|d| d.insert_temp(id, (key, e.clone()))); + e + } + }; + let Some(first) = errors.first() else { return }; + let layer = first["layer"].as_u64().unwrap_or(0); + let lname = app.session.active_comp().and_then(|c| c.layer(effectcraft_engine::project::LayerId(layer))).map(|l| l.name.clone()).unwrap_or_default(); + let msg = format!( + "⚠ Expression error{}: {} ▸ {}: {}", + if errors.len() > 1 { format!(" (1 of {})", errors.len()) } else { String::new() }, + lname, + first["name"].as_str().unwrap_or(""), + first["error"].as_str().unwrap_or("").lines().next().unwrap_or("") + ); + let r = Rect::from_min_max(pos2(area.min.x, area.max.y - 22.0), area.max); + let p = ui.painter(); + p.rect_filled(r, 0.0, Color32::from_rgb(0xf0, 0xc0, 0x30)); + p.text(pos2(r.min.x + 8.0, r.center().y), egui::Align2::LEFT_CENTER, msg, Tokens::ui(11.5), Color32::BLACK); + app.auto.add("viewer.expressionErrorBanner", r, "Expression error banner"); + if ui.interact(r, id.with("click"), egui::Sense::click()).clicked() { + let p = serde_json::json!({"props": [{"layer": layer, "prop": first["prop"]}]}); + if let Err(e) = crate::menus::invoke(app, ui.ctx(), "timeline.revealProps", p) { + app.ui.status = e; + } + } +} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3cf2dae1..68b13a0b 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -7,6 +7,7 @@ pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; pub mod effects_presets; +pub mod expr_editor; pub mod flowchart; pub mod forms; pub mod fx_widgets; diff --git a/crates/ui-egui/src/panels/text_panels.rs b/crates/ui-egui/src/panels/text_panels.rs index edb1132f..5ab0a73a 100644 --- a/crates/ui-egui/src/panels/text_panels.rs +++ b/crates/ui-egui/src/panels/text_panels.rs @@ -98,9 +98,8 @@ pub fn character(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { widgets::open_popup(ui, pop); } app.auto.add("character.font", fr, "Font family"); - let fams: Vec = effectcraft_engine::text_families(); - if let Some(i) = widgets::popup_menu(ui, pop, fr.left_bottom(), &fams, fams.iter().position(|f| *f == doc.font)) { - actions.push(json!({"font": fams[i]})); + if let Some(f) = font_popup(app, ui, pop, fr.left_bottom(), &doc.font) { + actions.push(json!({"font": f})); } y += 28.0; let sr = Rect::from_min_size(pos2(x0, y), vec2(w, 22.0)); @@ -349,20 +348,29 @@ pub fn paragraph(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } y += 3.0 * 28.0 + 6.0; - // Direction and composer popups. - let dr = Rect::from_min_size(pos2(x0, y), vec2(w / 2.0 - 4.0, 22.0)); - let dirs = vec!["Left-to-Right Text".to_string(), "Right-to-Left Text".to_string()]; - let dpop = egui::Id::new("para-direction-pop"); + // Direction and composer popups. Settings ▸ Type ▸ Text Engine: the South Asian and Middle + // Eastern engine offers the paragraph direction and the World-Ready composers (the Latin + // engine shows the direction only for right-to-left text already set). + let world_ready = app.session.prefs.type_.text_engine == "southAsian"; let rtl = doc.direction == Direction::Rtl; - if widgets::dropdown(ui, dr, &dirs[rtl as usize], &t, egui::Id::new("para-direction")).clicked() && enabled { - widgets::open_popup(ui, dpop); - } - app.auto.add("paragraph.direction", dr, "Text direction"); - if let Some(i) = widgets::popup_menu(ui, dpop, dr.left_bottom(), &dirs, Some(rtl as usize)) { - actions.push(json!({"direction": if i == 1 { "rtl" } else { "ltr" }})); + if world_ready || rtl { + let dr = Rect::from_min_size(pos2(x0, y), vec2(w / 2.0 - 4.0, 22.0)); + let dirs = vec!["Left-to-Right Text".to_string(), "Right-to-Left Text".to_string()]; + let dpop = egui::Id::new("para-direction-pop"); + if widgets::dropdown(ui, dr, &dirs[rtl as usize], &t, egui::Id::new("para-direction")).clicked() && enabled { + widgets::open_popup(ui, dpop); + } + app.auto.add("paragraph.direction", dr, "Text direction"); + if let Some(i) = widgets::popup_menu(ui, dpop, dr.left_bottom(), &dirs, Some(rtl as usize)) { + actions.push(json!({"direction": if i == 1 { "rtl" } else { "ltr" }})); + } } let cr = Rect::from_min_size(pos2(x0 + w / 2.0, y), vec2(w / 2.0, 22.0)); - let comps = vec!["Every-line Composer".to_string(), "Single-line Composer".to_string()]; + let comps = if world_ready { + vec!["World-Ready Every-line Composer".to_string(), "World-Ready Single-line Composer".to_string()] + } else { + vec!["Every-line Composer".to_string(), "Single-line Composer".to_string()] + }; let cpop = egui::Id::new("para-composer-pop"); let single = doc.composer == Composer::SingleLine; if widgets::dropdown(ui, cr, &comps[single as usize], &t, egui::Id::new("para-composer")).clicked() && enabled { @@ -390,6 +398,56 @@ pub fn paragraph(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// The font menu: recent fonts first, names in English or the fonts' own language, and a +/// "Sample" preview in each font (Settings ▸ Type). +fn font_popup(app: &EffectcraftApp, ui: &mut egui::Ui, id: egui::Id, pos: egui::Pos2, current: &str) -> Option { + if !ui.data(|d| d.get_temp::(id.with("open")).unwrap_or(false)) { + return None; + } + let rows = effectcraft_engine::font_menu(&app.session.prefs); + let preview = app.session.prefs.type_.font_preview; + let t = app.tokens; + let mut chosen = None; + let area = egui::Area::new(id.with("area")).order(egui::Order::Foreground).fixed_pos(pos).show(ui.ctx(), |ui| { + egui::Frame::popup(ui.style()).show(ui, |ui| { + ui.set_min_width(if preview { 300.0 } else { 180.0 }); + egui::ScrollArea::vertical().max_height(420.0).show(ui, |ui| { + for r in &rows { + if r.family.is_empty() { + ui.separator(); + continue; + } + let resp = ui.selectable_label(r.family == current, &r.display); + if preview && ui.is_rect_visible(resp.rect) { + let key = egui::Id::new(("font-preview", &r.family)); + let lines: std::sync::Arc>> = match ui.data(|d| d.get_temp(key)) { + Some(l) => l, + None => { + let l = std::sync::Arc::new(effectcraft_engine::font_preview(&r.family, 14.0)); + ui.data_mut(|d| d.insert_temp(key, l.clone())); + l + } + }; + let o = pos2(resp.rect.max.x - 120.0, resp.rect.center().y + 5.0); + for poly in lines.iter() { + let pts: Vec = poly.iter().map(|p| o + vec2(p[0], p[1])).collect(); + ui.painter().add(egui::Shape::line(pts, egui::Stroke::new(1.0, t.text_dim))); + } + } + if resp.clicked() { + chosen = Some(r.family.clone()); + } + } + }); + }); + }); + let outside = ui.input(|i| i.pointer.any_pressed()) && !area.response.contains_pointer() && !area.response.hovered(); + if chosen.is_some() || outside || ui.input(|i| i.key_pressed(egui::Key::Escape)) { + ui.data_mut(|d| d.insert_temp(id.with("open"), false)); + } + chosen +} + /// Align panel: align selected layers to the composition (or to the selection). pub fn align(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let t = app.tokens; diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5eb4caa6..dc6b98e2 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -1510,15 +1510,17 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let er = Rect::from_min_max(pos2(graph_x0 + 8.0, r.min.y + 3.0), pos2(rect.max.x - 14.0, r.max.y - 3.0)); let buf_id = egui::Id::new(("expr-buf", uid)); let mut buf: String = ctx.data(|d| d.get_temp(buf_id)).unwrap_or_else(|| ex.text.clone()); - let mut child = ui.new_child(egui::UiBuilder::new().max_rect(er)); - let resp = child.add( - egui::TextEdit::multiline(&mut buf) - .id(egui::Id::new(("expr-edit", uid))) - .font(Tokens::mono(11.5)) - .text_color(if ex.enabled { expr_col } else { t.text_dim }) - .desired_width(er.width()) - .desired_rows((*lines).clamp(1, 8)) - .frame(egui::Frame::NONE), + // Settings ▸ Scripting & Expressions ▸ Expressions Editor. + let sp = app.session.prefs.scripting.clone(); + let resp = super::expr_editor::editor( + ui, + egui::Id::new(("expr-edit", uid)), + &mut buf, + er, + &sp, + if ex.enabled { expr_col } else { t.text_dim }, + (*lines).clamp(1, 8), + &t, ); app.auto.add(&format!("timeline.prop.{uid}.expression"), er, &prop.name); let commit = resp.has_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter) && (i.modifiers.command || i.modifiers.ctrl)); @@ -1647,7 +1649,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let ks = sel_keys.contains(&kref); let icon = k.icon(); let c = pos2(x, cy); - icons::keyframe(&gp, c, 11.0, icon.left, icon.right, if ks { t.keyframe_selected } else { t.keyframe }, Color32::from_black_alpha(200)); + // Keyframe colour labels (Edit ▸ Label on selected keyframes). + let fill = match effectcraft_engine::color::Label::ALL.get(k.label as usize).filter(|_| k.label > 0) { + Some(l) if ks => t.label(*l).lerp_to_gamma(Color32::WHITE, 0.45), + Some(l) => t.label(*l), + None if ks => t.keyframe_selected, + None => t.keyframe, + }; + icons::keyframe(&gp, c, 11.0, icon.left, icon.right, fill, Color32::from_black_alpha(200)); let kr = Rect::from_center_size(c, vec2(12.0, 14.0)); let kresp = ui.interact(kr, egui::Id::new(("key", uid, k.time.0)), Sense::click_and_drag()); app.auto.add(&format!("timeline.key.{uid}.{}", fr.frame_at(ct)), kr, &format!("{} key", prop.name)); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index c9dd70db..2057df99 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -423,12 +423,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let sub = g.grid_subdivisions.max(1) as f32; let sstep = step / sub; let draw_minor = sstep >= 4.0; + // Settings ▸ Grids & Guides ▸ Grid Style (lines, dashed lines or dots). + let style = g.grid_style.clone(); let mut i = 0u32; let mut gx = comp_rect.min.x; while gx <= comp_rect.max.x { let is_major = i.is_multiple_of(g.grid_subdivisions.max(1)); if is_major || draw_minor { - painter.line_segment([pos2(gx, comp_rect.min.y), pos2(gx, comp_rect.max.y)], if is_major { major } else { minor }); + styled_line(&painter, [pos2(gx, comp_rect.min.y), pos2(gx, comp_rect.max.y)], if is_major { major } else { minor }, &style); } gx += sstep; i += 1; @@ -438,7 +440,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { while gy <= comp_rect.max.y { let is_major = i.is_multiple_of(g.grid_subdivisions.max(1)); if is_major || draw_minor { - painter.line_segment([pos2(comp_rect.min.x, gy), pos2(comp_rect.max.x, gy)], if is_major { major } else { minor }); + styled_line(&painter, [pos2(comp_rect.min.x, gy), pos2(comp_rect.max.x, gy)], if is_major { major } else { minor }, &style); } gy += sstep; i += 1; @@ -451,13 +453,15 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if app.ui.viewer.guides { let [r, gg, b] = hex_rgb(&app.session.prefs.grids.guide_color).unwrap_or([0x3c, 0xc8, 0xf0]); let stroke = Stroke::new(1.0, Color32::from_rgb(r, gg, b)); + // Settings ▸ Grids & Guides ▸ Guide Style. + let style = app.session.prefs.grids.guide_style.clone(); for g in &comp.guides { if g.vertical { let x = comp_rect.min.x + g.position as f32 * zoom; - painter.line_segment([pos2(x, area.min.y), pos2(x, area.max.y)], stroke); + styled_line(&painter, [pos2(x, area.min.y), pos2(x, area.max.y)], stroke, &style); } else { let y = comp_rect.min.y + g.position as f32 * zoom; - painter.line_segment([pos2(area.min.x, y), pos2(area.max.x, y)], stroke); + styled_line(&painter, [pos2(area.min.x, y), pos2(area.max.x, y)], stroke, &style); } } } @@ -501,6 +505,28 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let hover = ui.input(|i| i.pointer.hover_pos()); ov::draw_paths(app, &painter, &map, &ectx, l, col, &sel_vertices, pen, hover, &mut vertex_hits, &mut paths); } + // Settings ▸ Previews ▸ Show Internal Wireframes: outlines of the layers inside a + // selected collapsed precomp. + if app.session.prefs.previews.show_internal_wireframes + && l.switches.collapse + && let effectcraft_engine::project::LayerSource::Comp { item } = &l.source + && let Some(nc) = app.session.project.comp(*item) + { + let (outer, _) = l2c(&ectx, l); + let nctx = EvalCtx::new(&app.session.project, *item, nc, ectx.source_time(l)); + for nl in nc.layers.iter().filter(|nl| nl.is_active_at(nctx.time) && !nl.is_3d()) { + if let Some((_, q, _)) = layer_quad(&nctx, nl) { + let sq: Vec = q + .iter() + .map(|p| { + let c = outer.apply(gv2(p[0], p[1])); + map.to_screen([c.x, c.y]) + }) + .collect(); + painter.add(egui::Shape::closed_line(sq, Stroke::new(1.0, col.gamma_multiply(0.6)))); + } + } + } let Some((m, q, _)) = layer_quad(&ectx, l) else { // Cameras, lights, empty layers: just the anchor. if let Some(tr) = l.transform() { @@ -549,6 +575,41 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } + // Settings ▸ 3D: Extended Viewer (3D layers outlined beyond the comp frame) and Show 3D + // Reference Axes (world X/Y/Z in the corner, turning with the view). + let has_3d = comp.layers.iter().any(|l| l.is_3d() && l.is_active_at(time)); + if has_3d && app.session.prefs.three_d.extended_viewer { + for l in comp.layers.iter().filter(|l| l.is_3d() && l.is_active_at(time) && l.has_video()) { + if let Some((_, q, _)) = layer_quad(&ectx, l) { + let sq: Vec = q.iter().map(|p| map.to_screen(*p)).collect(); + let outside = sq.iter().any(|p| !comp_rect.contains(*p)); + if outside { + painter.add(egui::Shape::closed_line(sq, Stroke::new(1.0, t.label(l.label).gamma_multiply(0.55)))); + } + } + } + } + if has_3d && app.session.prefs.three_d.show_reference_axes { + let pc = cam_state(&ectx).projection(cw as f64, ch as f64); + let o3 = effectcraft_engine::geom::vec3(cw as f64 / 2.0, ch as f64 / 2.0, 0.0); + let o = pc.apply(o3); + let corner = pos2(area.min.x + 34.0, area.max.y - 34.0); + for (axis, col, name) in [ + (effectcraft_engine::geom::vec3(1.0, 0.0, 0.0), Color32::from_rgb(0xe0, 0x50, 0x50), "X"), + (effectcraft_engine::geom::vec3(0.0, 1.0, 0.0), Color32::from_rgb(0x60, 0xd0, 0x60), "Y"), + (effectcraft_engine::geom::vec3(0.0, 0.0, 1.0), Color32::from_rgb(0x50, 0x8c, 0xf0), "Z"), + ] { + let e = pc.apply(o3 + axis * 10.0); + let d = vec2((e.x - o.x) as f32, (e.y - o.y) as f32); + let d = if d.length() > 1e-4 { d.normalized() * 22.0 } else { vec2(0.0, 0.0) }; + painter.arrow(corner, d, Stroke::new(1.5, col)); + painter.text(corner + d * 1.25, Align2::CENTER_CENTER, name, Tokens::ui(10.0), col); + } + app.auto.add("viewer.referenceAxes", Rect::from_center_size(corner, vec2(56.0, 56.0)), "3D reference axes"); + } + + super::expr_editor::error_banner(app, ui, area); + // Track points of the current track (Tracker panel) on its layer, while that layer is selected. let mut track_hits = vec![]; let track_cur = super::tracker::viewer_track(app); @@ -1437,6 +1498,26 @@ fn empty_state(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } /// `#rrggbb` → sRGB bytes. +/// A grid or guide line in the Grids & Guides style: `lines`, `dashed` or `dots`. +pub(crate) fn styled_line(painter: &egui::Painter, pts: [Pos2; 2], stroke: Stroke, style: &str) { + match style { + "dashed" => { + painter.add(egui::Shape::dashed_line(&pts, stroke, 6.0, 4.0)); + } + "dots" => { + painter.add(egui::Shape::dotted_line(&pts, stroke.color, 4.0, stroke.width.max(1.0) * 0.75)); + } + _ => { + painter.line_segment(pts, stroke); + } + } +} + +/// Texture filtering for Viewer Zoom Quality: More Accurate = bilinear, Faster = nearest. +pub(crate) fn zoom_texture_options(smooth: bool) -> egui::TextureOptions { + if smooth { egui::TextureOptions::LINEAR } else { egui::TextureOptions::NEAREST } +} + pub(crate) fn hex_rgb(s: &str) -> Option<[u8; 3]> { let h = s.trim().trim_start_matches('#'); if h.len() != 6 || !h.is_ascii() { @@ -1450,14 +1531,17 @@ pub(crate) fn hex_rgb(s: &str) -> Option<[u8; 3]> { /// straight from their wgpu texture (registered with egui-wgpu, no readback). fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames::FrameKey, img: crate::frames::FrameImage) { use crate::frames::FrameImage; + // Settings ▸ Previews ▸ Viewer Zoom Quality. + let smooth = app.session.prefs.viewer_zoom_smooth(); + let opts = zoom_texture_options(smooth); match img { FrameImage::Cpu(img) => { match &mut app.viewer_tex { Some((tex, k)) if tex.size() == img.size => { - tex.set((*img).clone(), egui::TextureOptions::LINEAR); + tex.set((*img).clone(), opts); *k = key; } - _ => app.viewer_tex = Some((ctx.load_texture("viewer-frame", (*img).clone(), egui::TextureOptions::LINEAR), key)), + _ => app.viewer_tex = Some((ctx.load_texture("viewer-frame", (*img).clone(), opts), key)), } app.viewer_shown = app.viewer_tex.as_ref().map(|(t, k)| (t.id(), *k)); app.viewer_image = Some(img); @@ -1467,12 +1551,13 @@ fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames: let Some(rs) = &app.wgpu else { return }; let view = f.texture.create_view(&Default::default()); let mut ren = rs.renderer.write(); + let filter = if smooth { eframe::wgpu::FilterMode::Linear } else { eframe::wgpu::FilterMode::Nearest }; let id = match &app.viewer_native { Some((id, _, _)) => { - ren.update_egui_texture_from_wgpu_texture(&rs.device, &view, eframe::wgpu::FilterMode::Linear, *id); + ren.update_egui_texture_from_wgpu_texture(&rs.device, &view, filter, *id); *id } - None => ren.register_native_texture(&rs.device, &view, eframe::wgpu::FilterMode::Linear), + None => ren.register_native_texture(&rs.device, &view, filter), }; drop(ren); app.viewer_native = Some((id, key, f)); diff --git a/crates/ui-egui/src/panels/viewer_overlays.rs b/crates/ui-egui/src/panels/viewer_overlays.rs index 08cc32b5..95181d7c 100644 --- a/crates/ui-egui/src/panels/viewer_overlays.rs +++ b/crates/ui-egui/src/panels/viewer_overlays.rs @@ -222,7 +222,9 @@ pub(crate) fn motion_path(app: &mut EffectcraftApp, painter: &egui::Painter, map let c = p2c.apply(gv2(v[0], v[1])); map.to_screen([c.x, c.y]) }; - let (lt0, lt1) = (pos.keys[0].time, pos.keys[pos.keys.len() - 1].time); + // Settings ▸ Composition ▸ Motion Path: all keyframes, none, or a window around the CTI. + let times: Vec = pos.keys.iter().map(|k| k.time).collect(); + let Some((lt0, lt1)) = app.session.prefs.motion_path_span(×, l.layer_time(ectx.time)) else { return }; let fd = ectx.comp.frame_duration(); let mut pts = vec![]; let mut lt = lt0; @@ -238,6 +240,9 @@ pub(crate) fn motion_path(app: &mut EffectcraftApp, painter: &egui::Painter, map } let sel = &app.session.state.selected_keys; for (i, k) in pos.keys.iter().enumerate() { + if k.time < lt0 || k.time > lt1 { + continue; + } let v = k.value.as_vec3(); let s = to_scr(v); let (tin, tout) = effectcraft_engine::keyframe::spatial_tangents(&pos.keys, i); diff --git a/crates/ui-egui/src/panels/viewer_tools.rs b/crates/ui-egui/src/panels/viewer_tools.rs index 79688e19..ac61a4a2 100644 --- a/crates/ui-egui/src/panels/viewer_tools.rs +++ b/crates/ui-egui/src/panels/viewer_tools.rs @@ -282,11 +282,11 @@ pub(crate) fn proportional_grid(app: &EffectcraftApp, painter: &egui::Painter, c let (nx, ny) = (g.proportional_horizontal.max(1), g.proportional_vertical.max(1)); for i in 1..nx { let x = comp_rect.min.x + comp_rect.width() * i as f32 / nx as f32; - painter.line_segment([pos2(x, comp_rect.min.y), pos2(x, comp_rect.max.y)], s); + super::viewer::styled_line(painter, [pos2(x, comp_rect.min.y), pos2(x, comp_rect.max.y)], s, &g.grid_style); } for i in 1..ny { let y = comp_rect.min.y + comp_rect.height() * i as f32 / ny as f32; - painter.line_segment([pos2(comp_rect.min.x, y), pos2(comp_rect.max.x, y)], s); + super::viewer::styled_line(painter, [pos2(comp_rect.min.x, y), pos2(comp_rect.max.x, y)], s, &g.grid_style); } } @@ -319,6 +319,11 @@ fn transformed(img: &egui::ColorImage, ch: Channel, colorized: bool, stops: f32) pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: &egui::Painter, comp_rect: Rect, cid: ItemId, ectx: &EvalCtx) { let opts = app.session.state.viewer.clone(); let snap = showing_snapshot(app, ctx); + // Settings ▸ Video ▸ Mirror on Computer Monitor off: playback goes to Video Preview only. + if crate::prefs_live::main_viewer_hidden(app) && !snap { + painter.text(comp_rect.center(), Align2::CENTER_CENTER, "Playing on Video Preview", Tokens::ui(13.0), Color32::from_gray(150)); + return; + } if opts.fast_previews == FastPreviews::Wireframe && !snap { let zoom = comp_rect.width() / ectx.comp.width.max(1) as f32; let map = ViewerMap { origin: comp_rect.min, zoom, comp: [ectx.comp.width as f32, ectx.comp.height as f32], area: comp_rect }; @@ -375,6 +380,7 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: return; } let Some(src) = app.viewer_image.clone() else { return }; + let zoom_opts = super::viewer::zoom_texture_options(app.session.prefs.viewer_zoom_smooth()); let key = { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -385,7 +391,7 @@ pub(crate) fn draw_frame(app: &mut EffectcraftApp, ctx: &egui::Context, painter: let tex = match ctx.data(|d| d.get_temp::<(u64, egui::TextureHandle)>(id)) { Some((k, tex)) if k == key => tex, _ => { - let tex = ctx.load_texture("viewer-display", transformed(&src, opts.channel, opts.colorized, opts.exposure), egui::TextureOptions::LINEAR); + let tex = ctx.load_texture("viewer-display", transformed(&src, opts.channel, opts.colorized, opts.exposure), zoom_opts); ctx.data_mut(|d| d.insert_temp(id, (key, tex.clone()))); tex } diff --git a/crates/ui-egui/src/panels/waveform.rs b/crates/ui-egui/src/panels/waveform.rs index f9bf00db..ab3fbd79 100644 --- a/crates/ui-egui/src/panels/waveform.rs +++ b/crates/ui-egui/src/panels/waveform.rs @@ -47,10 +47,25 @@ pub fn summary(app: &EffectcraftApp, ctx: &egui::Context, item: ItemId) -> Optio let ItemKind::Footage(f) = &app.session.project.item(item)?.kind else { return None }; m.insert(item.0, None); let (f, footage, map, ctx) = (f.clone(), app.session.footage.clone(), app.waveforms.map.clone(), ctx.clone()); + // Settings ▸ Disk ▸ Database and Cache Folder: summaries are kept on disk between sessions. + let cache = app.session.media_cache_folder(); std::thread::Builder::new() .name("ec-waveform".into()) .spawn(move || { - let peaks = effectcraft_engine::render::audio::footage_peaks(footage.as_ref(), item, &f, f.duration, BINS_PER_SEC); + use effectcraft_engine::media_cache; + let file = cache.and_then(|d| media_cache::peaks_path(&d, &f.path, BINS_PER_SEC)); + let peaks = match file.as_deref().and_then(media_cache::load_peaks) { + Some(p) => p, + None => { + let p = effectcraft_engine::render::audio::footage_peaks(footage.as_ref(), item, &f, f.duration, BINS_PER_SEC); + if let Some(file) = &file + && let Err(e) = media_cache::store_peaks(file, &p) + { + log::warn!("media cache: {}: {e}", file.display()); + } + p + } + }; if let Ok(mut m) = map.lock() { m.insert(item.0, Some(Arc::new(Summary { bins_per_sec: BINS_PER_SEC, peaks }))); } diff --git a/crates/ui-egui/src/prefs_live.rs b/crates/ui-egui/src/prefs_live.rs new file mode 100644 index 00000000..9e730196 --- /dev/null +++ b/crates/ui-egui/src/prefs_live.rs @@ -0,0 +1,141 @@ +//! Settings the desktop frontend applies every frame: dropped files (Import ▸ Default Drag +//! Import As), the memory watch (Memory & CPU ▸ RAM Reserved / Reduce Cache Size When System Is +//! Low on Memory), Startup ▸ Show Home Screen When Opening a Project, and Video ▸ Video Preview +//! output (a second window showing the composition frame, Mercury Transmit-style). + +use egui::{Color32, Rect}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::frames::FrameKey; + +/// Run once per frame, before the panels draw. +pub fn frame(app: &mut EffectcraftApp, ctx: &egui::Context) { + dropped_files(app, ctx); + memory_watch(app, ctx); + home_on_open(app, ctx); + video_preview(app, ctx); +} + +/// Files dropped on the window are imported (layered files as Default Drag Import As says). +fn dropped_files(app: &mut EffectcraftApp, ctx: &egui::Context) { + let paths: Vec = + ctx.input(|i| i.raw.dropped_files.iter().map(|f| f.path().to_string_lossy().to_string()).filter(|p| std::path::Path::new(p).is_absolute()).collect()); + if paths.is_empty() { + return; + } + let (proj, rest): (Vec, Vec) = paths.into_iter().partition(|p| { + let l = p.to_ascii_lowercase(); + l.ends_with(".ecproj") || l.ends_with(".ecprojx") + }); + if let Some(p) = proj.first() + && let Err(e) = crate::menus::invoke(app, ctx, "file.open", json!({"path": p})) + { + app.ui.status = e; + } + if !rest.is_empty() { + let params = json!({"paths": rest, "drag": true, "importAs": app.session.prefs.drag_import_as()}); + if let Err(e) = crate::menus::invoke(app, ctx, "file.import", params) { + app.ui.status = e; + } + } +} + +/// Re-read the system's free memory every few seconds (cache budgets follow it). +fn memory_watch(app: &mut EffectcraftApp, ctx: &egui::Context) { + let now = ctx.input(|i| i.time); + let id = egui::Id::new("prefs-memory-watch"); + let last = ctx.data(|d| d.get_temp::(id)); + if last.is_some_and(|t| now - t < 10.0) { + return; + } + ctx.data_mut(|d| d.insert_temp(id, now)); + // The first frame only records the time: tests and short runs don't query the system. + if last.is_some() { + app.session.memory_tick(); + } +} + +/// Startup ▸ Show Home Screen When Opening a Project. +fn home_on_open(app: &mut EffectcraftApp, ctx: &egui::Context) { + let id = egui::Id::new("prefs-last-project"); + let path = app.session.path.clone(); + let last = ctx.data(|d| d.get_temp::>(id)); + ctx.data_mut(|d| d.insert_temp(id, path.clone())); + let Some(last) = last else { return }; + let opened = app.session.journal.last().is_some_and(|(c, _)| matches!(c.as_str(), "file.open" | "file.openRecent" | "file.revert")); + if path.is_some() && path != last && opened && app.session.prefs.startup.show_home_on_open_project { + app.ui.start_screen = true; + } +} + +/// Video ▸ Mirror on Computer Monitor off: during playback the Composition panel leaves the +/// frame to the Video Preview window. +pub fn main_viewer_hidden(app: &EffectcraftApp) -> bool { + let v = &app.session.prefs.video; + v.enable_output && !v.mirror_on_monitor && app.playback.playing +} + +/// Whether the Video Preview window shows now (Disable Video Output When in Background). +pub fn video_preview_active(app: &EffectcraftApp, focused: bool) -> bool { + let v = &app.session.prefs.video; + v.enable_output && (focused || !v.disable_when_background) +} + +/// Video ▸ Enable Video Preview Output: a second native window with the current composition +/// frame, letterboxed on black. Video Device "Full Screen" opens it full screen (move it to the +/// external display first); Video Output During Playback off holds the last frame while playing. +fn video_preview(app: &mut EffectcraftApp, ctx: &egui::Context) { + let focused = ctx.input(|i| i.focused); + if !video_preview_active(app, focused) { + return; + } + let v = app.session.prefs.video.clone(); + let tex_id = egui::Id::new("video-preview-tex"); + let update = !app.playback.playing || v.output_during_playback; + let key = app.viewer_shown.as_ref().map(|(_, k)| *k); + let cur: Option<(FrameKey, egui::TextureHandle)> = ctx.data(|d| d.get_temp(tex_id)); + let tex = match (update, key, cur) { + (true, Some(k), Some((ck, t))) if ck == k => Some(t), + (true, Some(k), prev) => match app.viewer_pixels() { + Some(img) => { + let opts = crate::panels::viewer::zoom_texture_options(app.session.prefs.viewer_zoom_smooth()); + let t = match prev { + Some((_, mut t)) => { + t.set((*img).clone(), opts); + t + } + None => ctx.load_texture("video-preview", (*img).clone(), opts), + }; + ctx.data_mut(|d| d.insert_temp(tex_id, (k, t.clone()))); + Some(t) + } + None => prev.map(|p| p.1), + }, + (_, _, prev) => prev.map(|p| p.1), + }; + let title = + format!("Video Preview — {}", app.session.active_comp_id().and_then(|c| app.session.project.item(c)).map(|i| i.name.clone()).unwrap_or_default()); + let mut builder = egui::ViewportBuilder::default().with_title(title).with_inner_size([960.0, 540.0]); + if v.device == "fullscreen" { + builder = builder.with_fullscreen(true); + } + let mut closed = false; + ctx.show_viewport_immediate(egui::ViewportId::from_hash_of("effectcraft-video-preview"), builder, |vctx, _| { + egui::CentralPanel::default().frame(egui::Frame::NONE.fill(Color32::BLACK)).show(vctx, |ui| { + let area = ui.max_rect(); + if let Some(t) = &tex { + let [w, h] = t.size().map(|x| x.max(1) as f32); + let s = (area.width() / w).min(area.height() / h); + let r = Rect::from_center_size(area.center(), egui::vec2(w * s, h * s)); + ui.painter().image(t.id(), r, Rect::from_min_max(egui::pos2(0.0, 0.0), egui::pos2(1.0, 1.0)), Color32::WHITE); + } + }); + if vctx.input(|i| i.viewport().close_requested()) { + closed = true; + } + }); + if closed && let Err(e) = app.set_pref("video.enableOutput", Value::Bool(false)) { + app.ui.status = e; + } +} diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index c32ecb24..eb02bf48 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -75,6 +75,29 @@ pub struct Tokens { pub gap: f32, pub tab_h: f32, pub row_h: f32, + /// Settings ▸ Appearance ▸ Use Gradients: panel tab strips and buttons get a soft vertical + /// gradient (off: flat fills). + pub gradients: bool, +} + +/// A vertical gradient fill (`top` → `bottom`) over `rect`. +pub fn gradient_rect(painter: &egui::Painter, rect: egui::Rect, top: Color32, bottom: Color32) { + let mut m = egui::Mesh::default(); + m.colored_vertex(rect.left_top(), top); + m.colored_vertex(rect.right_top(), top); + m.colored_vertex(rect.right_bottom(), bottom); + m.colored_vertex(rect.left_bottom(), bottom); + m.add_triangle(0, 1, 2); + m.add_triangle(0, 2, 3); + painter.add(egui::Shape::mesh(m)); +} + +impl Tokens { + /// The gradient top colour for a fill (a little lighter), or the fill itself when gradients + /// are off. + pub fn grad_top(&self, c: Color32) -> Color32 { + if self.gradients { c.lerp_to_gamma(Color32::WHITE, 0.06) } else { c } + } } impl Tokens { @@ -123,6 +146,7 @@ impl Tokens { gap: 4.0, tab_h: 30.0, row_h: 22.0, + gradients: true, }; match kind { ThemeKind::Dark => dark, @@ -192,6 +216,7 @@ impl Tokens { let [r, g, b] = p.label_rgb(*l); t.labels[i] = Color32::from_rgb(r, g, b); } + t.gradients = p.appearance.use_gradients; t } diff --git a/crates/ui-egui/tests/ui_text_edit.rs b/crates/ui-egui/tests/ui_text_edit.rs index 2c943740..2c9af608 100644 --- a/crates/ui-egui/tests/ui_text_edit.rs +++ b/crates/ui-egui/tests/ui_text_edit.rs @@ -134,7 +134,9 @@ fn double_click_enters_editing_and_selects_words() { assert_eq!(edited(&h).1, "alpha gamma"); key(&mut h, Key::Z, Modifiers::COMMAND); assert_eq!(edited(&h).1, "alpha beta"); - // Paragraph panel: alignment applies to the edited paragraph. + // Paragraph panel: alignment applies to the edited paragraph (the South Asian and Middle + // Eastern text engine shows the direction popup too). + h.state_mut().session.prefs.type_.text_engine = "southAsian".into(); h.state_mut().show_panel(PanelKind::Paragraph); h.run_steps(3); let r = rect(&h, "paragraph.right"); diff --git a/docs/preferences.md b/docs/preferences.md index cb05229e..57400a31 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -70,71 +70,79 @@ Commands (CLI, MCP, control channel): - `threeD.defaultRenderer`: Default 3D Renderer - `labels.N.name` / `labels.N.color`: the 16 label names and colours, used by the Label menu, the timeline, project panel, render queue and viewer handles +- `general.switchesAffectNestedComps`: a precomp layer's Quality (Draft / Wireframe) and Motion + Blur switches limit the layers of the nested comp (viewer, Render Queue, `render`) +- `general.preserveConstantVertexCount`: adding or deleting mask / shape path vertices at one + keyframe does the same on every other keyframe (added vertices split the matching segment) +- `general.syncTimeRelatedItems`: moving the current time moves it in the comps nested in this + one and in the comps that nest it (through the precomp layers' timing) +- `general.expressionPickWhipCompact`: off, the pick whip writes match names + (`thisComp.layer("A")("transform")("opacity")`) +- `general.createSplitLayersAbove`: where Edit ▸ Split Layer puts the new layer +- `startup.showHomeOnOpenProject`: the Home screen comes up after File ▸ Open / Open Recent +- `composition.motionPath`, `composition.motionPathSeconds`, `composition.motionPathKeyframes`: + how much of a position motion path the viewer draws (all, none, N seconds or N keyframes + around the current time) +- `previews.showInternalWireframes`: outlines of the layers inside a selected collapsed precomp +- `previews.zoomQuality`: Faster = nearest-neighbour viewer scaling, More Accurate = bilinear +- `appearance.useLabelColorForTabs`: Composition / Timeline tabs show the comp's label colour, + Effect Controls / Properties the layer's +- `appearance.cycleMaskColors`: new masks cycle through the mask colours (off: all the first) +- `appearance.useGradients`: soft gradient on panel tab strips +- `grids.gridStyle`, `grids.guideStyle`: lines, dashed lines or dots (grid, proportional grid, + guides); `grids.proportionalHorizontal` / `grids.proportionalVertical`: View ▸ Show + Proportional Grid divisions +- `type.textEngine`: South Asian and Middle Eastern shows the paragraph direction and the + World-Ready composers in the Paragraph panel +- `type.fontPreview`: a "Sample" preview in each font of the Character panel's font menu +- `type.recentFonts`: how many recently used fonts head the font menu +- `type.fontNamesInEnglish`: off, fonts with a native-language family name show it +- `import.reportMissingFrames`: gaps in an image sequence's numbering are reported on import +- `import.unlabeledAlpha`: alpha of TGA / TIFF / movie footage (Ask opens Interpret Footage; + Guess takes premultiplied for movies, straight for stills) +- `import.dragImportAs`: layered files dropped on the window import as footage or a comp +- `export.defaultOutputFolder`: where relative Render Queue outputs go +- `export.segmentSequences`, `export.segmentSequenceFiles`: sequences split into numbered + folders of N files; `export.segmentMovies`, `export.segmentMovieMb`: movies split into + numbered files of about N MB (cut at the frame count the output's data rate gives) +- `export.appendBitsToName`: `Comp 1.png` → `Comp 1_16bpc.png` +- `audio.previewSampleRate`: the rate previews are mixed at (and the device is opened at, when + it supports it) +- `disk.diskCacheEnabled`, `disk.diskCacheMaxGb`, `disk.diskCacheFolder`: the persistent disk + cache of processed layers +- `disk.mediaCacheFolder`: audio waveform summaries kept between sessions (`Peaks/`) +- `disk.conformedMediaFolder`: decoded (conformed) footage audio, written once per file and + sample rate and read back instead of decoding again +- `memory.ramReservedGb`: the cache budgets together leave this much physical memory free +- `memory.reduceCacheWhenLow`: cache budgets halve while the system is low on memory (checked + every 10 s by the desktop app) +- `video.enableOutput`, `video.device`, `video.outputDuringPlayback`, `video.mirrorOnMonitor`, + `video.disableWhenBackground`: Video Preview, a second window (or full screen on the display + it is on) showing the composition frame, Mercury Transmit-style +- `threeD.showReferenceAxes`: world X/Y/Z axes in the viewer corner of comps with 3D layers +- `threeD.extendedViewer`: 3D layers reaching beyond the comp frame are outlined on the + pasteboard +- `threeD.realtimeShadows`: draft (Fast Previews) renders keep 3D shadows +- `scripting.allowScriptsWriteFiles`: the scripting file / network gate (and `File.execute()`) +- `scripting.warnExecutingFiles`: `File.execute()` asks before opening a file +- `scripting.editorFontSize`, `scripting.syntaxHighlighting`, `scripting.lineNumbers`, + `scripting.autoComplete` (Tab accepts), `scripting.bracketMatching`, `scripting.wordWrap`: + the Timeline's expression editor +- `scripting.errorBanner`: a banner along the bottom of the Composition panel names the first + failing expression (click: reveal it) -### Not wired yet (TODO) +### Display-only settings -These rows are in the dialog for After Effects parity and are stored, but nothing reads them yet. -`prefs.pages` reports them with `"live": false`. A test keeps this list in step with the schema. +These rows are in the dialog for After Effects parity and are stored, but have no EffectCraft +behaviour to change. `prefs.pages` reports them with `"live": false`; a test keeps this list in +step with the schema. -- `general.switchesAffectNestedComps`: Switches Affect Nested Comps -- `general.preserveConstantVertexCount`: Preserve Constant Vertex and Feather Point Count when Editing Masks -- `general.syncTimeRelatedItems`: Synchronize Time of All Related Items -- `general.expressionPickWhipCompact`: Expression Pick Whip Writes Compact English -- `general.createSplitLayersAbove`: Create Split Layers Above Original Layer -- `general.useSystemColorPicker`: Use System Color Picker -- `startup.showHomeOnOpenProject`: Show Home Screen When Opening a Project -- `composition.motionPath`: Motion Path -- `composition.motionPathSeconds`: Seconds -- `composition.motionPathKeyframes`: Keyframes -- `composition.hardwareAcceleratePanels`: Hardware Accelerate Composition, Layer and Footage Panels -- `previews.showInternalWireframes`: Show Internal Wireframes -- `previews.zoomQuality`: Viewer Zoom Quality -- `appearance.useLabelColorForTabs`: Use Label Color for Related Tabs -- `appearance.cycleMaskColors`: Cycle Mask Colors -- `appearance.useGradients`: Use Gradients -- `grids.gridStyle`: Style -- `grids.proportionalHorizontal`: Horizontal -- `grids.proportionalVertical`: Vertical -- `grids.guideStyle`: Style -- `type.textEngine`: Text Engine -- `type.fontPreview`: Show Font Preview -- `type.recentFonts`: Number of Recent Fonts to Display -- `type.fontNamesInEnglish`: Show Font Names in English -- `import.reportMissingFrames`: Report Missing Frames -- `import.unlabeledAlpha`: Interpret Unlabeled Alpha As -- `import.dragImportAs`: Default Drag Import As -- `export.defaultOutputFolder`: Default Output Folder -- `export.segmentSequences`: Segment Sequences -- `export.segmentSequenceFiles`: Files per Segment -- `export.segmentMovies`: Segment Movie Files -- `export.segmentMovieMb`: Segment Size -- `export.appendBitsToName`: Append Bit Depth to File Name -- `audio.previewSampleRate`: Preview Sample Rate -- `disk.diskCacheEnabled`: Enable Disk Cache -- `disk.diskCacheMaxGb`: Maximum Disk Cache Size -- `disk.diskCacheFolder`: Disk Cache Folder -- `disk.mediaCacheFolder`: Database and Cache Folder -- `disk.conformedMediaFolder`: Conformed Audio Folder -- `memory.ramReservedGb`: RAM Reserved for Other Applications -- `memory.reduceCacheWhenLow`: Reduce Cache Size When System Is Low on Memory -- `video.enableOutput`: Enable Video Preview Output -- `video.device`: Video Device -- `video.outputDuringPlayback`: Video Output During Playback -- `video.mirrorOnMonitor`: Mirror on Computer Monitor -- `video.disableWhenBackground`: Disable Video Output When in Background -- `threeD.showReferenceAxes`: Show 3D Reference Axes -- `threeD.extendedViewer`: Extended Viewer -- `threeD.realtimeShadows`: Realtime Shadows in Draft -- `scripting.allowScriptsWriteFiles`: Allow Scripts to Write Files and Access Network -- `scripting.warnExecutingFiles`: Warn User When Executing Files -- `scripting.enableJsDebugger`: Enable JavaScript Debugger -- `scripting.editorFontSize`: Font Size -- `scripting.syntaxHighlighting`: Syntax Highlighting -- `scripting.lineNumbers`: Line Numbers -- `scripting.autoComplete`: Auto-complete -- `scripting.bracketMatching`: Bracket Matching -- `scripting.wordWrap`: Word Wrap -- `scripting.errorBanner`: Show Expression Error Banner +- `general.useSystemColorPicker`: EffectCraft has one colour picker on every platform and the + web; there is no pure-Rust way to open the system colour panels. +- `composition.hardwareAcceleratePanels`: the Composition, Layer and Footage panels are always + drawn by the GPU (egui on wgpu). +- `scripting.enableJsDebugger`: the JavaScript engine has no step debugger; script errors report + their file and line in the Script Console. ## Keyboard shortcuts From c8623b1864e0c9a04b09bae8427e3f75feebdfa7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:46:09 -0400 Subject: [PATCH 10/36] M12.7: GPU blur/transition/generate effects (radial, camera lens, CC radial fast, wipes, venetian, cell pattern, checkerboard, grid, 4-color, noise, add grain) Kernels in fx_generate.wgsl mirror the CPU effects; wipe edges use f64 per-column/per-row terms uploaded as (hi, lo) pairs. CPU helpers shared with the GPU are now public (camera_lens_spans, gen_mode, pattern_kind, GrainLook, place_layer). GPU tests in tests_fx_generate.rs. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/blur2.rs | 18 +- crates/effects/src/generate.rs | 2 +- crates/effects/src/generate2.rs | 2 +- crates/effects/src/lib.rs | 20 + crates/effects/src/noise.rs | 4 +- crates/effects/src/transition.rs | 2 +- crates/gpu/src/fx_generate.rs | 400 ++++++++++++++- crates/gpu/src/lib.rs | 2 + crates/gpu/src/shaders/fx_generate.wgsl | 629 ++++++++++++++++++++++++ crates/gpu/src/tests_fx_generate.rs | 200 ++++++++ docs/effects.md | 28 +- 11 files changed, 1275 insertions(+), 32 deletions(-) create mode 100644 crates/gpu/src/tests_fx_generate.rs diff --git a/crates/effects/src/blur2.rs b/crates/effects/src/blur2.rs index 529a8dee..3b84d283 100644 --- a/crates/effects/src/blur2.rs +++ b/crates/effects/src/blur2.rs @@ -277,11 +277,9 @@ fn iris_spans(r: f64, sides: u32, rotation_deg: f64, roundness: f64) -> Vec<(i64 spans } -fn camera_lens_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { - let r = ctx.params.f("blurRadius").max(0.0) * b.scale; - if r < 0.5 { - return b; - } +/// Camera Lens Blur's iris spans at blur radius `r` pixels (shape, rotation, roundness and +/// aspect ratio applied) and the iris' horizontal reach (shared with the GPU kernel). +pub fn camera_lens_spans(ctx: &EffectCtx, r: f64) -> (Vec<(i64, i64, i64)>, f64) { let shape = ctx.params.e("irisProperties/irisShape"); let sides = if shape >= 8 { 0 } else { shape + 3 }; let mut spans = iris_spans(r, sides, ctx.params.f("irisProperties/irisRotation"), ctx.params.f("irisProperties/irisRoundness").clamp(0.0, 100.0) / 100.0); @@ -293,7 +291,15 @@ fn camera_lens_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { s.2 = (s.2 as f64 * aspect).round() as i64; } } - let reach = r * aspect.max(1.0); + (spans, r * aspect.max(1.0)) +} + +fn camera_lens_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let r = ctx.params.f("blurRadius").max(0.0) * b.scale; + if r < 0.5 { + return b; + } + let (spans, reach) = camera_lens_spans(ctx, r); let repeat = ctx.params.b("repeatEdge") || ctx.adjustment; if !repeat { b.pad(reach.ceil() as u32 + 1); diff --git a/crates/effects/src/generate.rs b/crates/effects/src/generate.rs index 511a8fea..09b25a27 100644 --- a/crates/effects/src/generate.rs +++ b/crates/effects/src/generate.rs @@ -75,7 +75,7 @@ pub(crate) const GEN_BLEND_MODES: [&str; 20] = [ "Silhouette Alpha", ]; -fn gen_mode(i: u32) -> Option { +pub fn gen_mode(i: u32) -> Option { Some(match i { 0 => return None, 2 => BlendMode::Add, diff --git a/crates/effects/src/generate2.rs b/crates/effects/src/generate2.rs index 19a9b6a3..ddde36d7 100644 --- a/crates/effects/src/generate2.rs +++ b/crates/effects/src/generate2.rs @@ -61,7 +61,7 @@ const CELL_PATTERNS: [&str; 12] = [ ]; /// Shaping used for Cell Pattern option `i` (see [`cell_value`]). -fn pattern_kind(i: u32) -> u32 { +pub fn pattern_kind(i: u32) -> u32 { match i { 0..=5 => i, 6 => 1, diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index a5b4f346..4f347231 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -67,6 +67,12 @@ use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; pub use effectcraft_raster::{AuxChannels, Image}; pub use misc::{INVERT_ALPHA, INVERT_CHANNELS}; +// CPU helpers the GPU kernels share (effectcraft-gpu). +pub use blur2::camera_lens_spans; +pub use generate::gen_mode; +pub use generate2::pattern_kind as cell_pattern_kind; +pub use noise::GrainLook; +pub use transition::place_layer; /// Effect categories in Effects & Presets order. pub const CATEGORIES: &[&str] = &[ @@ -641,6 +647,20 @@ pub const GPU_EFFECTS: &[&str] = &[ "ec.color.exposure", "ec.channel.invert", "ec.distort.transform", + // Blur, transition and generate family (effectcraft-gpu `fx_generate`). + "ec.blur.radial", + "ec.blur.cameralens", + "ec.blur.ccradialfast", + "ec.transition.venetian", + "ec.transition.linearwipe", + "ec.transition.radialwipe", + "ec.transition.gradientwipe", + "ec.generate.cellpattern", + "ec.generate.checkerboard", + "ec.generate.grid", + "ec.generate.fourcolor", + "ec.noise.noise", + "ec.noise.addgrain", ]; /// Effects whose output depends on [`EffectCtx::time`] directly (not only through animated diff --git a/crates/effects/src/noise.rs b/crates/effects/src/noise.rs index 4555d16a..4181c070 100644 --- a/crates/effects/src/noise.rs +++ b/crates/effects/src/noise.rs @@ -41,7 +41,7 @@ pub(crate) const GRAIN_BLEND_MODES: [&str; 5] = ["Film", "Multiply", "Add", "Scr /// The Tweaking / Color / Application / Animation controls Add Grain and Match Grain share /// (twirl-down groups with the same ids in both effects). -pub(crate) struct GrainLook { +pub struct GrainLook { pub intensity: f32, pub channel_intensity: [f32; 3], pub size: f32, @@ -62,7 +62,7 @@ pub(crate) struct GrainLook { } impl GrainLook { - pub(crate) fn from_params(ctx: &EffectCtx) -> GrainLook { + pub fn from_params(ctx: &EffectCtx) -> GrainLook { let pr = ctx.params; let ch = |pre: &str, a: &str, b: &str, c: &str| [a, b, c].map(|k| pr.get(&format!("{pre}{k}")).map(Value::as_f64).unwrap_or(1.0) as f32); let ft = ctx.time * 24.0 * pr.f("animation/animationSpeed"); diff --git a/crates/effects/src/transition.rs b/crates/effects/src/transition.rs index c41c6449..9701f0c4 100644 --- a/crates/effects/src/transition.rs +++ b/crates/effects/src/transition.rs @@ -30,7 +30,7 @@ fn scale_px(px: &mut Px, k: f32) { /// Resample `other` into `b`'s pixel grid with placement `mode` (Gradient Wipe's Gradient /// Placement, Texturize's Texture Placement): 0 tile from the top-left, 1 centre once, /// 2 stretch to fit. -pub(crate) fn place_layer(ctx: &EffectCtx, b: &Buf, other: &crate::LayerPixels, mode: u32) -> effectcraft_raster::Image { +pub fn place_layer(ctx: &EffectCtx, b: &Buf, other: &crate::LayerPixels, mode: u32) -> effectcraft_raster::Image { let (ls, os) = (ctx.layer_size, other.size); let inv = 1.0 / b.scale.max(1e-9); crate::util::gen_image(b.img.width, b.img.height, |x, y| { diff --git a/crates/gpu/src/fx_generate.rs b/crates/gpu/src/fx_generate.rs index 61c18def..976bfc8e 100644 --- a/crates/gpu/src/fx_generate.rs +++ b/crates/gpu/src/fx_generate.rs @@ -1,18 +1,404 @@ //! GPU effects, blur, transition and generate family (kernels in `shaders/fx_generate.wgsl`): the CPU effects' exact //! steps with the pixel loops as compute kernels. +//! +//! Every effect here covers all of its parameters. Where the CPU works in f64 around a hard +//! edge (the wipes), the per-column and per-row terms are computed in f64 here and uploaded as +//! (hi, lo) f32 pairs so the GPU puts the edge where the CPU does. -use effectcraft_effects::EffectCtx; +use effectcraft_color::BlendMode; +use effectcraft_effects::{EffectCtx, GrainLook}; -use crate::context::Enc; -use crate::effects::GBuf; +use crate::context::{Enc, Params}; +use crate::effects::{GBuf, gaussian_blur}; /// Compute entry points in `fx_generate.wgsl`. -pub(crate) const KERNELS: &[&str] = &[]; +pub(crate) const KERNELS: &[&str] = &[ + "gen_radial_blur", + "gen_ccradialfast", + "gen_lens_pre", + "gen_row_prefix", + "gen_lens_blur", + "gen_linear_wipe", + "gen_venetian", + "gen_radial_wipe", + "gen_gradient_wipe", + "gen_cell_pattern", + "gen_checker", + "gen_grid", + "gen_fourcolor", + "gen_noise", + "gen_grain_planes", + "gen_grain_apply", +]; /// Effect ids implemented here. -pub(crate) const IDS: &[&str] = &[]; +pub(crate) const IDS: &[&str] = &[ + "ec.blur.radial", + "ec.blur.cameralens", + "ec.blur.ccradialfast", + "ec.transition.venetian", + "ec.transition.linearwipe", + "ec.transition.radialwipe", + "ec.transition.gradientwipe", + "ec.generate.cellpattern", + "ec.generate.checkerboard", + "ec.generate.grid", + "ec.generate.fourcolor", + "ec.noise.noise", + "ec.noise.addgrain", +]; /// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. -pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { - None +pub(crate) fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + match id { + "ec.blur.radial" => radial_blur(e, ctx, b), + "ec.blur.cameralens" => camera_lens(e, ctx, b), + "ec.blur.ccradialfast" => radial_fast(e, ctx, b), + "ec.transition.venetian" => venetian(e, ctx, b), + "ec.transition.linearwipe" => linear_wipe(e, ctx, b), + "ec.transition.radialwipe" => radial_wipe(e, ctx, b), + "ec.transition.gradientwipe" => gradient_wipe(e, ctx, b), + "ec.generate.cellpattern" => cell_pattern(e, ctx, b), + "ec.generate.checkerboard" => checkerboard(e, ctx, b), + "ec.generate.grid" => grid(e, ctx, b), + "ec.generate.fourcolor" => four_color(e, ctx, b), + "ec.noise.noise" => noise(e, ctx, b), + "ec.noise.addgrain" => add_grain(e, ctx, b), + _ => None, + } +} + +/// Run a per-pixel kernel over the buffer. +fn run(e: &mut Enc, entry: &str, p: &Params, b: GBuf, aux: Option<&crate::context::GpuImage>, data: Option<&[f32]>) -> Option { + let buf = data.map(|d| e.data(d)); + let out = e.image(b.img.width, b.img.height); + e.pixels(entry, p, &b.img, aux, &out, buf.as_ref()); + Some(GBuf { img: out, ..b }) +} + +/// `v` as an f32 (hi, lo) pair. +fn hl(v: f64) -> [f32; 2] { + let h = v as f32; + [h, (v - h as f64) as f32] +} + +/// Generate blending mode index → `BlendMode::ALL` index (255 = None: replace). +fn gen_mode(i: u32) -> u32 { + match effectcraft_effects::gen_mode(i) { + None => 255, + Some(m) => BlendMode::ALL.iter().position(|&x| x == m).unwrap_or(0) as u32, + } +} + +// ---------------------------------------------------------------- blurs + +fn radial_blur(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let amt = ctx.params.f("amount"); + let zoom = ctx.params.e("type") == 1; + let c = b.to_px(ctx.params.v2("center")); + let amount = if zoom { amt / 100.0 } else { amt }; + if amount.abs() < 1e-6 { + return Some(b); + } + let n = 32usize; + let mut data = Vec::with_capacity(2 * n); + for i in 0..n { + let t = i as f64 / (n - 1) as f64 - 0.5; + if zoom { + data.push((1.0 + t * amount) as f32); + } else { + let (s, c) = (t * amount).to_radians().sin_cos(); + data.extend([s as f32, c as f32]); + } + } + let mut p = Params::default(); + p.u[0] = [zoom as u32, n as u32, 0, 0]; + p.f[0] = [c.0 as f32, c.1 as f32, 0.0, 0.0]; + run(e, "gen_radial_blur", &p, b, None, Some(&data)) +} + +fn radial_fast(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let s = (ctx.params.f("amount") / 100.0).clamp(0.0, 1.0) * 0.5; + if s <= 0.0 { + return Some(b); + } + let c = b.to_px(ctx.params.v2("center")); + let mut p = Params::default(); + p.u[0][0] = ctx.params.e("zoom"); + p.f[0] = [c.0 as f32, c.1 as f32, s as f32, 0.0]; + run(e, "gen_ccradialfast", &p, b, None, None) +} + +fn camera_lens(e: &mut Enc, ctx: &EffectCtx, mut b: GBuf) -> Option { + let r = ctx.params.f("blurRadius").max(0.0) * b.scale; + if r < 0.5 { + return Some(b); + } + let (spans, reach) = effectcraft_effects::camera_lens_spans(ctx, r); + let repeat = ctx.params.b("repeatEdge") || ctx.adjustment; + if !repeat { + b.pad(e, reach.ceil() as u32 + 1)?; + } + let linear = ctx.params.b("useLinear"); + let thr = ctx.params.f("highlight/specularThreshold") as f32 / 255.0; + let boost = 1.0 + ctx.params.f("highlight/specularBrightness").max(0.0) as f32 / 100.0 * 4.0; + let sat = (ctx.params.f("highlight/highlightSaturation") / 100.0).clamp(0.0, 1.0) as f32; + let (w, h) = (b.img.width, b.img.height); + let mut src = b.img.clone(); + if linear || boost > 1.0 { + let mut p = Params::default(); + p.u[0] = [linear as u32, (boost > 1.0) as u32, 0, 0]; + p.f[0] = [thr, boost, sat, 0.0]; + let out = e.image(w, h); + e.pixels("gen_lens_pre", &p, &src, None, &out, None); + src = out; + } + let count: i64 = spans.iter().map(|(_, l, r)| r - l + 1).sum(); + let norm = 1.0 / count as f64; + // Large irises sum each span from per-row prefix sums; small ones add the pixels directly. + let prefix = count > 256; + let pre = if prefix { + if !e.g.fits(w + 1, h) { + return None; + } + let pre = e.image(w + 1, h); + e.dispatch("gen_row_prefix", &Params::default(), &src, None, &pre, None, (h.div_ceil(64), 1)); + Some(pre) + } else { + None + }; + let data: Vec = spans.iter().flat_map(|&(dy, l, r)| [dy as f32, l as f32, r as f32]).collect(); + let mut p = Params::default(); + p.u[0] = [spans.len() as u32, repeat as u32, linear as u32, prefix as u32]; + p.f[0][0] = norm as f32; + let buf = e.data(&data); + let out = e.image(w, h); + e.pixels("gen_lens_blur", &p, &src, pre.as_ref(), &out, Some(&buf)); + b.img = out; + Some(b) +} + +// ---------------------------------------------------------------- transitions + +fn linear_wipe(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let done = ctx.params.f("completion") / 100.0; + let ang = ctx.params.f("angle").to_radians(); + let feather = (ctx.params.f("feather") * b.scale).max(0.001); + let (w, h) = (b.img.width as f64, b.img.height as f64); + let (dx, dy) = (ang.sin(), -ang.cos()); + let corners = [(0.0, 0.0), (w, 0.0), (0.0, h), (w, h)]; + let proj: Vec = corners.iter().map(|(x, y)| x * dx + y * dy).collect(); + let (lo, hi) = (proj.iter().cloned().fold(f64::INFINITY, f64::min), proj.iter().cloned().fold(f64::NEG_INFINITY, f64::max)); + let edge = lo - feather + (hi - lo + 2.0 * feather) * done; + let mut data = Vec::with_capacity(2 * (b.img.width + b.img.height) as usize); + for x in 0..b.img.width { + data.extend(hl((x as f64 + 0.5) * dx - edge)); + } + for y in 0..b.img.height { + data.extend(hl((y as f64 + 0.5) * dy)); + } + let mut p = Params::default(); + p.u[0][0] = 2 * b.img.width; + p.f[0][0] = feather as f32; + run(e, "gen_linear_wipe", &p, b, None, Some(&data)) +} + +fn venetian(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let done = ctx.params.f("completion") / 100.0; + let ang = ctx.params.f("direction").to_radians(); + let width = (ctx.params.f("width") * b.scale).max(1.0); + let feather = (ctx.params.f("feather") * b.scale).max(0.001); + let (dx, dy) = (ang.cos(), ang.sin()); + let mut data = Vec::with_capacity((6 * b.img.width + 2 * b.img.height) as usize); + for x in 0..b.img.width { + let cx = ((x as f64 + 0.5) * dx).rem_euclid(width); + data.extend(hl(cx - done * width)); + data.extend(hl(cx - width - done * width)); + data.extend(hl(cx - width)); + } + for y in 0..b.img.height { + data.extend(hl(((y as f64 + 0.5) * dy).rem_euclid(width))); + } + let mut p = Params::default(); + p.u[0][0] = 6 * b.img.width; + p.f[0][0] = feather as f32; + run(e, "gen_venetian", &p, b, None, Some(&data)) +} + +fn radial_wipe(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let done = ctx.params.f("completion") / 100.0; + let start = ctx.params.f("startAngle"); + let c = b.to_px(ctx.params.v2("center")); + let dir = ctx.params.e("wipe"); + let feather = ctx.params.f("feather").max(0.01); + let edge = if dir == 2 { done * 180.0 } else { done * 360.0 }; + // Rotations: the start angle, and the start angle ± the edge (where the wipe's edge lies). + let rots = [start, start + edge, start - edge].map(|r| r.to_radians().sin_cos()); + let mut data = Vec::with_capacity(9 * (b.img.width + b.img.height) as usize); + for x in 0..b.img.width { + let ux = x as f64 + 0.5 - c.0; + for (s, co) in rots { + data.extend(hl(ux * co)); + data.push((ux * s) as f32); + } + } + for y in 0..b.img.height { + let uy = -(y as f64 + 0.5 - c.1); + for (s, co) in rots { + data.extend(hl(uy * s)); + data.push((uy * co) as f32); + } + } + let mut p = Params::default(); + p.u[0] = [9 * b.img.width, dir.min(2), 0, 0]; + p.f[0] = [edge as f32, feather as f32, 0.0, 0.0]; + run(e, "gen_radial_wipe", &p, b, None, Some(&data)) +} + +fn gradient_wipe(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let done = (ctx.params.f("completion") / 100.0).clamp(0.0, 1.0) as f32; + if done <= 0.0 { + return Some(b); + } + let soft = (ctx.params.f("softness") / 100.0) as f32; + // The gradient layer is fetched and placed on the CPU (as the CPU effect does). + let grad = match ctx.layer_param("gradientLayer", false) { + Some(o) => { + // place_layer reads only the buffer's size, offset and scale. + let img = effectcraft_raster::Image { width: b.img.width, height: b.img.height, data: vec![] }; + let cpu = effectcraft_effects::Buf { img, offset: b.offset, scale: b.scale }; + let img = effectcraft_effects::place_layer(ctx, &cpu, &o, ctx.params.e("gradientPlacement")); + Some(e.g.upload_image(&img)?) + } + None => None, + }; + let mut p = Params::default(); + p.u[0][0] = ctx.params.b("invert") as u32; + p.f[0] = [done, soft, 0.0, 0.0]; + let out = e.image(b.img.width, b.img.height); + e.pixels("gen_gradient_wipe", &p, &b.img, Some(grad.as_ref().unwrap_or(&b.img)), &out, None); + Some(GBuf { img: out, ..b }) +} + +// ---------------------------------------------------------------- generate + +fn cell_pattern(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let pr = ctx.params; + let size = (pr.f("size") * b.scale).max(1.0); + let off = b.to_px(pr.v2("offset")); + let mut evo = pr.f("evolution") / 360.0; + if pr.b("evolutionOptions/cycleEvolution") { + evo = evo.rem_euclid(pr.f("evolutionOptions/cycle").round().max(1.0)); + } + let seed = (pr.f("evolutionOptions/randomSeed") as i64 as u32).wrapping_mul(7919) ^ 0xce11; + let tile = pr.b("tilingOptions/enableTiling"); + let nx = pr.f("tilingOptions/cellsHorizontal").round().max(1.0) as i64; + let ny = pr.f("tilingOptions/cellsVertical").round().max(1.0) as i64; + if nx > i32::MAX as i64 || ny > i32::MAX as i64 { + return None; + } + let mut p = Params::default(); + p.u[0] = [effectcraft_effects::cell_pattern_kind(pr.e("cellPattern")), pr.e("overflow"), pr.b("invert") as u32, seed]; + p.u[1] = [tile as u32, nx as u32, ny as u32, 0]; + // cos / sin of 2π·(h + evo) only see evo modulo 1. + p.f[0] = [pr.f("contrast") as f32 / 100.0, pr.f("disperse").clamp(0.0, 1.5) as f32, size as f32, evo.rem_euclid(1.0) as f32]; + p.f[1] = [(0.5 - off.0) as f32, (0.5 - off.1) as f32, 0.0, 0.0]; + run(e, "gen_cell_pattern", &p, b, None, None) +} + +/// generate::cell_size. +fn cell_size(ctx: &EffectCtx, b: &GBuf, anchor: (f64, f64)) -> (f64, f64) { + let (w, h) = match ctx.params.e("sizeFrom") { + 0 => { + let c = b.to_px(ctx.params.v2("corner")); + ((c.0 - anchor.0).abs(), (c.1 - anchor.1).abs()) + } + 1 => (ctx.params.f("width") * b.scale, ctx.params.f("width") * b.scale), + _ => (ctx.params.f("width") * b.scale, ctx.params.f("height") * b.scale), + }; + (w.max(1.0), h.max(1.0)) +} + +fn checkerboard(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let anchor = b.to_px(ctx.params.v2("anchor")); + let (w, h) = cell_size(ctx, &b, anchor); + let fw = (ctx.params.f("feather/featherWidth") * b.scale).max(1.0); + let fh = (ctx.params.f("feather/featherHeight") * b.scale).max(1.0); + let mut p = Params::default(); + p.u[0][0] = gen_mode(ctx.params.e("blendingMode")); + p.f[0] = [(0.5 - anchor.0) as f32, (0.5 - anchor.1) as f32, w as f32, h as f32]; + p.f[1] = [fw as f32, fh as f32, ctx.params.f("opacity") as f32 / 100.0, 0.0]; + p.f[2] = ctx.params.color("color"); + run(e, "gen_checker", &p, b, None, None) +} + +fn grid(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let anchor = b.to_px(ctx.params.v2("anchor")); + let (w, h) = cell_size(ctx, &b, anchor); + let border = (ctx.params.f("border") * b.scale).max(0.0); + let fw = (ctx.params.f("feather/featherWidth") * b.scale).max(1.0); + let fh = (ctx.params.f("feather/featherHeight") * b.scale).max(1.0); + let mut p = Params::default(); + p.u[0] = [gen_mode(ctx.params.e("blendingMode")), ctx.params.b("invertGrid") as u32, 0, 0]; + p.f[0] = [(0.5 - anchor.0) as f32, (0.5 - anchor.1) as f32, w as f32, h as f32]; + p.f[1] = [fw as f32, fh as f32, ctx.params.f("opacity") as f32 / 100.0, border as f32]; + p.f[2] = ctx.params.color("color"); + run(e, "gen_grid", &p, b, None, None) +} + +fn four_color(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let mut p = Params::default(); + for i in 0..4 { + let q = b.to_px(ctx.params.v2(&format!("positionsColors/point{}", i + 1))); + p.f[i] = [q.0 as f32, q.1 as f32, 0.0, 0.0]; + p.f[4 + i] = ctx.params.color(&format!("positionsColors/color{}", i + 1)); + } + let blend = ctx.params.f("blend").max(1.0); + let jitter = (ctx.params.f("jitter") / 100.0).clamp(0.0, 1.0) as f32; + p.u[0] = [gen_mode(ctx.params.e("blendingMode")), ctx.seed, 0, 0]; + p.f[8] = [(blend * 100.0) as f32, jitter, ctx.params.f("opacity") as f32 / 100.0, 0.0]; + run(e, "gen_fourcolor", &p, b, None, None) +} + +// ---------------------------------------------------------------- noise + +fn noise(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let amt = ctx.params.f("amount") as f32 / 100.0; + let clip = ctx.params.get("clip").is_none_or(|v| v.as_bool()); + let frame_seed = (ctx.time * 1000.0) as u32 ^ ctx.seed; + let mut p = Params::default(); + p.u[0] = [frame_seed, ctx.params.b("color") as u32, clip as u32, b.img.width]; + p.f[0][0] = amt; + run(e, "gen_noise", &p, b, None, None) +} + +fn add_grain(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let look = GrainLook::from_params(ctx); + if look.intensity <= 0.0 { + return Some(b); + } + let (w, h) = (b.img.width, b.img.height); + // GrainLook::planes. + let size = look.size * b.scale as f32; + let s = |k: usize| (size * if look.mono { 1.0 } else { look.channel_size[k].max(0.05) }).max(0.05); + let mut p = Params::default(); + p.u[0] = [look.mono as u32, look.seed, 101, 0]; + p.f[0] = [s(0) * look.aspect, s(1) * look.aspect, s(2) * look.aspect, look.frame]; + p.f[1] = [s(0), s(1), s(2), 0.0]; + let mut planes = e.image(w, h); + e.pixels("gen_grain_planes", &p, &b.img, None, &planes, None); + let soft = look.softness * b.scale; + if soft > 0.05 { + planes = gaussian_blur(e, &planes, soft * look.aspect as f64, soft, true); + } + // GrainLook::color's tint factors. + let l = effectcraft_color::luminance(look.tint[0], look.tint[1], look.tint[2]).max(1e-3); + let tint = [0, 1, 2].map(|k| (1.0 - look.tint_amount) + look.tint_amount * look.tint[k] / l); + let mut p = Params::default(); + p.u[0] = [look.mono as u32, look.mode, (look.tint_amount > 0.0) as u32, 0]; + p.f[0] = [look.intensity * 0.25, look.saturation, look.midpoint, 0.0]; + p.f[1] = [tint[0], tint[1], tint[2], 0.0]; + p.f[2] = [look.channel_intensity[0], look.channel_intensity[1], look.channel_intensity[2], 0.0]; + p.f[3] = [look.tones[0], look.tones[1], look.tones[2], 0.0]; + run(e, "gen_grain_apply", &p, b, Some(&planes), None) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 9ebce209..2da1af20 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -149,3 +149,5 @@ mod tests_3d; mod tests_adjust; #[cfg(test)] mod tests_adv3d; +#[cfg(test)] +mod tests_fx_generate; diff --git a/crates/gpu/src/shaders/fx_generate.wgsl b/crates/gpu/src/shaders/fx_generate.wgsl index bc67b93a..e5e2f411 100644 --- a/crates/gpu/src/shaders/fx_generate.wgsl +++ b/crates/gpu/src/shaders/fx_generate.wgsl @@ -1,2 +1,631 @@ // GPU effects (generate family): see src/fx_generate.rs. Each entry point mirrors the CPU effect in // effectcraft-effects operation for operation; parameter layouts are documented per entry. +// +// Hard edges (wipes) take per-column and per-row terms computed in f64 on the CPU and uploaded +// as (hi, lo) f32 pairs in `data`: near an edge the hi parts cancel exactly, so the edge sits +// where the CPU's f64 arithmetic puts it. + +const GEN_DEG: f32 = 57.29577951308232; + +// util::unpremul: (straight colour, alpha). +fn gen_unpremul(p: vec4) -> vec4 { + if (p.w > 1e-6) { + return vec4(p.xyz / p.w, p.w); + } + return vec4(0.0, 0.0, 0.0, max(p.w, 0.0)); +} + +// util::smoothstep. +fn gen_smoothstep(e0: f32, e1: f32, x: f32) -> f32 { + if (abs(e1 - e0) < 1e-9) { + return select(1.0, 0.0, x < e0); + } + let t = clamp((x - e0) / (e1 - e0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// generate::gen_blend: mode = index into BlendMode::ALL, 255 = None (replace). +fn gen_blend(o: vec4, g: vec4, mode: u32) -> vec4 { + if (mode == 255u) { + return g; + } + return blend_pixel(mode, o, g, 0.5); +} + +// A (hi, lo) pair from `data`. +fn gen_hl(i: u32) -> vec2 { + return vec2(data[i], data[i + 1u]); +} + +// (a + b) for (hi, lo) pairs whose hi parts cancel near zero. +fn gen_sum(a: vec2, b: vec2) -> f32 { + return (a.x + b.x) + (a.y + b.y); +} + +// ---------------------------------------------------------------- Radial Blur (raster::radial_blur) + +// u[0] = (zoom, samples); f[0] = (cx, cy); data = per sample (sin, cos) (spin) or k (zoom). +@compute @workgroup_size(16, 16) +fn gen_radial_blur(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let zoom = P.u[0].x != 0u; + let n = P.u[0].y; + let c = P.f[0].xy; + let v = vec2(f32(p.x) + 0.5 - c.x, f32(p.y) + 0.5 - c.y); + var acc = vec4(0.0); + for (var i = 0u; i < n; i++) { + var s: vec2; + if (zoom) { + let k = data[i]; + s = vec2(c.x + v.x * k, c.y + v.y * k); + } else { + let sn = data[2u * i]; + let cs = data[2u * i + 1u]; + s = vec2(c.x + v.x * cs - v.y * sn, c.y + v.x * sn + v.y * cs); + } + acc += sample_bilinear(src, s.x, s.y); + } + textureStore(out, p, acc / f32(n)); +} + +// ---------------------------------------------------------------- CC Radial Fast Blur + +// u[0].x = zoom mode (0 standard, 1 brightest, 2 darkest); f[0] = (cx, cy, s). +@compute @workgroup_size(16, 16) +fn gen_ccradialfast(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let mode = P.u[0].x; + let c = P.f[0].xy; + let s = P.f[0].z; + let d = vec2(f32(p.x) + 0.5 - c.x, f32(p.y) + 0.5 - c.y); + let len = sqrt(d.x * d.x + d.y * d.y) * s; + let n = clamp(u32(ceil(len)), 2u, 64u); + var acc = vec4(0.0); + if (mode == 1u) { + acc = vec4(-3.0e38); + } else if (mode == 2u) { + acc = vec4(3.0e38); + } + for (var i = 0u; i < n; i++) { + let k = 1.0 - s * f32(i) / f32(n - 1u); + let q = sample_bilinear(src, c.x + d.x * k, c.y + d.y * k); + if (mode == 1u) { + acc = max(acc, q); + } else if (mode == 2u) { + acc = min(acc, q); + } else { + acc += q / f32(n); + } + } + let a = acc.w; + textureStore(out, p, vec4(max(min(acc.xyz, vec3(a)), vec3(0.0)), clamp(a, 0.0, 1.0))); +} + +// ---------------------------------------------------------------- Camera Lens Blur + +// Linear working space and highlight boost before the blur. +// u[0] = (linear, boost); f[0] = (threshold, boost, saturation). +@compute @workgroup_size(16, 16) +fn gen_lens_pre(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + var px = textureLoad(src, p, 0); + if (P.u[0].x != 0u) { + let u = gen_unpremul(px); + px = vec4(srgb_to_linear(u.x) * u.w, srgb_to_linear(u.y) * u.w, srgb_to_linear(u.z) * u.w, u.w); + } + if (P.u[0].y != 0u) { + let u = gen_unpremul(px); + let c = u.xyz; + let a = u.w; + if (a > 0.0 && luminance(c) >= P.f[0].x) { + let boost = P.f[0].y; + let sat = P.f[0].z; + let m = max(max(c.x, c.y), c.z); + let tinted = c + (m - c) * (1.0 - sat); + px = vec4((c + tinted * (boost - 1.0)) * a, px.w); + } + } + textureStore(out, p, px); +} + +// Per-row prefix sums: out(x + 1, y) = sum of src(0..=x, y), out(0, y) = 0. One thread per row. +@compute @workgroup_size(64, 1) +fn gen_row_prefix(@builtin(global_invocation_id) gid: vec3) { + let dims = vec2(textureDimensions(src)); + let y = i32(gid.x); + if (y >= dims.y) { + return; + } + var acc = vec4(0.0); + textureStore(out, vec2(0, y), acc); + for (var x = 0; x < dims.x; x++) { + acc += textureLoad(src, vec2(x, y), 0); + textureStore(out, vec2(x + 1, y), acc); + } +} + +// The iris sum. data = spans (dy, left, right); aux = row prefix sums (when u[0].w). +// u[0] = (spans, repeat, linear, prefix); f[0].x = 1 / pixel count. +@compute @workgroup_size(16, 16) +fn gen_lens_blur(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let n = P.u[0].x; + let repeat = P.u[0].y != 0u; + let prefix = P.u[0].w != 0u; + let w = dims.x; + let h = dims.y; + var acc = vec4(0.0); + for (var s = 0u; s < n; s++) { + var yy = p.y + i32(data[3u * s]); + if (yy < 0 || yy >= h) { + if (!repeat) { + continue; + } + yy = clamp(yy, 0, h - 1); + } + var a = p.x + i32(data[3u * s + 1u]); + var bnd = p.x + i32(data[3u * s + 2u]); + if (!prefix) { + for (var i = a; i <= bnd; i++) { + if (repeat) { + acc += textureLoad(src, vec2(clamp(i, 0, w - 1), yy), 0); + } else if (i >= 0 && i < w) { + acc += textureLoad(src, vec2(i, yy), 0); + } + } + continue; + } + if (repeat) { + if (a < 0) { + let k = min(-a, bnd - a + 1); + acc += textureLoad(src, vec2(0, yy), 0) * f32(k); + a = 0; + } + if (bnd >= w) { + let k = min(bnd - (w - 1), bnd - a + 1); + acc += textureLoad(src, vec2(w - 1, yy), 0) * f32(k); + bnd = w - 1; + } + } else { + a = max(a, 0); + bnd = min(bnd, w - 1); + } + if (a <= bnd) { + acc += textureLoad(aux, vec2(bnd + 1, yy), 0) - textureLoad(aux, vec2(a, yy), 0); + } + } + var px = acc * P.f[0].x; + px.w = clamp(px.w, 0.0, 1.0); + if (P.u[0].z != 0u) { + let u = gen_unpremul(px); + px = vec4(linear_to_srgb(max(u.x, 0.0)) * u.w, linear_to_srgb(max(u.y, 0.0)) * u.w, linear_to_srgb(max(u.z, 0.0)) * u.w, u.w); + } + textureStore(out, p, px); +} + +// ---------------------------------------------------------------- wipes + +// Linear Wipe. data = per column ((x + 0.5)·dx − edge) (hi, lo), then per row ((y + 0.5)·dy) +// (hi, lo); u[0].x = row table offset; f[0].x = feather. +@compute @workgroup_size(16, 16) +fn gen_linear_wipe(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let d = gen_sum(gen_hl(2u * u32(p.x)), gen_hl(P.u[0].x + 2u * u32(p.y))); + let k = clamp(d / P.f[0].x + 0.5, 0.0, 1.0); + textureStore(out, p, textureLoad(src, p, 0) * k); +} + +// Venetian Blinds. Per column (6 floats): (cx − done·w), (cx − w − done·w), (cx − w) as +// (hi, lo), with cx = ((x + 0.5)·dx) mod w; per row (2 floats): ((y + 0.5)·dy) mod w. +// u[0].x = row table offset; f[0].x = feather. +@compute @workgroup_size(16, 16) +fn gen_venetian(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let col = 6u * u32(p.x); + let row = gen_hl(P.u[0].x + 2u * u32(p.y)); + // (d mod w) wraps once the column and row terms reach w. + let wrapped = gen_sum(gen_hl(col + 4u), row) >= 0.0; + var t: f32; + if (wrapped) { + t = gen_sum(gen_hl(col + 2u), row); + } else { + t = gen_sum(gen_hl(col), row); + } + let k = clamp(t / P.f[0].x + 0.5, 0.0, 1.0); + textureStore(out, p, textureLoad(src, p, 0) * k); +} + +// Angle (degrees, clockwise from up) of the pixel vector rotated by rotation `r` (0 = start, +// 1 = start + edge, 2 = start − edge), from the per-column / per-row tables of Radial Wipe. +fn gen_psi(r: u32, x: u32, y: u32) -> f32 { + let c = 9u * x + 3u * r; + let w = P.u[0].x + 9u * y + 3u * r; + // x' = ux·cos R − uy·sin R (cancels at the edge), y' = uy·cos R + ux·sin R. + let xr = (data[c] - data[w]) + (data[c + 1u] - data[w + 1u]); + let yr = data[w + 2u] + data[c + 2u]; + return atan2(xr, yr) * GEN_DEG; +} + +// Radial Wipe. data: per column 9 floats (for each rotation: ux·cos R (hi, lo), ux·sin R), +// per row 9 floats (uy·sin R (hi, lo), uy·cos R); u[0] = (row offset, wipe); +// f[0] = (edge, feather). +@compute @workgroup_size(16, 16) +fn gen_radial_wipe(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let x = u32(p.x); + let y = u32(p.y); + let dir = P.u[0].y; + let edge = P.f[0].x; + var a = gen_psi(0u, x, y); + if (a < 0.0) { + a += 360.0; + } + var base: f32; + if (dir == 0u || (dir == 2u && a <= 180.0)) { + base = a - edge; + if (abs(base) < 90.0) { + base = gen_psi(1u, x, y); + } + } else { + base = (360.0 - a) - edge; + if (abs(base) < 90.0) { + base = -gen_psi(2u, x, y); + } + } + let k = clamp(base / P.f[0].y + 0.5, 0.0, 1.0); + textureStore(out, p, textureLoad(src, p, 0) * k); +} + +// Gradient Wipe: aux = the placed gradient layer. u[0].x = invert; f[0] = (done, softness). +@compute @workgroup_size(16, 16) +fn gen_gradient_wipe(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let done = P.f[0].x; + let soft = P.f[0].y; + let g = gen_unpremul(textureLoad(aux, p, 0)); + var t = clamp(luminance(g.xyz), 0.0, 1.0); + if (P.u[0].x != 0u) { + t = 1.0 - t; + } + var k = 0.0; + if (done >= 1.0) { + k = 0.0; + } else if (soft > 0.0) { + k = clamp((t - (done * (1.0 + soft) - soft)) / soft, 0.0, 1.0); + } else if (t >= done) { + k = 1.0; + } + textureStore(out, p, textureLoad(src, p, 0) * k); +} + +// ---------------------------------------------------------------- Cell Pattern (generate2) + +// Worley distances for (u, v) in cell units: (F1, F2, hash of the nearest cell). +fn gen_worley(u: f32, v: f32, disperse: f32, evo: f32, seed: u32) -> vec3 { + let tile = P.u[1].x != 0u; + let nx = i32(P.u[1].y); + let ny = i32(P.u[1].z); + let cx = i32(floor(u)); + let cy = i32(floor(v)); + var f1 = 3.0e38; + var f2 = 3.0e38; + var id = 0.0; + for (var j = -1; j <= 1; j++) { + for (var i = -1; i <= 1; i++) { + let gx = cx + i; + let gy = cy + j; + var tx = gx; + var ty = gy; + if (tile) { + tx = ((gx % nx) + nx) % nx; + ty = ((gy % ny) + ny) % ny; + } + let hx = bitcast(tx); + let hy = bitcast(ty); + let h1 = hash_noise(hx, hy, seed); + let h2 = hash_noise(hx, hy, seed + 1u); + let h3 = hash_noise(hx, hy, seed + 2u); + let ang = 6.283185307179586 * (h3 + evo); + let px = f32(gx) + 0.5 + (h1 - 0.5) * disperse + 0.15 * disperse * cos(ang); + let py = f32(gy) + 0.5 + (h2 - 0.5) * disperse + 0.15 * disperse * sin(ang); + let d = sqrt((px - u) * (px - u) + (py - v) * (py - v)); + if (d < f1) { + f2 = f1; + f1 = d; + id = hash_noise(hx, hy, seed + 3u); + } else if (d < f2) { + f2 = d; + } + } + } + return vec3(f1, f2, id); +} + +fn gen_cell_value(pattern: u32, f1: f32, f2: f32, id: f32) -> f32 { + let e = f2 - f1; + switch pattern { + case 0u: { return clamp(1.0 - f1 * 1.3, 0.0, 1.0); } + case 1u: { return id * 0.8 + 0.2 * clamp(1.0 - f1, 0.0, 1.0); } + case 2u: { return clamp(e * 2.0, 0.0, 1.0); } + case 3u: { return id * gen_smoothstep(0.0, 0.06, e); } + case 4u: { return id; } + case 5u: { return clamp(1.0 - f1 * f1 * 2.0, 0.0, 1.0); } + case 6u: { return id * clamp(e * 3.0, 0.0, 1.0); } + default: { return 1.0 - clamp(abs(e - 0.15) * 5.0, 0.0, 1.0); } + } +} + +fn gen_overflow(v: f32, mode: u32) -> f32 { + if (mode == 1u) { + return 0.5 + 0.5 * tanh((v - 0.5) * 2.0); + } + if (mode == 2u) { + let t = v - 2.0 * floor(v * 0.5); + return select(t, 2.0 - t, t > 1.0); + } + return clamp(v, 0.0, 1.0); +} + +// u[0] = (pattern kind, overflow, invert, seed); u[1] = (tile, columns, rows); +// f[0] = (contrast, disperse, size, evolution mod 1); f[1] = (0.5 − offset x, 0.5 − offset y). +@compute @workgroup_size(16, 16) +fn gen_cell_pattern(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let size = P.f[0].z; + let u = (f32(p.x) + P.f[1].x) / size; + let v = (f32(p.y) + P.f[1].y) / size; + let w = gen_worley(u, v, P.f[0].y, P.f[0].w, P.u[0].w); + var val = gen_cell_value(P.u[0].x, w.x, w.y, w.z); + val = gen_overflow((val - 0.5) * P.f[0].x + 0.5, P.u[0].y); + if (P.u[0].z != 0u) { + val = 1.0 - val; + } + textureStore(out, p, vec4(val, val, val, 1.0)); +} + +// ---------------------------------------------------------------- Checkerboard / Grid / 4-Color + +// Signed distance (in feather units) to the nearest cell edge, positive in even cells. +fn gen_axis(q: f32, size: f32, f: f32) -> f32 { + let u = q / size; + let fl = floor(u); + let fr = u - fl; + let d = min(fr, 1.0 - fr) * size; + let n = i32(fl); + let s = select(-1.0, 1.0, ((n % 2) + 2) % 2 == 0); + return s * min(d / (f * 0.5), 1.0); +} + +// u[0].x = mode; f[0] = (0.5 − anchor x, 0.5 − anchor y, cell w, cell h); +// f[1] = (feather w, feather h, opacity); f[2] = colour. +@compute @workgroup_size(16, 16) +fn gen_checker(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let sx = gen_axis(f32(p.x) + P.f[0].x, P.f[0].z, P.f[1].x); + let sy = gen_axis(f32(p.y) + P.f[0].y, P.f[0].w, P.f[1].y); + let cov = clamp(0.5 + 0.5 * sx * sy, 0.0, 1.0) * P.f[1].z; + let c = P.f[2].xyz; + textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4(c * cov, cov), P.u[0].x)); +} + +// f32::rem_euclid. +fn gen_rem(x: f32, m: f32) -> f32 { + let r = x - m * trunc(x / m); + return select(r, r + m, r < 0.0); +} + +// u[0] = (mode, invert); f[0] = (0.5 − anchor x, 0.5 − anchor y, cell w, cell h); +// f[1] = (feather w, feather h, opacity, border); f[2] = colour. +@compute @workgroup_size(16, 16) +fn gen_grid(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let w = P.f[0].z; + let h = P.f[0].w; + let border = P.f[1].w; + let gx = gen_rem(f32(p.x) + P.f[0].x, w); + let gy = gen_rem(f32(p.y) + P.f[0].y, h); + let dx = min(gx, w - gx); + let dy = min(gy, h - gy); + let cx = clamp((border / 2.0 - dx) / P.f[1].x + 0.5, 0.0, 1.0); + let cy = clamp((border / 2.0 - dy) / P.f[1].y + 0.5, 0.0, 1.0); + var cov = 0.0; + if (border > 0.0) { + cov = max(cx, cy); + } + if (P.u[0].y != 0u) { + cov = 1.0 - cov; + } + let a = cov * P.f[1].z; + textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4(P.f[2].xyz * a, a), P.u[0].x)); +} + +// u[0] = (mode, seed); f[0..4] = points (x, y); f[4..8] = colours; +// f[8] = (blend · 100, jitter, opacity). +@compute @workgroup_size(16, 16) +fn gen_fourcolor(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let bl = P.f[8].x; + var acc = vec3(0.0); + var wsum = 0.0; + for (var i = 0; i < 4; i++) { + let q = P.f[i].xy; + let dx = f32(p.x) + 0.5 - q.x; + let dy = f32(p.y) + 0.5 - q.y; + let d2 = dx * dx + dy * dy; + let w = 1.0 / pow(d2 / bl + 1e-6, 1.5); + acc += P.f[4 + i].xyz * w; + wsum += w; + } + var c = acc / wsum; + let jitter = P.f[8].y; + if (jitter > 0.0) { + let n = (hash_noise(u32(p.x), u32(p.y), P.u[0].y) - 0.5) * jitter * (8.0 / 255.0); + c += vec3(n); + } + let op = P.f[8].z; + textureStore(out, p, gen_blend(textureLoad(src, p, 0), vec4(c * op, op), P.u[0].x)); +} + +// ---------------------------------------------------------------- Noise / Add Grain + +// Noise (misc::noise). u[0] = (frame seed, colour, clip, width); f[0].x = amount. +@compute @workgroup_size(16, 16) +fn gen_noise(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + var px = textureLoad(src, p, 0); + let a = px.w; + if (a <= 0.0) { + textureStore(out, p, px); + return; + } + let seed = P.u[0].x; + let w = P.u[0].w; + let amt = P.f[0].x; + let x = u32(p.x); + let y = u32(p.y); + var n = vec3((hash_noise(x, y, seed) - 0.5) * amt); + if (P.u[0].y != 0u) { + n.y = (hash_noise(x + w, y, seed) - 0.5) * amt; + n.z = (hash_noise(x + 2u * w, y, seed) - 0.5) * amt; + } + if (P.u[0].z != 0u) { + px = vec4(clamp(px.xyz + n * a, vec3(0.0), vec3(a)), a); + } else { + // Unclipped: values past black / white wrap around. + px = vec4(fract_euclid(px.x / a + n.x) * a, fract_euclid(px.y / a + n.y) * a, fract_euclid(px.z / a + n.z) * a, a); + } + textureStore(out, p, px); +} + +// Add Grain's noise planes (GrainLook::planes before Softness): rgb = per-channel zero-mean +// value noise. u[0] = (mono, seed, salt); f[0].xyz = x divisors (size · aspect), f[0].w = frame; +// f[1].xyz = y divisors (size). +@compute @workgroup_size(16, 16) +fn gen_grain_planes(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let x = f32(p.x); + let y = f32(p.y); + let frame = P.f[0].w; + let seed = P.u[0].y; + let salt = P.u[0].z; + let n0 = value_noise(x / P.f[0].x, y / P.f[1].x, frame, seed) - 0.5; + var g = vec3(n0); + if (P.u[0].x == 0u) { + g.y = value_noise(x / P.f[0].y, y / P.f[1].y, frame, seed + salt) - 0.5; + g.z = value_noise(x / P.f[0].z, y / P.f[1].z, frame, seed + 2u * salt) - 0.5; + } + textureStore(out, p, vec4(g, 0.0)); +} + +// GrainLook::blend. +fn gen_grain_blend(mode: u32, c: f32, n: f32) -> f32 { + var v: f32; + switch mode { + case 1u: { v = c * (1.0 + n); } + case 2u: { v = c + n; } + case 3u: { v = 1.0 - (1.0 - c) * (1.0 - abs(n)); } + case 4u: { v = c + n * 4.0 * c * (1.0 - c); } + default: { v = c + n * (1.0 - 0.5 * c); } + } + return max(v, 0.0); +} + +// Add Grain applied: aux = noise planes. u[0] = (mono, mode, tint); f[0] = (amp, saturation, +// midpoint); f[1].xyz = tint factors; f[2].xyz = channel intensities; f[3].xyz = tone weights. +@compute @workgroup_size(16, 16) +fn gen_grain_apply(@builtin(global_invocation_id) gid: vec3) { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return; + } + let px = textureLoad(src, p, 0); + let u = gen_unpremul(px); + let a = u.w; + if (a <= 0.0) { + textureStore(out, p, px); + return; + } + let c = u.xyz; + var g = textureLoad(aux, p, 0).xyz; + if (P.u[0].x != 0u) { + g = vec3(g.x); + } else { + let gm = (g.x + g.y + g.z) / 3.0; + g = gm + (g - gm) * P.f[0].y; + } + if (P.u[0].z != 0u) { + g = g * P.f[1].xyz; + } + let mid = P.f[0].z; + let l = clamp(luminance(c), 0.0, 1.0); + let s = 1.0 - gen_smoothstep(0.0, mid, l); + let hi = gen_smoothstep(mid, 1.0, l); + let tones = P.f[3].xyz; + let weight = tones.x * s + tones.y * (1.0 - s - hi) + tones.z * hi; + let amp = P.f[0].x; + let ci = P.f[2].xyz; + let mode = P.u[0].y; + let o = vec3( + gen_grain_blend(mode, c.x, g.x * amp * ci.x * weight), + gen_grain_blend(mode, c.y, g.y * amp * ci.y * weight), + gen_grain_blend(mode, c.z, g.z * amp * ci.z * weight), + ); + textureStore(out, p, vec4(o * a, a)); +} diff --git a/crates/gpu/src/tests_fx_generate.rs b/crates/gpu/src/tests_fx_generate.rs new file mode 100644 index 00000000..e15abb37 --- /dev/null +++ b/crates/gpu/src/tests_fx_generate.rs @@ -0,0 +1,200 @@ +//! GPU vs CPU for the blur, transition and generate family (`fx_generate`). + +use effectcraft_keyframe::Value; +use effectcraft_project::BitDepth; +use effectcraft_render::RenderOpts; +use effectcraft_time::Tick; + +use crate::tests::{Scene, c, check, compare_at, effect_case, n, opts, set}; + +fn e(v: u32) -> Value { + Value::Enum(v) +} + +fn b(v: bool) -> Value { + Value::Bool(v) +} + +fn pt(x: f64, y: f64) -> Value { + Value::Vec2([x, y]) +} + +#[test] +fn radial_blur() { + effect_case("ec.blur.radial", &[("amount", n(12.0))]); + effect_case("ec.blur.radial", &[("amount", n(30.0)), ("type", e(1)), ("center", pt(20.3, 14.7))]); + effect_case("ec.blur.radial", &[("amount", n(-45.0)), ("center", pt(50.0, 30.0))]); +} + +#[test] +fn cc_radial_fast_blur() { + effect_case("ec.blur.ccradialfast", &[("amount", n(40.0))]); + effect_case("ec.blur.ccradialfast", &[("amount", n(70.0)), ("zoom", e(1)), ("center", pt(22.4, 16.1))]); + effect_case("ec.blur.ccradialfast", &[("amount", n(25.0)), ("zoom", e(2))]); +} + +#[test] +fn camera_lens_blur() { + effect_case("ec.blur.cameralens", &[("blurRadius", n(5.0))]); + effect_case( + "ec.blur.cameralens", + &[ + ("blurRadius", n(7.5)), + ("irisProperties/irisShape", e(2)), + ("irisProperties/irisRotation", n(20.0)), + ("irisProperties/irisAspectRatio", n(1.6)), + ("repeatEdge", b(true)), + ], + ); + effect_case( + "ec.blur.cameralens", + &[ + ("blurRadius", n(4.0)), + ("irisProperties/irisShape", e(8)), + ("highlight/specularBrightness", n(60.0)), + ("highlight/specularThreshold", n(150.0)), + ("highlight/highlightSaturation", n(40.0)), + ("useLinear", b(true)), + ], + ); + // Large iris: per-row prefix sums. + effect_case("ec.blur.cameralens", &[("blurRadius", n(14.0)), ("irisProperties/irisRoundness", n(60.0))]); + effect_case("ec.blur.cameralens", &[("blurRadius", n(12.0)), ("repeatEdge", b(true)), ("useLinear", b(true))]); +} + +#[test] +fn wipes() { + effect_case("ec.transition.linearwipe", &[("completion", n(40.0)), ("angle", n(30.0)), ("feather", n(12.0))]); + effect_case("ec.transition.linearwipe", &[("completion", n(55.0)), ("angle", n(-120.0))]); + effect_case("ec.transition.linearwipe", &[("completion", n(20.0)), ("angle", n(90.0)), ("feather", n(3.0))]); + effect_case("ec.transition.venetian", &[("completion", n(35.0)), ("direction", n(30.0)), ("width", n(11.0)), ("feather", n(2.0))]); + effect_case("ec.transition.venetian", &[("completion", n(60.0)), ("width", n(17.0))]); + effect_case("ec.transition.venetian", &[("completion", n(25.0)), ("direction", n(-70.0)), ("width", n(9.0)), ("feather", n(0.5))]); + effect_case("ec.transition.radialwipe", &[("completion", n(30.0)), ("startAngle", n(25.0)), ("feather", n(10.0))]); + effect_case("ec.transition.radialwipe", &[("completion", n(45.0)), ("wipe", e(1)), ("center", pt(30.2, 19.6))]); + effect_case("ec.transition.radialwipe", &[("completion", n(60.0)), ("wipe", e(2)), ("startAngle", n(-40.0)), ("feather", n(3.0))]); + effect_case("ec.transition.radialwipe", &[("completion", n(0.0)), ("feather", n(20.0))]); + effect_case("ec.transition.gradientwipe", &[("completion", n(40.0))]); + effect_case("ec.transition.gradientwipe", &[("completion", n(55.0)), ("softness", n(30.0)), ("invert", b(true))]); + effect_case("ec.transition.gradientwipe", &[("completion", n(100.0))]); +} + +/// Gradient Wipe driven by another layer (fetched through the effect host, placed on the CPU). +#[test] +fn gradient_wipe_with_a_gradient_layer() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + for placement in 0..3 { + let mut s = Scene::new(depth); + let bg = s.solid([0.15, 0.1, 0.2], 97, 61); + s.push(bg); + let grad = s.footage(40, 30); + let gid = s.push(grad); + let mut l = s.footage(70, 44); + s.effect( + &mut l, + "ec.transition.gradientwipe", + &[("completion", n(45.0)), ("softness", n(20.0)), ("gradientLayer", Value::Layer(Some(gid.0))), ("gradientPlacement", e(placement))], + ); + set(&mut l, "transform/rotation", Value::Scalar(8.0)); + s.push(l); + let label = format!("gradient wipe layer {placement} {depth:?}"); + check(&label, compare_at(&s, opts(), Tick::ZERO), 0.0); + check(&format!("{label} half"), compare_at(&s, RenderOpts { scale: 0.5, ..opts() }, Tick::ZERO), 0.0); + } + } +} + +#[test] +fn generators() { + effect_case("ec.generate.cellpattern", &[("size", n(14.0))]); + effect_case("ec.generate.cellpattern", &[("cellPattern", e(2)), ("size", n(11.0)), ("evolution", n(130.0)), ("overflow", e(1)), ("contrast", n(150.0))]); + effect_case( + "ec.generate.cellpattern", + &[ + ("cellPattern", e(11)), + ("size", n(9.0)), + ("tilingOptions/enableTiling", b(true)), + ("tilingOptions/cellsHorizontal", n(4.0)), + ("tilingOptions/cellsVertical", n(3.0)), + ("overflow", e(2)), + ("contrast", n(250.0)), + ("invert", b(true)), + ("evolutionOptions/randomSeed", n(17.0)), + ], + ); + effect_case("ec.generate.cellpattern", &[("cellPattern", e(5)), ("size", n(16.0)), ("disperse", n(0.6)), ("offset", pt(3.3, -7.1))]); + effect_case("ec.generate.checkerboard", &[("width", n(9.0))]); + effect_case( + "ec.generate.checkerboard", + &[ + ("sizeFrom", e(2)), + ("width", n(13.0)), + ("height", n(7.0)), + ("feather/featherWidth", n(4.0)), + ("anchor", pt(5.3, 2.2)), + ("blendingMode", e(5)), + ("opacity", n(70.0)), + ], + ); + effect_case("ec.generate.checkerboard", &[("sizeFrom", e(0)), ("corner", pt(16.0, 11.0)), ("color", c(0.9, 0.3, 0.1)), ("blendingMode", e(1))]); + effect_case("ec.generate.grid", &[("width", n(12.0)), ("border", n(3.0))]); + effect_case( + "ec.generate.grid", + &[ + ("sizeFrom", e(2)), + ("width", n(15.0)), + ("height", n(9.0)), + ("border", n(2.5)), + ("feather/featherWidth", n(3.0)), + ("feather/featherHeight", n(2.0)), + ("invertGrid", b(true)), + ("blendingMode", e(4)), + ], + ); + effect_case( + "ec.generate.grid", + &[("sizeFrom", e(0)), ("corner", pt(11.0, 8.0)), ("anchor", pt(2.5, 3.5)), ("blendingMode", e(14)), ("color", c(0.2, 0.8, 0.4))], + ); + effect_case("ec.generate.fourcolor", &[]); + effect_case("ec.generate.fourcolor", &[("blend", n(40.0)), ("jitter", n(50.0)), ("opacity", n(80.0)), ("blendingMode", e(3))]); + effect_case("ec.generate.fourcolor", &[("blend", n(600.0)), ("blendingMode", e(16)), ("positionsColors/point2", pt(30.0, 20.0))]); +} + +#[test] +fn noise_and_grain() { + effect_case("ec.noise.noise", &[("amount", n(30.0))]); + effect_case("ec.noise.noise", &[("amount", n(50.0)), ("color", b(false))]); + effect_case("ec.noise.noise", &[("amount", n(40.0)), ("clip", b(false))]); + effect_case("ec.noise.addgrain", &[("tweaking/intensity", n(2.0))]); + effect_case( + "ec.noise.addgrain", + &[ + ("tweaking/intensity", n(3.0)), + ("tweaking/size", n(2.5)), + ("tweaking/softness", n(1.5)), + ("tweaking/aspectRatio", n(1.4)), + ("tweaking/channelSize/redSize", n(0.6)), + ("color/saturation", n(0.5)), + ("color/tintAmount", n(0.4)), + ("color/tintColor", c(1.0, 0.6, 0.2)), + ("application/blendingMode", e(4)), + ], + ); + effect_case( + "ec.noise.addgrain", + &[ + ("tweaking/intensity", n(1.5)), + ("color/monochromatic", b(true)), + ("application/blendingMode", e(3)), + ("application/shadows", n(0.3)), + ("application/highlights", n(1.8)), + ("application/midpoint", n(0.35)), + ("animation/randomSeed", n(42.0)), + ("animation/animationSpeed", n(2.0)), + ], + ); + effect_case( + "ec.noise.addgrain", + &[("tweaking/intensity", n(2.0)), ("application/blendingMode", e(1)), ("tweaking/channelIntensities/blueIntensity", n(2.5))], + ); +} diff --git a/docs/effects.md b/docs/effects.md index 75d32478..a69f91b6 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -5,7 +5,7 @@ EffectCraft ships 303 effects with 2434 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. -- **GPU**: 24 effects also run on the GPU compositor with identical results. +- **GPU**: 37 effects also run on the GPU compositor with identical results. - **32**: 303 effects process 32-bit float (HDR, overbright) pixels without clamping. - **Status**: 261 implemented in full, 42 partial (what is missing is listed). @@ -49,16 +49,16 @@ Effects are addressed by id (`ec..`) or display name in commands | Bilateral Blur | `ec.blur.bilateral` | 3 | | 32 | Implemented | | CC Cross Blur | `ec.blur.cccross` | 3 | | 32 | Implemented | | CC Radial Blur | `ec.blur.ccradial` | 4 | | 32 | Implemented | -| CC Radial Fast Blur | `ec.blur.ccradialfast` | 3 | | 32 | Implemented | +| CC Radial Fast Blur | `ec.blur.ccradialfast` | 3 | GPU | 32 | Implemented | | CC Vector Blur | `ec.blur.ccvector` | 4 | | 32 | Implemented | -| Camera Lens Blur | `ec.blur.cameralens` | 10 | | 32 | Partial: no Diffraction Fringe or Blur Map layer | +| Camera Lens Blur | `ec.blur.cameralens` | 10 | GPU | 32 | Partial: no Diffraction Fringe or Blur Map layer | | Camera-Shake Deblur | `ec.blur.camerashakedeblur` | 8 | | 32 | Implemented | | Channel Blur | `ec.blur.channel` | 6 | | 32 | Implemented | | Compound Blur | `ec.blur.compound` | 4 | | 32 | Implemented | | Directional Blur | `ec.blur.directional` | 2 | GPU | 32 | Implemented | | Fast Box Blur | `ec.blur.fastbox` | 4 | GPU | 32 | Implemented | | Gaussian Blur | `ec.blur.gaussian` | 3 | GPU | 32 | Implemented | -| Radial Blur | `ec.blur.radial` | 4 | | 32 | Implemented | +| Radial Blur | `ec.blur.radial` | 4 | GPU | 32 | Implemented | | Sharpen | `ec.blur.sharpen` | 1 | | 32 | Implemented | | Smart Blur | `ec.blur.smart` | 3 | | 32 | Implemented | | Unsharp Mask | `ec.blur.unsharp` | 3 | | 32 | Implemented | @@ -187,7 +187,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| 4-Color Gradient | `ec.generate.fourcolor` | 12 | | 32 | Implemented | +| 4-Color Gradient | `ec.generate.fourcolor` | 12 | GPU | 32 | Implemented | | Advanced Lightning | `ec.generate.advancedlightning` | 16 | | 32 | Partial: no Alpha Obstacle; most Expert Settings missing | | Audio Spectrum | `ec.generate.audiospectrum` | 23 | | 32 | Implemented | | Audio Waveform | `ec.generate.audiowaveform` | 17 | | 32 | Implemented | @@ -197,15 +197,15 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Light Rays | `ec.generate.cclightrays` | 8 | | 32 | Implemented | | CC Light Sweep | `ec.generate.cclightsweep` | 9 | | 32 | Implemented | | CC Threads | `ec.generate.ccthreads` | 8 | | 32 | Implemented | -| Cell Pattern | `ec.generate.cellpattern` | 14 | | 32 | Partial: HQ variants render like the standard ones | -| Checkerboard | `ec.generate.checkerboard` | 10 | | 32 | Implemented | +| Cell Pattern | `ec.generate.cellpattern` | 14 | GPU | 32 | Partial: HQ variants render like the standard ones | +| Checkerboard | `ec.generate.checkerboard` | 10 | GPU | 32 | Implemented | | Circle | `ec.generate.circle` | 11 | | 32 | Implemented | | Ellipse | `ec.generate.ellipse` | 8 | | 32 | Implemented | | Eyedropper Fill | `ec.generate.eyedropperfill` | 5 | | 32 | Implemented | | Fill | `ec.generate.fill` | 3 | GPU | 32 | Partial: no Fill Mask / All Masks / feather | | Fractal | `ec.generate.fractal` | 21 | | 32 | Implemented | | Gradient Ramp | `ec.generate.gradientramp` | 7 | GPU | 32 | Implemented | -| Grid | `ec.generate.grid` | 12 | | 32 | Implemented | +| Grid | `ec.generate.grid` | 12 | GPU | 32 | Implemented | | Lens Flare | `ec.generate.lensflare` | 4 | | 32 | Implemented | | Paint Bucket | `ec.generate.paintbucket` | 11 | | 32 | Implemented | | Radio Waves | `ec.generate.radiowaves` | 17 | | 32 | Partial: no Image Contours or Mask wave types, Parameters Are Set At, reflection or stroke profile | @@ -263,14 +263,14 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| Add Grain | `ec.noise.addgrain` | 22 | | 32 | Partial: no preview region, presets, channel balance or temporal controls | +| Add Grain | `ec.noise.addgrain` | 22 | GPU | 32 | Partial: no preview region, presets, channel balance or temporal controls | | Curl Noise | `ec.noise.curlnoise` | 10 | | 32 | Implemented | | Dust & Scratches | `ec.noise.dustscratches` | 3 | | 32 | Implemented | | Fractal Noise | `ec.noise.fractal` | 26 | GPU | 32 | Implemented | | Match Grain | `ec.noise.matchgrain` | 25 | | 32 | Partial: no preview region, presets, sampling or temporal controls | | Median | `ec.noise.median` | 2 | | 32 | Implemented | | Median (Legacy) | `ec.noise.medianlegacy` | 2 | | 32 | Implemented | -| Noise | `ec.noise.noise` | 3 | | 32 | Implemented | +| Noise | `ec.noise.noise` | 3 | GPU | 32 | Implemented | | Noise Alpha | `ec.noise.noisealpha` | 8 | | 32 | Implemented | | Noise HLS | `ec.noise.noisehls` | 6 | | 32 | Implemented | | Noise HLS Auto | `ec.noise.noisehlsauto` | 6 | | 32 | Implemented | @@ -401,11 +401,11 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Twister | `ec.transition.cctwister` | 5 | | 32 | Implemented | | CC WarpoMatic | `ec.transition.ccwarpomatic` | 8 | | 32 | Implemented | | Card Wipe | `ec.transition.cardwipe` | 12 | | 32 | Partial: 2D flip only; no back layer picker, camera, lighting or jitter | -| Gradient Wipe | `ec.transition.gradientwipe` | 5 | | 32 | Implemented | +| Gradient Wipe | `ec.transition.gradientwipe` | 5 | GPU | 32 | Implemented | | Iris Wipe | `ec.transition.iriswipe` | 7 | | 32 | Implemented | -| Linear Wipe | `ec.transition.linearwipe` | 3 | | 32 | Implemented | -| Radial Wipe | `ec.transition.radialwipe` | 5 | | 32 | Implemented | -| Venetian Blinds | `ec.transition.venetian` | 4 | | 32 | Implemented | +| Linear Wipe | `ec.transition.linearwipe` | 3 | GPU | 32 | Implemented | +| Radial Wipe | `ec.transition.radialwipe` | 5 | GPU | 32 | Implemented | +| Venetian Blinds | `ec.transition.venetian` | 4 | GPU | 32 | Implemented | ## Utility From 91ab3a3fc56feb1943e70970043658ff08839ca9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:46:52 -0400 Subject: [PATCH 11/36] M12.7: GPU colour effects (Color Balance, Vibrance, Lumetri, Black & White, Tritone, Colorama, Channel Mixer, Selective Color, Linear Color Key, Key Light) fx_color kernels mirror the CPU effects operation for operation: one per-pixel pass for the point effects, Lumetri with its Sharpen luminance blur, and Key Light's matte pipeline (pre-blur, clip, rollback, despot, shrink/grow, softness, edge band) as plane passes. CPU fallback: Key Light with Inside/Outside masks or Hard Colour after Screen Softness, Colorama with Interpolate Palette off. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/lib.rs | 11 + crates/gpu/src/fx_color.rs | 390 ++++++++++++++- crates/gpu/src/lib.rs | 2 + crates/gpu/src/shaders/fx_color.wgsl | 705 +++++++++++++++++++++++++++ crates/gpu/src/tests_fx_color.rs | 163 +++++++ docs/effects.md | 22 +- 6 files changed, 1276 insertions(+), 17 deletions(-) create mode 100644 crates/gpu/src/tests_fx_color.rs diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index a5b4f346..b5a1c9d2 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -641,6 +641,17 @@ pub const GPU_EFFECTS: &[&str] = &[ "ec.color.exposure", "ec.channel.invert", "ec.distort.transform", + // effectcraft-gpu fx_color + "ec.color.colorbalance", + "ec.color.vibrance", + "ec.color.lumetri", + "ec.color.blackwhite", + "ec.color.tritone", + "ec.color.colorama", + "ec.color.channelmixer", + "ec.color.selectivecolor", + "ec.key.linearcolor", + "ec.keying.keylight", ]; /// Effects whose output depends on [`EffectCtx::time`] directly (not only through animated diff --git a/crates/gpu/src/fx_color.rs b/crates/gpu/src/fx_color.rs index 8205f042..824f9a50 100644 --- a/crates/gpu/src/fx_color.rs +++ b/crates/gpu/src/fx_color.rs @@ -1,18 +1,396 @@ //! GPU effects, colour family (kernels in `shaders/fx_color.wgsl`): the CPU effects' exact //! steps with the pixel loops as compute kernels. +//! +//! Colour Balance, Vibrance, Black & White, Tritone, Colorama, Channel Mixer, Selective Color +//! and Linear Color Key are one per-pixel pass (`fxc_point`); Lumetri Color is a per-pixel pass +//! plus, with Sharpen, a luminance plane blurred like `util::gauss_plane`; Key Light runs its +//! matte pipeline (pre-blur, clip, rollback, despot, shrink / grow, softness, edge band) as +//! plane passes. Key Light with an Inside / Outside mask, or Hard Colour replacement after +//! Screen Softness, and Colorama with Interpolate Palette off run on the CPU. +use effectcraft_color::{luminance, rgb_to_hsl}; use effectcraft_effects::EffectCtx; -use crate::context::Enc; -use crate::effects::GBuf; +use crate::context::{Enc, GpuImage, Params}; +use crate::effects::{GBuf, gaussian_blur}; /// Compute entry points in `fx_color.wgsl`. -pub(crate) const KERNELS: &[&str] = &[]; +pub(crate) const KERNELS: &[&str] = &[ + "fxc_point", + "fxc_lumetri", + "fxc_luma", + "fxc_sharpen", + "fxc_morph", + "fxc_lerp", + "fxc_kl_raw", + "fxc_kl_rollback", + "fxc_kl_pack", + "fxc_kl_band", + "fxc_kl_edge", + "fxc_kl_final", +]; /// Effect ids implemented here. -pub(crate) const IDS: &[&str] = &[]; +pub(crate) const IDS: &[&str] = &[ + "ec.color.colorbalance", + "ec.color.vibrance", + "ec.color.lumetri", + "ec.color.blackwhite", + "ec.color.tritone", + "ec.color.colorama", + "ec.color.channelmixer", + "ec.color.selectivecolor", + "ec.key.linearcolor", + "ec.keying.keylight", +]; /// Run effect `id` (one of [`IDS`]); `None` = this parameter combination runs on the CPU. -pub(crate) fn apply(_e: &mut Enc, _id: &str, _ctx: &EffectCtx, _b: GBuf) -> Option { - None +pub(crate) fn apply(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + match id { + "ec.color.lumetri" => lumetri(e, ctx, b), + "ec.keying.keylight" => key_light(e, ctx, b), + _ => point(e, id, ctx, b), + } +} + +fn rgb(c: [f32; 4]) -> [f32; 4] { + [c[0], c[1], c[2], 0.0] +} + +/// Same selective colour ids as `color2::SC_IDS`. +const SC_IDS: [[&str; 4]; 9] = [ + ["redsCyan", "redsMagenta", "redsYellow", "redsBlack"], + ["yellowsCyan", "yellowsMagenta", "yellowsYellow", "yellowsBlack"], + ["greensCyan", "greensMagenta", "greensYellow", "greensBlack"], + ["cyansCyan", "cyansMagenta", "cyansYellow", "cyansBlack"], + ["bluesCyan", "bluesMagenta", "bluesYellow", "bluesBlack"], + ["magentasCyan", "magentasMagenta", "magentasYellow", "magentasBlack"], + ["whitesCyan", "whitesMagenta", "whitesYellow", "whitesBlack"], + ["neutralsCyan", "neutralsMagenta", "neutralsYellow", "neutralsBlack"], + ["blacksCyan", "blacksMagenta", "blacksYellow", "blacksBlack"], +]; + +fn point(e: &mut Enc, id: &str, ctx: &EffectCtx, b: GBuf) -> Option { + let pr = ctx.params; + let f = |k: &str| pr.f(k) as f32; + let mut p = Params::default(); + let mut data = None; + match id { + "ec.color.colorbalance" => { + let g = |k: &str| f(k) / 100.0; + p.u[0] = [1, pr.b("preserveLuminosity") as u32, 0, 0]; + p.f[0] = [g("shadowRed"), g("shadowGreen"), g("shadowBlue"), 0.0]; + p.f[1] = [g("midRed"), g("midGreen"), g("midBlue"), 0.0]; + p.f[2] = [g("hiRed"), g("hiGreen"), g("hiBlue"), 0.0]; + } + "ec.color.vibrance" => { + p.u[0][0] = 2; + p.f[0] = [f("vibrance") / 100.0, f("saturation") / 100.0, 0.0, 0.0]; + } + "ec.color.blackwhite" => { + let w = ["reds", "yellows", "greens", "cyans", "blues", "magentas"].map(|k| f(k) / 100.0); + let tc = pr.color("tintColor"); + let (th, ts, _) = rgb_to_hsl(tc[0], tc[1], tc[2]); + p.u[0] = [3, pr.b("tint") as u32, 0, 0]; + p.f[0] = [w[0], w[1], w[2], w[3]]; + p.f[1] = [w[4], w[5], 0.0, 0.0]; + p.f[2] = [th, ts, 0.0, 0.0]; + } + "ec.color.tritone" => { + p.u[0][0] = 4; + p.f[0] = rgb(pr.color("highlights")); + p.f[1] = rgb(pr.color("midtones")); + p.f[2] = rgb(pr.color("shadows")); + p.f[3][0] = 1.0 - f("blend") / 100.0; + } + "ec.color.colorama" => { + let smooth = pr.get("outputCycle/interpolatePalette").is_none_or(|v| v.as_bool()); + // The posterized palette is discontinuous: phases on a step (common with 8 bpc + // input) can round to the other side in the GPU's float maths, so it runs on the CPU. + if !smooth { + return None; + } + p.u[0] = [5, pr.e("inputPhase/mode"), smooth as u32, 0]; + p.f[0] = [pr.f("outputCycle/cycles").max(0.0) as f32, f("inputPhase/phaseShift") / 360.0, 1.0 - f("blend") / 100.0, 0.0]; + } + "ec.color.channelmixer" => { + let g = |k: &str| f(k) / 100.0; + p.u[0] = [6, pr.b("monochrome") as u32, 0, 0]; + p.f[0] = [g("rr"), g("rg"), g("rb"), g("rc")]; + p.f[1] = [g("gr"), g("gg"), g("gb"), g("gc")]; + p.f[2] = [g("br"), g("bg"), g("bb"), g("bc")]; + } + "ec.color.selectivecolor" => { + let adj: Vec = SC_IDS.iter().flat_map(|ids| ids.map(|k| f(k) / 100.0)).collect(); + if adj.iter().all(|v| *v == 0.0) { + return Some(b); + } + p.u[0] = [7, (pr.e("method") == 0) as u32, 0, 0]; + data = Some(e.data(&adj)); + } + "ec.key.linearcolor" => { + let view = pr.e("view"); + if view == 1 { + return Some(b); + } + let k = pr.color("keyColor"); + let s = k[0] + k[1] + k[2]; + let kc = if s <= 1e-6 { [1.0 / 3.0, 1.0 / 3.0] } else { [k[0] / s, k[1] / s] }; + let (kh, _, _) = rgb_to_hsl(k[0], k[1], k[2]); + p.u[0] = [8, pr.e("matchColors"), (pr.e("keyOperation") == 1) as u32, view]; + p.f[0] = [k[0], k[1], k[2], f("tolerance") / 100.0]; + p.f[1] = [f("softness") / 100.0, kc[0], kc[1], kh]; + } + _ => return None, + } + let out = e.image(b.img.width, b.img.height); + e.pixels("fxc_point", &p, &b.img, None, &out, data.as_ref()); + Some(GBuf { img: out, ..b }) +} + +/// color3::tint_offset. +fn tint_offset(c: [f32; 4]) -> [f32; 3] { + let l = (c[0] + c[1] + c[2]) / 3.0; + [c[0] - l, c[1] - l, c[2] - l] +} + +fn lumetri(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let pr = ctx.params; + let f = |k: &str| (pr.f(k) / 100.0) as f32; + let mut d: Vec = vec![ + f("basicCorrection/whiteBalance/temperature"), + f("basicCorrection/whiteBalance/tint"), + 2f32.powf(pr.f("basicCorrection/tone/exposure") as f32), + f("basicCorrection/tone/contrast"), + f("basicCorrection/tone/highlights"), + f("basicCorrection/tone/shadows"), + f("basicCorrection/tone/whites"), + f("basicCorrection/tone/blacks"), + f("basicCorrection/saturation"), + f("creative/lookIntensity"), + f("creative/adjustments/fadedFilm"), + f("creative/adjustments/vibrance"), + f("creative/adjustments/creativeSaturation"), + (0.5 + 0.25 * f("creative/adjustments/tintBalance")).clamp(0.05, 0.95), + ]; + d.extend(tint_offset(pr.color("creative/adjustments/shadowTint"))); + d.extend(tint_offset(pr.color("creative/adjustments/highlightTint"))); + for c in ["Master", "Red", "Green", "Blue"] { + for part in ["Shadows", "Midtones", "Highlights"] { + d.push(f(&format!("curves/curve{c}{part}"))); + } + } + for w in ["shadowsWheel", "midtonesWheel", "highlightsWheel"] { + d.extend(tint_offset(pr.color(&format!("colorWheels/{w}")))); + } + let (lx, ly) = (b.offset[0], b.offset[1]); + let (lw, lh) = (ctx.layer_size[0] * b.scale, ctx.layer_size[1] * b.scale); + d.extend([ + pr.f("vignette/vignetteAmount") as f32, + f("vignette/vignetteMidpoint") * 0.9, + f("vignette/vignetteRoundness"), + f("vignette/vignetteFeather"), + (lx + lw * 0.5) as f32, + (ly + lh * 0.5) as f32, + lw as f32, + lh as f32, + (lw / lh.max(1e-6)) as f32, + ]); + debug_assert_eq!(d.len(), 50); + let buf = e.data(&d); + let out = e.image(b.img.width, b.img.height); + e.pixels("fxc_lumetri", &Params::default(), &b.img, None, &out, Some(&buf)); + let sharpen = f("creative/adjustments/sharpen"); + if sharpen == 0.0 { + return Some(GBuf { img: out, ..b }); + } + let luma = e.image(out.width, out.height); + e.pixels("fxc_luma", &Params::default(), &out, None, &luma, None); + let blurred = gaussian_blur(e, &luma, b.scale, b.scale, true); + let mut p = Params::default(); + p.f[0][0] = sharpen * 2.0; + let sharp = e.image(out.width, out.height); + e.pixels("fxc_sharpen", &p, &out, Some(&blurred), &sharp, None); + Some(GBuf { img: sharp, ..b }) +} + +// ---------------------------------------------------------------- planes + +/// util::morph_plane with the same radius on both axes; `modes` = per-channel operation +/// (0 erode, 1 dilate, 2 keep). +fn morph(e: &mut Enc, img: &GpuImage, r: u32, modes: [u32; 4]) -> GpuImage { + if r == 0 { + return img.clone(); + } + let mut cur = img.clone(); + for vertical in [0, 1] { + let mut p = Params::default(); + p.u[0] = [r, vertical, 0, 0]; + p.u[1] = modes; + let out = e.image(cur.width, cur.height); + e.pixels("fxc_morph", &p, &cur, None, &out, None); + cur = out; + } + cur +} + +/// util::morph_frac (blend between the two nearest integer radii). +fn morph_frac(e: &mut Enc, img: &GpuImage, radius: f64, modes: [u32; 4]) -> GpuImage { + if radius <= 0.0 { + return img.clone(); + } + let r0 = radius.floor() as u32; + let t = (radius - r0 as f64) as f32; + let a = morph(e, img, r0, modes); + if t < 1e-4 { + return a; + } + let b = morph(e, img, r0 + 1, modes); + let mut p = Params::default(); + p.f[0][0] = t; + let out = e.image(img.width, img.height); + e.pixels("fxc_lerp", &p, &a, Some(&b), &out, None); + out +} + +const ERODE_X: [u32; 4] = [0, 2, 2, 2]; +const DILATE_X: [u32; 4] = [1, 2, 2, 2]; + +// ---------------------------------------------------------------- Key Light + +/// keylight::primary. +fn primary(s: [f32; 3]) -> [u32; 3] { + let pi: usize = if s[1] >= s[0] && s[1] >= s[2] { + 1 + } else if s[2] >= s[0] { + 2 + } else { + 0 + }; + let (a, b) = match pi { + 0 => (1, 2), + 1 => (0, 2), + _ => (0, 1), + }; + let ix = if s[a] >= s[b] { [pi, a, b] } else { [pi, b, a] }; + ix.map(|i| i as u32) +} + +fn neutralise(c: [f32; 3], bias: [f32; 3]) -> [f32; 3] { + let l = luminance(bias[0], bias[1], bias[2]).max(1e-4); + [0, 1, 2].map(|i| c[i] * l / bias[i].max(1e-4)) +} + +fn key_light(e: &mut Enc, ctx: &EffectCtx, b: GBuf) -> Option { + let pr = ctx.params; + let view = pr.e("view"); + if view == 0 { + return Some(b); + } + // Inside / Outside masks (polygon coverage) run on the CPU. + let has_mask = |id: &str| pr.e(id).checked_sub(1).is_some_and(|i| ctx.env.masks.get(i as usize).is_some()); + if has_mask("insideMask/insideMask") || has_mask("outsideMask/outsideMask") { + return None; + } + let sc = pr.color("screenColour"); + let screen = [sc[0], sc[1], sc[2]]; + let ix = primary(screen); + let (pi, o1, o2) = (ix[0] as usize, ix[1] as usize, ix[2] as usize); + let gain = pr.f("screenGain") as f32 / 100.0; + let bal = pr.f("screenBalance") as f32 / 100.0; + let dbias = pr.color("despillBias"); + let abias = if pr.b("lockBiasesTogether") { dbias } else { pr.color("alphaBias") }; + let (dbias, abias) = ([dbias[0], dbias[1], dbias[2]], [abias[0], abias[1], abias[2]]); + let s_alpha = neutralise(screen, abias); + let sd = (s_alpha[pi] - (bal * s_alpha[o1] + (1.0 - bal) * s_alpha[o2])).max(1e-3); + let cb = pr.f("screenMatte/clipBlack") as f32 / 100.0; + let cw = pr.f("screenMatte/clipWhite") as f32 / 100.0; + let rollback = pr.f("screenMatte/clipRollback") * b.scale; + let preblur = pr.f("screenPreblur") * b.scale; + let sg = pr.f("screenMatte/screenShrinkGrow") * b.scale; + let soft = pr.f("screenMatte/screenSoftness") * b.scale; + let despot_b = pr.f("screenMatte/screenDespotBlack") * b.scale; + let despot_w = pr.f("screenMatte/screenDespotWhite") * b.scale; + // Hard Colour replaces wherever the screen matte exceeds the raw matte by any amount; after + // Screen Softness's blur that compares float-rounded sums, so it runs on the CPU. + if pr.e("screenMatte/replaceMethod") == 2 && soft > 0.0 { + return None; + } + let (w, h) = (b.img.width, b.img.height); + + let key_src = if preblur > 0.0 { gaussian_blur(e, &b.img, preblur * 0.5, preblur * 0.5, true) } else { b.img.clone() }; + let mut p = Params::default(); + p.u[0] = [ix[0], ix[1], ix[2], 0]; + p.f[0] = [abias[0], abias[1], abias[2], sd]; + p.f[1] = [gain, bal, cb, cw]; + let raw = e.image(w, h); + e.pixels("fxc_kl_raw", &p, &key_src, None, &raw, None); + let mut m = raw.clone(); + if rollback > 0.0 { + let sg_img = morph_frac(e, &m, rollback, [0, 1, 2, 2]); + let out = e.image(w, h); + e.pixels("fxc_kl_rollback", &Params::default(), &sg_img, Some(&raw), &out, None); + m = out; + } + if despot_b > 0.0 { + let d = morph_frac(e, &m, despot_b, DILATE_X); + m = morph_frac(e, &d, despot_b, ERODE_X); + } + if despot_w > 0.0 { + let d = morph_frac(e, &m, despot_w, ERODE_X); + m = morph_frac(e, &d, despot_w, DILATE_X); + } + if sg < 0.0 { + m = morph_frac(e, &m, -sg, ERODE_X); + } else if sg > 0.0 { + m = morph_frac(e, &m, sg, DILATE_X); + } + if soft > 0.0 { + m = gaussian_blur(e, &m, soft * 0.5, soft * 0.5, true); + } + let mut planes = e.image(w, h); + e.pixels("fxc_kl_pack", &Params::default(), &m, Some(&raw), &planes, None); + let src_alpha_mode = pr.e("insideMask/sourceAlpha"); + if pr.b("edgeColourCorrection/enableEdgeColourCorrection") { + let grow = pr.f("edgeColourCorrection/edgeGrow") * b.scale; + let hard = (pr.f("edgeColourCorrection/edgeHardness") / 100.0) as f32; + let esoft = pr.f("edgeColourCorrection/edgeSoftness") * b.scale; + let mut p = Params::default(); + p.u[0][0] = src_alpha_mode; + let mut band = e.image(w, h); + e.pixels("fxc_kl_band", &p, &b.img, Some(&planes), &band, None); + if grow > 0.0 { + band = morph_frac(e, &band, grow, DILATE_X); + } + if esoft > 0.0 { + band = gaussian_blur(e, &band, esoft * 0.5, esoft * 0.5, true); + } + let mut p = Params::default(); + p.f[0][0] = hard; + let out = e.image(w, h); + e.pixels("fxc_kl_edge", &p, &planes, Some(&band), &out, None); + planes = out; + } + let pc = |k: &str| pr.f(k) as f32 / 100.0; + let l0 = (luminance(dbias[0], dbias[1], dbias[2]) / dbias[pi].max(1e-4)).max(1e-4); + let mut p = Params::default(); + p.u[0] = [ix[0], ix[1], ix[2], view]; + p.u[1] = [ + pr.e("screenMatte/replaceMethod"), + pr.b("foregroundColourCorrection/enableColourCorrection") as u32, + pr.b("unpremultiplyResult") as u32, + src_alpha_mode, + ]; + p.f[0] = [screen[0], screen[1], screen[2], bal]; + p.f[1] = [dbias[0], dbias[1], dbias[2], l0]; + p.f[2] = pr.color("screenMatte/replaceColour"); + p.f[3] = + [pc("foregroundColourCorrection/saturation"), pc("foregroundColourCorrection/contrast") + 1.0, pc("foregroundColourCorrection/brightness") + 1.0, 0.0]; + p.f[4] = [pc("edgeColourCorrection/edgeSaturation"), pc("edgeColourCorrection/edgeContrast") + 1.0, pc("edgeColourCorrection/edgeBrightness") + 1.0, 0.0]; + p.f[5] = ["sourceCrops/cropLeft", "sourceCrops/cropRight", "sourceCrops/cropTop", "sourceCrops/cropBottom"].map(|k| (pr.f(k) / 100.0) as f32); + p.f[6] = [b.offset[0] as f32, b.offset[1] as f32, b.scale as f32, 0.0]; + p.f[7] = [ctx.layer_size[0].max(1e-9) as f32, ctx.layer_size[1].max(1e-9) as f32, 0.0, 0.0]; + let out = e.image(w, h); + e.pixels("fxc_kl_final", &p, &b.img, Some(&planes), &out, None); + Some(GBuf { img: out, ..b }) } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 9ebce209..ce872460 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -149,3 +149,5 @@ mod tests_3d; mod tests_adjust; #[cfg(test)] mod tests_adv3d; +#[cfg(test)] +mod tests_fx_color; diff --git a/crates/gpu/src/shaders/fx_color.wgsl b/crates/gpu/src/shaders/fx_color.wgsl index 3e42688d..97771a46 100644 --- a/crates/gpu/src/shaders/fx_color.wgsl +++ b/crates/gpu/src/shaders/fx_color.wgsl @@ -1,2 +1,707 @@ // GPU effects (color family): see src/fx_color.rs. Each entry point mirrors the CPU effect in // effectcraft-effects operation for operation; parameter layouts are documented per entry. +// Every name here is prefixed `fxc_` (the family files share one module). + +// ---------------------------------------------------------------- helpers + +// util::unpremul: straight colour (0 when alpha ≤ 1e-6). +fn fxc_unpremul(px: vec4) -> vec3 { + if (px.w > 1e-6) { + return px.xyz / px.w; + } + return vec3(0.0); +} + +// util::smoothstep. +fn fxc_smoothstep(e0: f32, e1: f32, x: f32) -> f32 { + if (abs(e1 - e0) < 1e-9) { + return select(1.0, 0.0, x < e0); + } + let t = clamp((x - e0) / (e1 - e0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +fn fxc_mix(a: vec3, b: vec3, t: f32) -> vec3 { + return a + (b - a) * t; +} + +// color::hsv_to_rgb. +fn fxc_hsv_to_rgb(h: f32, s: f32, v: f32) -> vec3 { + let h6 = fract_euclid(h) * 6.0; + let i = floor(h6); + let f = h6 - i; + let p = v * (1.0 - s); + let q = v * (1.0 - s * f); + let t = v * (1.0 - s * (1.0 - f)); + switch i32(i) { + case 0: { return vec3(v, t, p); } + case 1: { return vec3(q, v, p); } + case 2: { return vec3(p, v, t); } + case 3: { return vec3(p, q, v); } + case 4: { return vec3(t, p, v); } + default: { return vec3(v, p, q); } + } +} + +fn fxc_pixel(gid: vec3) -> vec2 { + let dims = out_dims(); + let p = vec2(gid.xy); + if (p.x >= dims.x || p.y >= dims.y) { + return vec2(-1); + } + return p; +} + +// ---------------------------------------------------------------- per-pixel colour effects + +// Colorama's input phase (color_fx::colorama_phase). +fn fxc_colorama_phase(c: vec3, a: f32, mode: u32) -> f32 { + var v: f32; + switch mode { + case 1u: { v = c.x; } + case 2u: { v = c.y; } + case 3u: { v = c.z; } + case 4u: { v = rgb_to_hsl(c).x; } + case 5u: { v = rgb_to_hsl(c).z; } + case 6u: { v = rgb_to_hsl(c).y; } + case 7u: { v = max(max(c.x, c.y), c.z); } + case 8u: { v = a; } + case 9u: { v = 0.0; } + default: { v = luminance(c); } + } + return clamp(v, 0.0, 1.0); +} + +// Selective Color range weights (color2::sc_weights). +fn fxc_sc_weights(c0: vec3) -> array { + let c = clamp(c0, vec3(0.0), vec3(1.0)); + // Stable descending sort of the channel indices. + var idx = vec3(0u, 1u, 2u); + for (var i = 1u; i < 3u; i++) { + var j = i; + while (j > 0u && c[idx[j]] > c[idx[j - 1u]]) { + let t = idx[j]; + idx[j] = idx[j - 1u]; + idx[j - 1u] = t; + j--; + } + } + let mx = c[idx.x]; + let md = c[idx.y]; + let mn = c[idx.z]; + var w = array(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); + var prim = array(0u, 2u, 4u); + var sec = array(3u, 5u, 1u); + w[prim[idx.x]] = mx - md; + w[sec[idx.z]] = md - mn; + w[6] = max((mn - 0.5) * 2.0, 0.0); + w[8] = max((0.5 - mx) * 2.0, 0.0); + w[7] = clamp(1.0 - (abs(mx - 0.5) + abs(mn - 0.5)), 0.0, 1.0); + return w; +} + +// u[0].x = op: +// 1 Color Balance f[0] = shadows, f[1] = midtones, f[2] = highlights; u[0].y = preserve luminosity +// 2 Vibrance f[0] = (vibrance, saturation) +// 3 Black & White f[0] = (reds, yellows, greens, cyans), f[1] = (blues, magentas); +// u[0].y = tint; f[2] = (tint hue, tint saturation) +// 4 Tritone f[0] = highlights, f[1] = midtones, f[2] = shadows; f[3].x = 1 − blend +// 5 Colorama u[0].y = phase mode, u[0].z = interpolate; f[0] = (cycles, shift, 1 − blend) +// 6 Channel Mixer f[0..3] = output rows (r, g, b, const); u[0].y = monochrome +// 7 Selective Color data = 9 × (cyan, magenta, yellow, black); u[0].y = relative +// 8 Linear Color Key f[0] = (key rgb, tolerance), f[1] = (softness, key chroma xy, key hue); +// u[0] = (_, match mode, keep colours, view) +@compute @workgroup_size(16, 16) +fn fxc_point(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + var px = textureLoad(src, p, 0); + let a = px.w; + switch P.u[0].x { + case 1u: { + if (a > 0.0) { + let c = px.xyz / a; + let l = clamp(luminance(c), 0.0, 1.0); + let ws = clamp(1.0 - l * 2.0, 0.0, 1.0); + let wh = clamp(l * 2.0 - 1.0, 0.0, 1.0); + let wm = 1.0 - ws - wh; + var o = max(c + (P.f[0].xyz * ws + P.f[1].xyz * wm + P.f[2].xyz * wh) * 0.5, vec3(0.0)); + if (P.u[0].y != 0u) { + let d = luminance(c) - luminance(o); + o = max(o + d, vec3(0.0)); + } + px = vec4(o * a, a); + } + } + case 2u: { + if (a > 0.0) { + let c = px.xyz / a; + let hsl = rgb_to_hsl(c); + let vib = P.f[0].x; + let sat = P.f[0].y; + let s = clamp(hsl.y * (1.0 + sat) + vib * (1.0 - hsl.y) * min(hsl.y, 0.5), 0.0, 1.0); + px = vec4(hsl_to_rgb(hsl.x, s, hsl.z) * a, a); + } + } + case 3u: { + if (a > 0.0) { + let c = px.xyz / a; + var w = array(P.f[0].x, P.f[0].y, P.f[0].z, P.f[0].w, P.f[1].x, P.f[1].y); + let hsl = rgb_to_hsl(c); + let hp = hsl.x * 6.0; + let i = u32(max(floor(hp), 0.0)) % 6u; + let f = hp - floor(hp); + let wt = w[i] + (w[(i + 1u) % 6u] - w[i]) * f; + let base = luminance(c); + let gray = max(base + (wt - 0.5) * hsl.y * 0.6, 0.0); + var o = vec3(gray); + if (P.u[0].y != 0u) { + o = hsl_to_rgb(P.f[2].x, P.f[2].y, min(gray, 1.0)); + } + px = vec4(o * a, a); + } + } + case 4u: { + if (a > 0.0) { + let c = px.xyz / a; + let l = clamp(luminance(c), 0.0, 1.0); + var t: vec3; + if (l < 0.5) { + t = fxc_mix(P.f[2].xyz, P.f[1].xyz, l * 2.0); + } else { + t = fxc_mix(P.f[1].xyz, P.f[0].xyz, (l - 0.5) * 2.0); + } + px = vec4(fxc_mix(c, t, P.f[3].x) * a, a); + } + } + case 5u: { + if (a > 0.0) { + let c = px.xyz / a; + var ph = fract_euclid(fxc_colorama_phase(c, a, P.u[0].y) * P.f[0].x + P.f[0].y); + if (P.u[0].z == 0u) { + ph = floor(ph * 6.0) / 6.0; + } + let o = fxc_mix(c, fxc_hsv_to_rgb(ph, 1.0, 1.0), P.f[0].z); + px = vec4(o * a, a); + } + } + case 6u: { + if (a > 0.0) { + let c = px.xyz / a; + var o: vec3; + for (var i = 0; i < 3; i++) { + let r = P.f[i]; + o[i] = max(c.x * r.x + c.y * r.y + c.z * r.z + r.w, 0.0); + } + if (P.u[0].y != 0u) { + o = vec3(o.x); + } + px = vec4(o * a, a); + } + } + case 7u: { + // color2::map_ca: fully transparent black stays. + if (a <= 0.0 && px.x == 0.0 && px.y == 0.0 && px.z == 0.0) { + textureStore(out, p, px); + return; + } + let c = fxc_unpremul(px); + var w = fxc_sc_weights(c); + var o = c; + let relative = P.u[0].y != 0u; + for (var r = 0u; r < 9u; r++) { + let wr = w[r]; + if (wr <= 0.0) { + continue; + } + let k = data[r * 4u + 3u]; + for (var i = 0u; i < 3u; i++) { + let ink = 1.0 - c[i]; + var delta = data[r * 4u + i] + k; + if (relative) { + delta = delta * max(ink, 0.0); + } + o[i] -= wr * delta; + } + } + let na = clamp(max(a, 0.0), 0.0, 1.0); + px = vec4(max(o, vec3(0.0)) * na, na); + } + case 8u: { + let c = fxc_unpremul(px); + let key = P.f[0].xyz; + let tol = P.f[0].w; + let soft = P.f[1].x; + var d: f32; + switch P.u[0].y { + case 1u: { + let hsl = rgb_to_hsl(c); + if (hsl.y < 0.02) { + d = 1.0; + } else { + let dh = abs(hsl.x - P.f[1].w); + d = min(dh, 1.0 - dh) * 2.0; + } + } + case 2u: { + let s = c.x + c.y + c.z; + var pc = vec2(1.0 / 3.0, 1.0 / 3.0); + if (s > 1e-6) { + pc = vec2(c.x / s, c.y / s); + } + let dx = pc.x - P.f[1].y; + let dy = pc.y - P.f[1].z; + d = sqrt(dx * dx + dy * dy) / 1.4142135; + } + default: { + let e = c - key; + d = sqrt(e.x * e.x + e.y * e.y + e.z * e.z) / 1.7320508; + } + } + var v: f32; + if (soft > 1e-6) { + v = clamp((d - tol) / soft, 0.0, 1.0); + } else if (d > tol) { + v = 1.0; + } else { + v = 0.0; + } + if (P.u[0].z != 0u) { + v = 1.0 - v; + } + let k = clamp(v, 0.0, 1.0); + if (P.u[0].w == 2u) { + let m = px.w * k; + px = vec4(m, m, m, 1.0); + } else { + px = px * k; + } + } + default: {} + } + textureStore(out, p, px); +} + +// ---------------------------------------------------------------- Lumetri Color (color3) + +fn fxc_tone_curve(x: f32, s: f32, m: f32, h: f32) -> f32 { + if (s == 0.0 && m == 0.0 && h == 0.0) { + return x; + } + let t = clamp(x, 0.0, 1.0); + let d0 = (t - 0.25) / 0.25; + let d1 = (t - 0.5) / 0.25; + let d2 = (t - 0.75) / 0.25; + let b0 = max(1.0 - d0 * d0, 0.0); + let b1 = max(1.0 - d1 * d1, 0.0); + let b2 = max(1.0 - d2 * d2, 0.0); + return x + 0.25 * (s * (b0 * b0) + m * (b1 * b1) + h * (b2 * b2)); +} + +fn fxc_d3(i: u32) -> vec3 { + return vec3(data[i], data[i + 1u], data[i + 2u]); +} + +// data: 0 temperature, 1 tint, 2 exposure gain, 3 contrast, 4 highlights, 5 shadows, 6 whites, +// 7 blacks, 8 saturation, 9 look intensity, 10 faded film, 11 vibrance, 12 creative saturation, +// 13 tint balance pivot, 14..17 shadow tint, 17..20 highlight tint, 20..32 curves (master, red, +// green, blue × shadows / midtones / highlights), 32..41 wheels (shadows, midtones, highlights), +// 41 vignette amount, 42 vignette start, 43 roundness, 44 feather, 45 cx, 46 cy, 47 lw, 48 lh, +// 49 aspect. +@compute @workgroup_size(16, 16) +fn fxc_lumetri(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + var px = textureLoad(src, p, 0); + let a = px.w; + if (a <= 0.0) { + textureStore(out, p, px); + return; + } + var c = fxc_unpremul(px); + let temp = data[0]; + let tint = data[1]; + if (temp != 0.0 || tint != 0.0) { + c = vec3(c.x * (1.0 + 0.25 * temp), c.y * (1.0 - 0.25 * tint), c.z * (1.0 - 0.25 * temp)); + } + let expo = data[2]; + if (expo != 1.0) { + c = c * expo; + } + let contrast = data[3]; + if (contrast != 0.0) { + c = 0.5 + (c - 0.5) * (1.0 + contrast); + } + let hi = data[4]; + let sh = data[5]; + let wh = data[6]; + let bl = data[7]; + if (hi != 0.0 || sh != 0.0 || wh != 0.0 || bl != 0.0) { + let l = max(luminance(c), 0.0); + let lt = min(l, 1.0); + let il = 1.0 - lt; + let d = 0.25 * hi * fxc_smoothstep(0.5, 1.0, lt) + 0.25 * sh * (1.0 - fxc_smoothstep(0.0, 0.5, lt)) + 0.2 * wh * lt * lt + 0.2 * bl * (il * il); + c = c + d; + } + let l0 = luminance(c); + let sat = data[8]; + if (sat != 1.0) { + c = l0 + (c - l0) * sat; + } + let intensity = data[9]; + let faded = data[10]; + let vib = data[11]; + let csat = data[12]; + let st = fxc_d3(14u); + let ht = fxc_d3(17u); + let zero = vec3(0.0); + if (intensity != 0.0 && (faded != 0.0 || vib != 0.0 || csat != 1.0 || any(st != zero) || any(ht != zero))) { + var d = c; + if (faded != 0.0) { + d = d * (1.0 - 0.25 * faded) + 0.12 * faded; + } + let l = luminance(d); + if (vib != 0.0) { + let mx = max(max(d.x, d.y), d.z); + let mn = min(min(d.x, d.y), d.z); + var s = 0.0; + if (mx > 1e-6) { + s = (mx - mn) / mx; + } + let k = 1.0 + vib * (1.0 - clamp(s, 0.0, 1.0)); + d = l + (d - l) * k; + } + if (csat != 1.0) { + d = l + (d - l) * csat; + } + let pivot = data[13]; + let lt = clamp(l, 0.0, 1.0); + let wsh = 1.0 - fxc_smoothstep(0.0, pivot, lt); + let whi = fxc_smoothstep(pivot, 1.0, lt); + d = d + 0.5 * (st * wsh + ht * whi); + c = c + (d - c) * intensity; + } + for (var i = 0u; i < 3u; i++) { + c[i] = fxc_tone_curve(c[i], data[20], data[21], data[22]); + } + for (var i = 0u; i < 3u; i++) { + let b = 23u + i * 3u; + c[i] = fxc_tone_curve(c[i], data[b], data[b + 1u], data[b + 2u]); + } + let ws = fxc_d3(32u); + let wm = fxc_d3(35u); + let wh3 = fxc_d3(38u); + if (any(ws != zero) || any(wm != zero) || any(wh3 != zero)) { + let lt = clamp(luminance(c), 0.0, 1.0); + let ka = 1.0 - fxc_smoothstep(0.0, 0.5, lt); + let kz = fxc_smoothstep(0.5, 1.0, lt); + let km = 1.0 - ka - kz; + c = c + 0.5 * (ws * ka + wm * km + wh3 * kz); + } + let vig = data[41]; + if (vig != 0.0) { + let x = f32(p.x) + 0.5; + let y = f32(p.y) + 0.5; + let lw = data[47]; + let lh = data[48]; + var nx = (x - data[45]) / max(lw * 0.5, 1e-6); + var ny = (y - data[46]) / max(lh * 0.5, 1e-6); + let vround = data[43]; + let aspect = data[49]; + if (vround > 0.0) { + nx = nx * (1.0 + (max(aspect, 1.0) - 1.0) * vround); + ny = ny * (1.0 + (max(1.0 / aspect, 1.0) - 1.0) * vround); + } + let pw = 2.0 + max(-vround, 0.0) * 6.0; + let r = powz(powz(abs(nx), pw) + powz(abs(ny), pw), 1.0 / pw) / 1.4142135; + let start = data[42]; + let f = fxc_smoothstep(start, start + 0.05 + data[44] * (1.05 - start), r); + if (vig > 0.0) { + c = c + (1.0 - c) * min(vig * 0.2 * f, 1.0); + } else { + let k = 1.0 + vig * 0.2 * f; + c = c * max(k, 0.0); + } + } + c = max(c, zero); + textureStore(out, p, vec4(c * a, a)); +} + +// Straight-colour luminance plane (util::Plane::luma) in x. +@compute @workgroup_size(16, 16) +fn fxc_luma(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let px = textureLoad(src, p, 0); + textureStore(out, p, vec4(luminance(fxc_unpremul(px)), 0.0, 0.0, 0.0)); +} + +// Lumetri Sharpen: `src` + (luma − blurred luma `aux`.x) × amount. f[0].x = sharpen × 2. +@compute @workgroup_size(16, 16) +fn fxc_sharpen(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let px = textureLoad(src, p, 0); + let a = px.w; + if (a <= 0.0) { + textureStore(out, p, px); + return; + } + let c = fxc_unpremul(px); + let d = (luminance(c) - textureLoad(aux, p, 0).x) * P.f[0].x; + textureStore(out, p, vec4(max(c + d, vec3(0.0)) * a, a)); +} + +// ---------------------------------------------------------------- planes (util) + +// One morphology pass (util::minmax_row, edges repeated) of radius u[0].x along rows (u[0].y = 0) +// or columns; u[1] = per-channel operation: 0 min, 1 max, 2 copy. +@compute @workgroup_size(16, 16) +fn fxc_morph(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let dims = out_dims(); + let r = i32(P.u[0].x); + let vertical = P.u[0].y != 0u; + let modes = P.u[1]; + let centre = textureLoad(src, p, 0); + var lo = centre; + var hi = centre; + for (var k = -r; k <= r; k++) { + var q = p; + if (vertical) { + q.y = clamp(p.y + k, 0, dims.y - 1); + } else { + q.x = clamp(p.x + k, 0, dims.x - 1); + } + let v = textureLoad(src, q, 0); + lo = min(lo, v); + hi = max(hi, v); + } + var o = centre; + for (var i = 0; i < 4; i++) { + if (modes[i] == 0u) { + o[i] = lo[i]; + } else if (modes[i] == 1u) { + o[i] = hi[i]; + } + } + textureStore(out, p, o); +} + +// `src` + (`aux` − `src`) × f[0].x (Plane::zip_map lerp). +@compute @workgroup_size(16, 16) +fn fxc_lerp(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let x = textureLoad(src, p, 0); + let y = textureLoad(aux, p, 0); + textureStore(out, p, x + (y - x) * P.f[0].x); +} + +// ---------------------------------------------------------------- Key Light (keylight.rs) + +fn fxc_neutralise(c: vec3, bias: vec3) -> vec3 { + let l = max(luminance(bias), 1e-4); + return vec3(c.x * l / max(bias.x, 1e-4), c.y * l / max(bias.y, 1e-4), c.z * l / max(bias.z, 1e-4)); +} + +fn fxc_screen_diff(c: vec3, ix: vec3, bal: f32) -> f32 { + return c[ix.x] - (bal * c[ix.y] + (1.0 - bal) * c[ix.z]); +} + +fn fxc_correct(c0: vec3, sat: f32, contrast: f32, bright: f32) -> vec3 { + let l = luminance(c0); + let c = l + (c0 - l) * sat; + return ((c - 0.5) * contrast + 0.5) * bright; +} + +// Raw screen matte of `src` (the key source): out = (clipped, clipped, raw, 0). +// u[0].xyz = primary, other channels; f[0] = (alpha bias, screen difference); +// f[1] = (gain, balance, clip black, clip white). +@compute @workgroup_size(16, 16) +fn fxc_kl_raw(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let c = fxc_unpremul(textureLoad(src, p, 0)); + let d = fxc_screen_diff(fxc_neutralise(c, P.f[0].xyz), P.u[0].xyz, P.f[1].y); + let raw = clamp(1.0 - P.f[1].x * d / P.f[0].w, 0.0, 1.0); + let cb = P.f[1].z; + let m = clamp((raw - cb) / max(P.f[1].w - cb, 1e-4), 0.0, 1.0); + textureStore(out, p, vec4(m, m, raw, 0.0)); +} + +// Clip Rollback: `src` = (shrunk, grown) clipped matte, `aux` = raw planes (clipped, _, raw). +@compute @workgroup_size(16, 16) +fn fxc_kl_rollback(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let sg = textureLoad(src, p, 0); + let r = textureLoad(aux, p, 0); + var m = r.x; + if (sg.y > sg.x + 1e-4) { + m = min(max(m, r.z), sg.y); + } + textureStore(out, p, vec4(m, m, r.z, 0.0)); +} + +// Planes for the final pass: (screen matte from `src`.x, raw from `aux`.z, edge 0, 0). +@compute @workgroup_size(16, 16) +fn fxc_kl_pack(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + textureStore(out, p, vec4(textureLoad(src, p, 0).x, textureLoad(aux, p, 0).z, 0.0, 0.0)); +} + +// Combined matte from the screen matte and the source alpha (no inside / outside masks). +fn fxc_kl_combined(sm: f32, src_a: f32, mode: u32) -> f32 { + var a = sm; + let sa = clamp(src_a, 0.0, 1.0); + var ins = 0.0; + if (mode == 1u) { + ins = max(ins, sa); + } + a = max(a, ins); + if (mode == 2u) { + a = a * sa; + } + return clamp(a, 0.0, 1.0); +} + +// Edge band of the combined matte: `src` = source, `aux` = planes; u[0].x = source alpha mode. +@compute @workgroup_size(16, 16) +fn fxc_kl_band(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let a = fxc_kl_combined(textureLoad(aux, p, 0).x, textureLoad(src, p, 0).w, P.u[0].x); + textureStore(out, p, vec4(select(0.0, 1.0, a > 1e-3 && a < 0.999), 0.0, 0.0, 0.0)); +} + +// Planes `src` with the edge weight from the processed band `aux`.x; f[0].x = hardness. +@compute @workgroup_size(16, 16) +fn fxc_kl_edge(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let pl = textureLoad(src, p, 0); + let e = min(textureLoad(aux, p, 0).x * (1.0 + P.f[0].x * 4.0), 1.0); + textureStore(out, p, vec4(pl.x, pl.y, e, 0.0)); +} + +fn fxc_kl_replace(fg: vec3, c: vec3, method: u32, rc: vec3, amt: f32) -> vec3 { + if (amt <= 0.0) { + return fg; + } + switch method { + case 1u: { return fg + (c - fg) * amt; } + case 2u: { return rc; } + case 3u: { + let l = luminance(c) / max(luminance(rc), 1e-4); + return fg + (rc * l - fg) * amt; + } + default: { return fg; } + } +} + +// Final pass: `src` = source, `aux` = planes (screen matte, raw, edge). +// u[0] = (primary, others, view); u[1] = (replace method, foreground cc, unpremultiply, +// source alpha mode); f[0] = (screen, balance); f[1] = (despill bias, l0); f[2] = replace colour; +// f[3] = fg cc (sat, contrast, bright); f[4] = edge cc; f[5] = crops (l, r, t, b); +// f[6] = (offset, scale, _); f[7] = (layer w, layer h). +@compute @workgroup_size(16, 16) +fn fxc_kl_final(@builtin(global_invocation_id) gid: vec3) { + let p = fxc_pixel(gid); + if (p.x < 0) { + return; + } + let s = textureLoad(src, p, 0); + let pl = textureLoad(aux, p, 0); + let ix = P.u[0].xyz; + let view = P.u[0].w; + let c = fxc_unpremul(s); + let sm = pl.x; + let raw = pl.y; + let edge = pl.z; + let a = fxc_kl_combined(sm, s.w, P.u[1].w); + let screen = P.f[0].xyz; + let bal = P.f[0].w; + let dbias = P.f[1].xyz; + let fg_cc = P.u[1].y != 0u; + let ar = max(raw, 1e-4); + var fg = max((c - (1.0 - ar) * screen) / ar, vec3(0.0)); + let spill = max(fxc_screen_diff(fxc_neutralise(fg, dbias), ix, bal), 0.0); + fg[ix.x] -= spill / P.f[1].w; + let intermediate = fg; + fg = fxc_kl_replace(fg, c, P.u[1].x, P.f[2].xyz, max(sm - raw, 0.0)); + if (fg_cc) { + fg = fxc_correct(fg, P.f[3].x, P.f[3].y, P.f[3].z); + } + if (edge > 0.0) { + let ce = fxc_correct(fg, P.f[4].x, P.f[4].y, P.f[4].z); + fg = fg + (ce - fg) * edge; + } + var o: vec4; + switch view { + case 1u: { o = vec4(s.w, s.w, s.w, 1.0); } + case 2u: { + var cs = fxc_neutralise(c, dbias); + let s2 = max(fxc_screen_diff(cs, ix, bal), 0.0); + cs[ix.x] -= s2; + if (fg_cc) { + cs = fxc_correct(cs, P.f[3].x, P.f[3].y, P.f[3].z); + } + o = vec4(cs, 1.0); + } + case 3u: { o = vec4(edge, edge, edge, 1.0); } + case 4u: { o = vec4(sm, sm, sm, 1.0); } + case 5u, 6u: { o = vec4(0.0, 0.0, 0.0, 1.0); } + case 7u: { o = vec4(a, a, a, 1.0); } + case 8u: { + var v = 0.5; + if (a <= 1e-3) { + v = 0.0; + } else if (a >= 0.999) { + v = 1.0; + } + o = vec4(v, v, v, 1.0); + } + case 9u: { o = vec4(intermediate * sm, sm); } + default: { + let x = f32(p.x) + 0.5; + let y = f32(p.y) + 0.5; + let lx = (x - P.f[6].x) / P.f[6].z / P.f[7].x; + let ly = (y - P.f[6].y) / P.f[6].z / P.f[7].y; + let cr = P.f[5]; + let cropped = lx < cr.x || lx > 1.0 - cr.y || ly < cr.z || ly > 1.0 - cr.w; + if (cropped) { + o = vec4(0.0); + } else if (P.u[1].z != 0u) { + o = vec4(fg * a, a); + } else { + o = vec4(fg * a * a, a); + } + } + } + textureStore(out, p, o); +} diff --git a/crates/gpu/src/tests_fx_color.rs b/crates/gpu/src/tests_fx_color.rs new file mode 100644 index 00000000..8fb455b8 --- /dev/null +++ b/crates/gpu/src/tests_fx_color.rs @@ -0,0 +1,163 @@ +//! GPU colour-family effects vs the CPU effects (the oracle): direct on a buffer at full and +//! half resolution, as adjustment, and composited at 8 and 32 bpc. + +use effectcraft_keyframe::Value; + +use crate::tests::{c, effect_case, n}; + +fn e(v: u32) -> Value { + Value::Enum(v) +} + +fn on() -> Value { + Value::Bool(true) +} + +#[test] +fn color_balance_vibrance_tritone_mixer() { + effect_case("ec.color.colorbalance", &[("shadowRed", n(40.0)), ("midGreen", n(-30.0)), ("hiBlue", n(60.0))]); + effect_case("ec.color.colorbalance", &[("shadowBlue", n(-50.0)), ("midRed", n(70.0)), ("hiGreen", n(25.0)), ("preserveLuminosity", on())]); + effect_case("ec.color.vibrance", &[("vibrance", n(60.0)), ("saturation", n(-20.0))]); + effect_case("ec.color.vibrance", &[("vibrance", n(-40.0)), ("saturation", n(35.0))]); + effect_case("ec.color.tritone", &[("midtones", c(0.6, 0.3, 0.5)), ("shadows", c(0.0, 0.1, 0.3)), ("blend", n(30.0))]); + effect_case("ec.color.tritone", &[("highlights", c(1.0, 0.9, 0.6))]); + effect_case("ec.color.channelmixer", &[("rr", n(80.0)), ("rg", n(30.0)), ("gb", n(-40.0)), ("bc", n(10.0)), ("br", n(50.0))]); + effect_case("ec.color.channelmixer", &[("rr", n(40.0)), ("rg", n(40.0)), ("rb", n(20.0)), ("rc", n(-5.0)), ("monochrome", on())]); +} + +#[test] +fn black_white_colorama_selective_color() { + effect_case("ec.color.blackwhite", &[]); + effect_case("ec.color.blackwhite", &[("reds", n(120.0)), ("blues", n(-50.0)), ("cyans", n(200.0)), ("tint", on())]); + effect_case("ec.color.colorama", &[]); + effect_case("ec.color.colorama", &[("inputPhase/mode", e(4)), ("inputPhase/phaseShift", n(75.0)), ("outputCycle/cycles", n(2.5)), ("blend", n(40.0))]); + effect_case("ec.color.colorama", &[("inputPhase/mode", e(8)), ("outputCycle/cycles", n(1.3))]); + effect_case("ec.color.selectivecolor", &[("redsCyan", n(60.0)), ("bluesYellow", n(-40.0)), ("neutralsBlack", n(20.0)), ("whitesMagenta", n(30.0))]); + effect_case( + "ec.color.selectivecolor", + &[("method", e(1)), ("greensMagenta", n(50.0)), ("yellowsCyan", n(-30.0)), ("blacksYellow", n(40.0)), ("magentasBlack", n(25.0))], + ); +} + +#[test] +fn lumetri() { + effect_case( + "ec.color.lumetri", + &[ + ("basicCorrection/whiteBalance/temperature", n(30.0)), + ("basicCorrection/whiteBalance/tint", n(-20.0)), + ("basicCorrection/tone/exposure", n(0.5)), + ("basicCorrection/tone/contrast", n(25.0)), + ("basicCorrection/tone/highlights", n(-40.0)), + ("basicCorrection/tone/shadows", n(30.0)), + ("basicCorrection/tone/whites", n(10.0)), + ("basicCorrection/tone/blacks", n(-15.0)), + ("basicCorrection/saturation", n(130.0)), + ], + ); + effect_case( + "ec.color.lumetri", + &[ + ("creative/lookIntensity", n(80.0)), + ("creative/adjustments/fadedFilm", n(30.0)), + ("creative/adjustments/vibrance", n(40.0)), + ("creative/adjustments/creativeSaturation", n(70.0)), + ("creative/adjustments/shadowTint", c(0.3, 0.5, 0.7)), + ("creative/adjustments/highlightTint", c(0.7, 0.55, 0.4)), + ("creative/adjustments/tintBalance", n(-30.0)), + ("curves/curveMasterMidtones", n(30.0)), + ("curves/curveRedShadows", n(-20.0)), + ("curves/curveBlueHighlights", n(40.0)), + ], + ); + effect_case( + "ec.color.lumetri", + &[ + ("colorWheels/shadowsWheel", c(0.4, 0.5, 0.6)), + ("colorWheels/highlightsWheel", c(0.6, 0.5, 0.45)), + ("vignette/vignetteAmount", n(-2.0)), + ("vignette/vignetteMidpoint", n(30.0)), + ("vignette/vignetteRoundness", n(40.0)), + ("creative/adjustments/sharpen", n(50.0)), + ], + ); + effect_case( + "ec.color.lumetri", + &[ + ("vignette/vignetteAmount", n(1.5)), + ("vignette/vignetteRoundness", n(-60.0)), + ("vignette/vignetteFeather", n(20.0)), + ("creative/adjustments/sharpen", n(-40.0)), + ], + ); +} + +#[test] +fn linear_color_key() { + effect_case("ec.key.linearcolor", &[("keyColor", c(0.6, 0.5, 0.4)), ("tolerance", n(15.0)), ("softness", n(20.0))]); + effect_case("ec.key.linearcolor", &[("keyColor", c(0.2, 0.8, 0.3)), ("matchColors", e(1)), ("keyOperation", e(1)), ("view", e(2))]); + effect_case("ec.key.linearcolor", &[("keyColor", c(0.7, 0.3, 0.6)), ("matchColors", e(2)), ("tolerance", n(5.0)), ("softness", n(30.0))]); +} + +#[test] +fn key_light() { + let screen = ("screenColour", c(0.3, 0.75, 0.35)); + effect_case("ec.keying.keylight", std::slice::from_ref(&screen)); + effect_case( + "ec.keying.keylight", + &[ + // A low gain against a pure screen keeps the raw matte away from 0 (the screen + // removal divides by it; near 0 it amplifies the blurs' float rounding). + ("screenColour", c(0.0, 1.0, 0.0)), + ("screenGain", n(60.0)), + ("screenBalance", n(30.0)), + ("screenPreblur", n(3.0)), + ("screenMatte/clipBlack", n(10.0)), + ("screenMatte/clipWhite", n(80.0)), + ("screenMatte/clipRollback", n(2.5)), + ("screenMatte/screenSoftness", n(3.0)), + ("screenMatte/replaceMethod", e(3)), + ("screenMatte/replaceColour", c(0.6, 0.4, 0.3)), + ("unpremultiplyResult", Value::Bool(false)), + ], + ); + effect_case( + "ec.keying.keylight", + &[ + ("screenColour", c(0.2, 0.4, 0.8)), + ("lockBiasesTogether", Value::Bool(false)), + ("alphaBias", c(0.5, 0.45, 0.6)), + ("despillBias", c(0.55, 0.5, 0.45)), + ("screenMatte/clipWhite", n(70.0)), + ("screenMatte/screenDespotBlack", n(2.0)), + ("screenMatte/screenDespotWhite", n(1.5)), + ("screenMatte/screenShrinkGrow", n(-1.5)), + ("screenMatte/replaceMethod", e(2)), + ("foregroundColourCorrection/enableColourCorrection", on()), + ("foregroundColourCorrection/saturation", n(120.0)), + ("foregroundColourCorrection/contrast", n(15.0)), + ("edgeColourCorrection/enableEdgeColourCorrection", on()), + ("edgeColourCorrection/edgeSoftness", n(2.0)), + ("edgeColourCorrection/edgeHardness", n(30.0)), + ("edgeColourCorrection/edgeBrightness", n(-20.0)), + ("sourceCrops/cropLeft", n(13.0)), + ("sourceCrops/cropBottom", n(7.0)), + ("insideMask/sourceAlpha", e(1)), + ], + ); + // Soft Colour without Screen Softness (after the blur, the screen-removed foreground of + // nearly transparent pixels, divided by a raw matte near 1e-4, amplifies float rounding). + effect_case( + "ec.keying.keylight", + &[ + screen.clone(), + ("screenMatte/clipBlack", n(15.0)), + ("screenMatte/clipRollback", n(1.5)), + ("screenMatte/replaceColour", c(0.6, 0.4, 0.3)), + ("insideMask/sourceAlpha", e(0)), + ], + ); + for view in [1, 2, 4, 7, 9] { + effect_case("ec.keying.keylight", &[screen.clone(), ("view", e(view)), ("screenMatte/screenShrinkGrow", n(1.0)), ("screenMatte/replaceMethod", e(1))]); + } +} diff --git a/docs/effects.md b/docs/effects.md index 75d32478..dd479f0b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -5,7 +5,7 @@ EffectCraft ships 303 effects with 2434 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. -- **GPU**: 24 effects also run on the GPU compositor with identical results. +- **GPU**: 34 effects also run on the GPU compositor with identical results. - **32**: 303 effects process 32-bit float (HDR, overbright) pixels without clamping. - **Status**: 261 implemented in full, 42 partial (what is missing is listed). @@ -88,7 +88,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Auto Color | `ec.color.autocolor` | 6 | | 32 | Implemented | | Auto Contrast | `ec.color.autocontrast` | 5 | | 32 | Implemented | | Auto Levels | `ec.color.autolevels` | 5 | | 32 | Implemented | -| Black & White | `ec.color.blackwhite` | 8 | | 32 | Implemented | +| Black & White | `ec.color.blackwhite` | 8 | GPU | 32 | Implemented | | Brightness & Contrast | `ec.color.brightnesscontrast` | 3 | GPU | 32 | Implemented | | Broadcast Colors | `ec.color.broadcast` | 3 | | 32 | Implemented | | CC Color Neutralizer | `ec.color.cccolorneutralizer` | 11 | | 32 | Implemented | @@ -97,12 +97,12 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Toner | `ec.color.cctoner` | 7 | | 32 | Implemented | | Change Color | `ec.color.changecolor` | 9 | | 32 | Implemented | | Change to Color | `ec.color.changetocolor` | 9 | | 32 | Implemented | -| Channel Mixer | `ec.color.channelmixer` | 13 | | 32 | Implemented | -| Color Balance | `ec.color.colorbalance` | 10 | | 32 | Implemented | +| Channel Mixer | `ec.color.channelmixer` | 13 | GPU | 32 | Implemented | +| Color Balance | `ec.color.colorbalance` | 10 | GPU | 32 | Implemented | | Color Balance (HLS) | `ec.color.colorbalancehls` | 3 | | 32 | Implemented | | Color Link | `ec.color.colorlink` | 6 | | 32 | Implemented | | Color Stabilizer | `ec.color.colorstabilizer` | 6 | | 32 | Implemented | -| Colorama | `ec.color.colorama` | 5 | | 32 | Partial: no Add Phase, editable Output Cycle, Modify, Pixel Selection or Masking | +| Colorama | `ec.color.colorama` | 5 | GPU | 32 | Partial: no Add Phase, editable Output Cycle, Modify, Pixel Selection or Masking | | Curves | `ec.color.curves` | 0 | GPU | 32 | Partial: no Channel popup | | Equalize | `ec.color.equalize` | 2 | | 32 | Implemented | | Exposure | `ec.color.exposure` | 14 | GPU | 32 | Implemented | @@ -111,7 +111,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Leave Color | `ec.color.leavecolor` | 5 | | 32 | Implemented | | Levels | `ec.color.levels` | 7 | GPU | 32 | Partial: master channel only (no Channel popup) | | Levels (Individual Controls) | `ec.color.levelsic` | 27 | | 32 | Implemented | -| Lumetri Color | `ec.color.lumetri` | 38 | | 32 | Partial: no HSL Secondary, hue/saturation curves, HDR mode or look files | +| Lumetri Color | `ec.color.lumetri` | 38 | GPU | 32 | Partial: no HSL Secondary, hue/saturation curves, HDR mode or look files | | OCIO CDL Transform | `ec.color.ociocdl` | 12 | | 32 | Implemented | | OCIO Color Space Transform | `ec.color.ociocolorspace` | 4 | | 32 | Implemented | | OCIO Display Transform | `ec.color.ociodisplay` | 5 | | 32 | Implemented | @@ -119,11 +119,11 @@ Effects are addressed by id (`ec..`) or display name in commands | OCIO Look Transform | `ec.color.ociolook` | 6 | | 32 | Implemented | | PS Arbitrary Map | `ec.color.psarbitrarymap` | 2 | | 32 | Implemented | | Photo Filter | `ec.color.photofilter` | 4 | | 32 | Implemented | -| Selective Color | `ec.color.selectivecolor` | 37 | | 32 | Partial: simplified Colors / Details layout | +| Selective Color | `ec.color.selectivecolor` | 37 | GPU | 32 | Partial: simplified Colors / Details layout | | Shadow/Highlight | `ec.color.shadowhighlight` | 14 | | 32 | Implemented | | Tint | `ec.color.tint` | 3 | GPU | 32 | Implemented | -| Tritone | `ec.color.tritone` | 4 | | 32 | Implemented | -| Vibrance | `ec.color.vibrance` | 2 | | 32 | Implemented | +| Tritone | `ec.color.tritone` | 4 | GPU | 32 | Implemented | +| Vibrance | `ec.color.vibrance` | 2 | GPU | 32 | Implemented | | Video Limiter | `ec.color.videolimiter` | 5 | | 32 | Implemented | ## Distort @@ -243,8 +243,8 @@ Effects are addressed by id (`ec..`) or display name in commands | Extract | `ec.key.extract` | 6 | | 32 | Implemented | | Inner/Outer Key | `ec.key.innerouter` | 10 | | 32 | Partial: one additional mask per side; no cleanup strokes | | Key Cleaner | `ec.key.keycleaner` | 4 | | 32 | Implemented | -| Key Light | `ec.keying.keylight` | 42 | | 32 | Partial: no Source Crops X/Y Method or Edge Colour; no colour suppression / balancing | -| Linear Color Key | `ec.key.linearcolor` | 6 | | 32 | Implemented | +| Key Light | `ec.keying.keylight` | 42 | GPU | 32 | Partial: no Source Crops X/Y Method or Edge Colour; no colour suppression / balancing | +| Linear Color Key | `ec.key.linearcolor` | 6 | GPU | 32 | Implemented | | Screen Key | `ec.key.screen` | 11 | | 32 | Implemented | | Unmult | `ec.key.unmult` | 6 | | 32 | Implemented | From 01701193c44a3c2bb7a522fc5b564f04e645bffe Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:51:56 -0400 Subject: [PATCH 12/36] M7.7: merge fixes (Exposure param path, remaining stubs list, effects doc) Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/tests_color_view.rs | 2 +- crates/engine/src/tests_stubs.rs | 2 +- docs/effects.md | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/tests_color_view.rs b/crates/engine/src/tests_color_view.rs index 316d2e5a..c0f83d66 100644 --- a/crates/engine/src/tests_color_view.rs +++ b/crates/engine/src/tests_color_view.rs @@ -155,7 +155,7 @@ fn color_engine_aces_working_spaces_and_hdr() { s.execute("layer.newSolid", json!({"color": "#ffffff", "width": 64, "height": 48})).unwrap(); let lid = s.active_comp().unwrap().layers[0].id.0; s.execute("effect.apply", json!({"layers": [lid], "effect": "Exposure"})).unwrap(); - s.execute("prop.set", json!({"layer": lid, "path": "effects/#1/exposure", "value": 2.0})).unwrap(); + s.execute("prop.set", json!({"layer": lid, "path": "effects/#1/master/exposure", "value": 2.0})).unwrap(); let mapped = s.render(cid, Tick::ZERO, RenderOpts::default()).data[0]; assert!(mapped[0] > 0.5 && mapped[0] < 0.99, "tone mapped: {mapped:?}"); s.execute("file.projectSettings", json!({"hdr": "clip"})).unwrap(); diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index b1525969..997552bc 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -332,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "keys.audioToKeyframes"] { + for id in ["layer.autoTrace", "keys.audioToKeyframes"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/docs/effects.md b/docs/effects.md index 75d32478..a7717d75 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 303 effects with 2434 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. -- **GPU**: 24 effects also run on the GPU compositor with identical results. -- **32**: 303 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 261 implemented in full, 42 partial (what is missing is listed). +- **GPU**: 25 effects also run on the GPU compositor with identical results. +- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 262 implemented in full, 42 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -113,8 +113,8 @@ Effects are addressed by id (`ec..`) or display name in commands | Levels (Individual Controls) | `ec.color.levelsic` | 27 | | 32 | Implemented | | Lumetri Color | `ec.color.lumetri` | 38 | | 32 | Partial: no HSL Secondary, hue/saturation curves, HDR mode or look files | | OCIO CDL Transform | `ec.color.ociocdl` | 12 | | 32 | Implemented | -| OCIO Color Space Transform | `ec.color.ociocolorspace` | 4 | | 32 | Implemented | -| OCIO Display Transform | `ec.color.ociodisplay` | 5 | | 32 | Implemented | +| OCIO Color Space Transform | `ec.color.ociocolorspace` | 7 | | 32 | Implemented | +| OCIO Display Transform | `ec.color.ociodisplay` | 9 | | 32 | Implemented | | OCIO File Transform | `ec.color.ociofile` | 4 | | 32 | Implemented | | OCIO Look Transform | `ec.color.ociolook` | 6 | | 32 | Implemented | | PS Arbitrary Map | `ec.color.psarbitrarymap` | 2 | | 32 | Implemented | @@ -182,6 +182,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Layer Control | `ec.control.layer` | 1 | GPU | 32 | Implemented | | Point Control | `ec.control.point` | 1 | GPU | 32 | Implemented | | Slider Control | `ec.control.slider` | 1 | GPU | 32 | Implemented | +| Stereo 3D Controls | `ec.control.stereo3d` | 5 | GPU | 32 | Implemented | ## Generate From c4a8c141b563cad36405448f4c0e0379067d4558 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 02:59:57 -0400 Subject: [PATCH 13/36] M12.7: bench --gpu adds an adjustment-layer comp, speedup and GPU-vs-CPU agreement columns Co-Authored-By: Claude Opus 5.5 --- apps/effectcraft-cli/src/main.rs | 55 +++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 9e1a4a8b..8aa1381b 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -749,6 +749,43 @@ fn time_ms(n: usize, mut f: impl FnMut()) -> f64 { median(&mut v) } +/// `bench --gpu`'s adjustment-layer comp: a copy of the first comp with a full-frame adjustment +/// layer on top running Gaussian Blur, Levels, Vibrance and Glow (all GPU effects). +fn adjustment_bench_comp(p: &mut effectcraft_engine::project::Project) -> Option { + use effectcraft_engine::effects; + use effectcraft_engine::keyframe::Value; + use effectcraft_engine::project::build::{self, Ids}; + use effectcraft_engine::project::{ItemKind, LayerSource, Solid}; + let (_, base) = p.comps().next().map(|(id, c)| (*id, c.clone()))?; + let mut comp = base.clone(); + let (w, h) = (comp.width, comp.height); + let sid = p.add_item( + "Adjustment (bench)", + effectcraft_engine::color::Label::None, + None, + ItemKind::Solid(Solid { color: [1.0; 3], width: w, height: h, pixel_aspect: 1.0 }), + ); + let mut l = build::layer(p, &comp, "Adjustment (bench)", LayerSource::Solid { item: sid }, (w, h), None); + l.switches.adjustment = true; + for (id, vals) in [ + ("ec.blur.gaussian", vec![("blurriness", Value::Scalar(12.0))]), + ("ec.color.levels", vec![("gamma", Value::Scalar(1.3))]), + ("ec.color.vibrance", vec![("vibrance", Value::Scalar(40.0))]), + ("ec.stylize.glow", vec![("threshold", Value::Scalar(55.0))]), + ] { + let spec = effects::find(id)?; + let mut next = p.next_id; + let mut g = effects::instantiate(spec, &mut Ids(&mut next), spec.name, [w as f64, h as f64]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k)?.value = v; + } + l.props.sub_mut("effects")?.children.push(g.into()); + } + comp.layers.insert(0, l); + Some(p.add_item("Adjustment Layers (bench)", effectcraft_engine::color::Label::None, None, ItemKind::Comp(comp.into()))) +} + /// `bench --gpu`: CPU vs GPU ms/frame for every comp (or `--comp`) at Full and Half. "cold" /// renders everything (no layer cache); "warm" reuses the layer cache (playback / scrubbing of /// unchanged layers: compositing cost); "viewer" is the GPU display path without readback. @@ -756,22 +793,30 @@ fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> { use effectcraft_render::Backend; let gpu = effectcraft_gpu::Gpu::headless().ok_or_else(|| Failure::Error("--gpu: no usable GPU adapter".into()))?; let n = args.num("--n")?.unwrap_or(10.0).max(1.0) as usize; + // Every comp, plus an adjustment-layer comp built for the benchmark (the main comp under a + // full-frame adjustment layer with a GPU effect stack). + let mut project = (*s.project).clone(); let comps: Vec = match args.opt("--comp") { Some(_) => s.active_comp_id().into_iter().collect(), - None => s.project.comps().map(|(id, _)| *id).collect(), + None => { + let mut v: Vec = s.project.comps().map(|(id, _)| *id).collect(); + v.extend(adjustment_bench_comp(&mut project)); + v + } }; + let project = &project; eprintln!("GPU: {} — median of {n} runs (ms/frame)", effectcraft_render::Accelerator::name(&gpu)); eprintln!( "{:<28} {:>5} {:>10} {:>9} {:>9} {:>9} {:>9} {:>9} {:>8} {:>9}", "comp", "res", "size", "cpu cold", "gpu cold", "cpu warm", "gpu warm", "gpu view", "speedup", "gpu≠cpu" ); for cid in comps { - let Some(comp) = s.project.comp(cid) else { continue }; - let name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let Some(comp) = project.comp(cid) else { continue }; + let name = project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); let t = Tick::from_seconds_f64(args.num("--time")?.unwrap_or(3.0).min(comp.duration.seconds() * 0.5)); for (label, scale) in [("Full", 1.0), ("Half", 0.5)] { let mk = |backend: Backend, cache: Option<&LayerCache>| { - let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts { scale, backend, ..Default::default() }); + let mut r = Renderer::new(project, s.footage.as_ref(), RenderOpts { scale, backend, ..Default::default() }); r.expr = s.expr.as_deref(); r.cache = cache; r.accel = Some(&gpu); @@ -792,7 +837,7 @@ fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> { std::hint::black_box(mk(Backend::Gpu, Some(&gcache))); }); let view = time_ms(n, || { - let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts { scale, backend: Backend::Gpu, ..Default::default() }); + let mut r = Renderer::new(project, s.footage.as_ref(), RenderOpts { scale, backend: Backend::Gpu, ..Default::default() }); r.expr = s.expr.as_deref(); r.cache = Some(&gcache); r.accel = Some(&gpu); From f85468c6948d1a9a6e20f7234899131b4955c440 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:00:29 -0400 Subject: [PATCH 14/36] M12.7: docs (architecture, parity, gpu crate docs) Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/context.rs | 2 +- crates/gpu/src/lib.rs | 10 ++++++++-- crates/gpu/src/walk.rs | 5 +++-- docs/architecture.md | 26 ++++++++++++++++++++------ docs/parity.md | 13 ++++++------- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index 7dd14ba8..d6f38883 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -289,7 +289,7 @@ impl GpuContext { let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { label: Some("effectcraft gpu"), required_limits, ..Default::default() })) .ok()?; - device.on_uncaptured_error(Arc::new(|e| eprintln!("wgpu: {e}"))); + device.on_uncaptured_error(Arc::new(|e| log::error!("wgpu: {e}"))); GpuContext::new(&adapter, device, queue).map_err(|e| log::info!("gpu: {e}")).ok() } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 2241560b..bf47e3c5 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -6,8 +6,10 @@ //! layer buffers (once per buffer), applies the layer transforms with the CPU's sampling //! (nearest / bilinear / bicubic, minification pre-filter), motion-blur sub-samples, track //! mattes, Preserve Transparency, layer styles' passes, all 38 blend modes and the 8/16 bpc -//! clamping and quantisation, and converts colour spaces. 3D runs, adjustment layers and -//! wireframes run on the CPU between GPU steps (read back, draw, upload). +//! clamping and quantisation, and converts colour spaces. Classic 3D runs composite here too +//! (`classic3d`: per-pixel fragment sort, lights, ray-cast shadows), and adjustment layers run +//! their effect stacks on the GPU-resident comp. Advanced 3D compositing and wireframes run on +//! the CPU between GPU steps (read back, draw, upload). //! //! Advanced 3D comps rasterise on a render pipeline (`advanced3d.wgsl`: depth buffer, PBR, //! image-based light, shadow maps), see [`Accelerator::raster_3d`]. @@ -16,6 +18,10 @@ //! effect's exact steps (padding, box-blur radii, parameter conversions); chains of them are //! uploaded and read back once. //! +//! GPU particles (`particles`): the stepped particle effects hand their simulation to +//! [`effectcraft_effects::psim::ParticleSim`], implemented here with one invocation per particle +//! and GPU-resident checkpoints. +//! //! Plug a [`Gpu`] into [`Renderer::accel`] (it implements [`Accelerator`]); renders then use it //! when [`RenderOpts::backend`](effectcraft_render::RenderOpts) asks for it. The viewer can //! skip readback entirely with [`Gpu::render_display`], which leaves an RGBA8 texture for diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index cfec880d..64a1d2e7 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -1,8 +1,9 @@ //! The per-frame compositing walk on the GPU: the same steps as `Renderer::draw_comp` (layer //! order, 3D runs, collapsed precomps, track mattes, Preserve Transparency, layer styles, motion //! blur, bit-depth quantisation, colour conversions), with layer pixels taken from the CPU -//! renderer and its layer cache. Steps without a GPU implementation (3D runs, adjustment and -//! wireframe layers) read the canvas back, run on the CPU and upload the result. +//! renderer and its layer cache. Classic 3D runs and adjustment layers run on the GPU; steps +//! without a GPU implementation (Advanced 3D runs, wireframes, anything a kernel declines) read +//! the canvas back, run on the CPU and upload the result. use std::sync::Arc; diff --git a/docs/architecture.md b/docs/architecture.md index 39acd396..83c3f3ea 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -157,17 +157,31 @@ accelerator is attached and the project's renderer is the GPU). A GPU frame walk sampling (nearest, bilinear, Catmull-Rom bicubic, the same minification pre-filter), motion-blur sub-samples are accumulated, and track mattes, Preserve Transparency, layer style passes, knockout, all 38 blend modes (a WGSL port of `color::blend`), the 8/16 bpc clamp-and-quantise steps -and colour-space conversions run on the GPU. 3D runs, adjustment layers and wireframes run on the -CPU between GPU steps (read back, draw, upload). GPU effects (`effects::GPU_EFFECTS`: Gaussian, -Fast Box and Directional Blur, Glow, Levels, Curves, Hue/Saturation, Tint, Fill, Gradient Ramp, -Fractal Noise, Drop Shadow, Brightness & Contrast, Exposure, Invert, Transform) repeat the CPU -effect's steps (padding, box radii, parameters) as kernels; consecutive GPU effects run as one +and colour-space conversions run on the GPU. Classic 3D runs composite on the GPU too +(`gpu::classic3d`): the render crate prepares the planes (`Renderer::prepare_3d_run`: layer buffers +with their bokeh depth of field, homographies per motion-blur sub-sample, materials, lights, track +mattes), the GPU packs the buffers into one atlas texture and one kernel does the per-pixel +fragment sort (intersecting planes, coplanar stack order), Blinn-Phong lighting, ray-cast shadows +against the caster planes (Shadow Diffusion, Light Transmission) and blending. Adjustment layers +run their effect stacks on the GPU-resident comp (`Renderer::run_effects_on` with an `FxTarget`): +runs of GPU effects stay on the device and only non-GPU effects read back and upload. Advanced 3D +compositing and wireframes still run on the CPU between GPU steps (read back, draw, upload). GPU +effects (`effects::GPU_EFFECTS`, 58 of them: blurs, colour, keying incl. Key Light, distortion, +transitions, generators, noise and grain; see [effects.md](effects.md)) repeat the CPU effect's +steps (padding, box radii, parameters, hashes) as kernels; consecutive GPU effects run as one chain with one upload and one readback. Tests render scenes on both paths and compare them (≤ 1/255 at 8 bpc, ≤ 1e-3 at 32 bpc); they skip without an adapter. The desktop viewer builds the `Gpu` on egui-wgpu's device and shows frames from GPU textures without reading them back (`ui-egui::frames`); headless renders, the CLI (unless `--gpu`) and CI use the CPU. On the web (WebGPU) the GPU composites viewer frames; steps that need a readback fall back to the CPU (the -Info panel's pixel readout reads GPU frames back asynchronously). +Info panel's pixel readout reads GPU frames back asynchronously). **GPU particles** +(`effects::psim`, `gpu::particles`): CC Particle World, CC Particle Systems II and Particle +Playground cannons without interacting forces evolve every particle independently, so the GPU +backend (reached through `EffectHost::particles` when the GPU compositor is active) simulates one +particle per invocation from the CPU's birth schedule and keeps per-key state checkpoints on the +GPU, like `SimCache`; the CPU simulation stays the oracle (tests compare ids and positions). +`effectcraft-cli bench --gpu` reports CPU vs GPU ms/frame, speed-up and pixel agreement for every +comp plus an adjustment-layer comp. **Colour and bit depth** (`crates/render/src/color.rs`, `crates/color/src/space.rs`). Pixels are `f32`, but 8 and 16 bpc projects clamp and quantise each layer after its source and masks and diff --git a/docs/parity.md b/docs/parity.md index fbe12a5f..f703644a 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -20,13 +20,12 @@ Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking -renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU -rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur and blend modes; a few viewer and colour-management menu items; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and -the "better than After Effects" items (a plug-in API, branching history, GPU particles). +the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -121,11 +120,11 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Text | ≈ 91% | 0.8 | no OpenType feature panel, no extruded strokes, variable-axis animation changes outlines but not advances (vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | | 3D | 78% | 9.3 | multi-view layouts, Advanced 3D motion blur and blend modes (and iris shapes in Advanced 3D's depth of field), cameras/lights from models; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | -| Effects | ≈ 80% | 4.5 | GPU versions of more effects and the missing controls listed as partial in [effects.md](effects.md) (every After Effects effect exists since M9.11, M12.5 and M12.6; parameter names, order, twirl-downs, popups, units and defaults were aligned in M9.12) | +| Effects | ≈ 80% | 4.0 | GPU versions of the remaining effects (58 run on the GPU since M12.7) and the missing controls listed as partial in [effects.md](effects.md) (every After Effects effect exists since M9.11, M12.5 and M12.6; parameter names, order, twirl-downs, popups, units and defaults were aligned in M9.12) | | Interface | 74% | 5.0 | a richer Learn area (Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | -| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -172,8 +171,8 @@ taper/wave, Advanced 3D's DOF has no iris shapes, and variable axes don't change 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10). 2. Effect Controls widgets: angle dial, point crosshair, eyedropper, curves and levels editors. 3. ~~Viewer basics: snapping, rulers, channel view, snapshots, exposure, a drawable region of interest.~~ (M0.13) -4. ~~A GPU (wgpu) compositor, then GPU effects~~ (M12.2: 2D compositing and 16 GPU effects; 3D runs - and adjustment layers still composite on the CPU). +4. ~~A GPU (wgpu) compositor, then GPU effects~~ (M12.2: 2D compositing and 16 GPU effects; M12.7: + Classic 3D runs, adjustment layers, 58 GPU effects and GPU particles). 5. ~~Pen tool for shape paths, shape vertex editing, free transform~~ (M6.5); the Layer viewer. 6. ~~Motion-path handles in the viewer; graph editor transform box and snapping.~~ (M5.8) 7. Point tracking and stabilization (in progress). From b29d833b2d3b9cc69ab11105340829ffee44f71a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:12:36 -0400 Subject: [PATCH 15/36] M13.6: Lumetri Scopes, Footage, Media Browser, Metadata, Progress and Content-Aware Fill panels; Content-Aware Fill, Scene Edit Detection, Auto-trace, Align Video to Data Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/align_data.rs | 256 +++++++++ crates/engine/src/commands/autotrace.rs | 211 +++++++ crates/engine/src/commands/content_fill.rs | 344 ++++++++++++ crates/engine/src/commands/footage_panel.rs | 224 ++++++++ crates/engine/src/commands/frontend.rs | 2 +- crates/engine/src/commands/mod.rs | 14 +- crates/engine/src/commands/panels_cmds.rs | 214 ++++++++ crates/engine/src/commands/scene_detect.rs | 182 ++++++ crates/engine/src/commands/stubs.rs | 8 - crates/engine/src/jobs.rs | 320 +++++++++++ crates/engine/src/lib.rs | 23 + crates/engine/src/media_browser.rs | 270 +++++++++ crates/engine/src/menus.rs | 12 +- crates/engine/src/tests_panels.rs | 518 ++++++++++++++++++ crates/project/src/lib.rs | 4 + crates/ui-egui/src/dock.rs | 34 +- crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/content_fill_panel.rs | 138 +++++ crates/ui-egui/src/panels/footage_panel.rs | 150 +++++ crates/ui-egui/src/panels/forms.rs | 54 ++ crates/ui-egui/src/panels/media_panels.rs | 366 +++++++++++++ crates/ui-egui/src/panels/mod.rs | 13 + crates/ui-egui/src/panels/panel_kit.rs | 70 +++ crates/ui-egui/src/panels/project.rs | 15 + crates/ui-egui/src/panels/scopes_panel.rs | 197 +++++++ crates/ui-egui/src/panels/timeline.rs | 1 + crates/ui-egui/src/state.rs | 4 + crates/ui-egui/tests/ui_panels.rs | 104 ++++ docs/architecture.md | 36 ++ docs/parity.md | 24 +- 30 files changed, 3786 insertions(+), 27 deletions(-) create mode 100644 crates/engine/src/commands/align_data.rs create mode 100644 crates/engine/src/commands/autotrace.rs create mode 100644 crates/engine/src/commands/content_fill.rs create mode 100644 crates/engine/src/commands/footage_panel.rs create mode 100644 crates/engine/src/commands/panels_cmds.rs create mode 100644 crates/engine/src/commands/scene_detect.rs create mode 100644 crates/engine/src/jobs.rs create mode 100644 crates/engine/src/media_browser.rs create mode 100644 crates/engine/src/tests_panels.rs create mode 100644 crates/ui-egui/src/panels/content_fill_panel.rs create mode 100644 crates/ui-egui/src/panels/footage_panel.rs create mode 100644 crates/ui-egui/src/panels/media_panels.rs create mode 100644 crates/ui-egui/src/panels/panel_kit.rs create mode 100644 crates/ui-egui/src/panels/scopes_panel.rs create mode 100644 crates/ui-egui/tests/ui_panels.rs diff --git a/crates/engine/src/commands/align_data.rs b/crates/engine/src/commands/align_data.rs new file mode 100644 index 00000000..1f18de5f --- /dev/null +++ b/crates/engine/src/commands/align_data.rs @@ -0,0 +1,256 @@ +//! Layer ▸ Time ▸ Align Video to Data: move a footage layer in time so its frames line up with +//! the samples of a data file (telemetry, GPS, sensor logs imported as data footage). +//! +//! The data's sample with timestamp `T` sits at comp time `dataStart + (T − T₀)` (`T₀` = the +//! first sample's timestamp; `dataStart` defaults to 0). A video frame at source time `s` was +//! recorded at `V₀ + s`, where `V₀` is the video's start: `videoStart` if given, else the +//! footage file's creation date (desktop). Aligning sets the layer's Start Time to +//! `dataStart + V₀ − T₀`, so the frame recorded at `T` plays when the data says `T`. +//! +//! **Supported data** (File ▸ Import as data footage; also in `docs/architecture.md`): +//! - JSON: an array of objects, or an object holding such an array (the first array of objects +//! found at the top level, e.g. `{"samples": [...]}`); +//! - CSV / TSV with a header row (quoted fields allowed). +//! +//! The time key is `key`, else the first field named `time`, `timestamp`, `t`, `date`, +//! `datetime`, `utc`, `gps_time`, `seconds` or `time_s` (any case). Timestamps may be ISO 8601 +//! date-times (`2026-05-01T10:15:30.5Z`, offsets `±hh:mm`), times of day (`10:15:30.5`), +//! timecode (`10:15:30:12` at the footage's rate), Unix seconds, or Unix milliseconds (numbers +//! above 10¹¹). When the data has only times of day, `V₀` is reduced to its UTC time of day. + +use effectcraft_project::{FootageKind, ItemId, ItemKind}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, f_p, has_layers, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// A parsed timestamp: seconds, and whether it carries a date (absolute) or is a time of day / +/// relative number. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Stamp { + pub secs: f64, + pub dated: bool, +} + +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let mp = (m + 9) % 12; + let doy = (153 * mp + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// `hh:mm:ss(.fff)` or timecode `hh:mm:ss:ff` / `hh:mm:ss;ff` → seconds. +fn clock(s: &str, fps: f64) -> Option { + let parts: Vec<&str> = s.split([':', ';']).collect(); + match parts.as_slice() { + [h, m, sec] => Some(h.trim().parse::().ok()? * 3600.0 + m.parse::().ok()? * 60.0 + sec.parse::().ok()?), + [h, m, sec, f] => { + Some(h.trim().parse::().ok()? * 3600.0 + m.parse::().ok()? * 60.0 + sec.parse::().ok()? + f.parse::().ok()? / fps.max(1.0)) + } + [m, sec] => Some(m.trim().parse::().ok()? * 60.0 + sec.parse::().ok()?), + _ => None, + } +} + +/// Parse a timestamp value (see the module docs). `fps` reads timecode frames. +pub fn parse_stamp(v: &Value, fps: f64) -> Option { + match v { + Value::Number(n) => { + let x = n.as_f64()?; + Some(if x > 1e11 { Stamp { secs: x / 1000.0, dated: true } } else { Stamp { secs: x, dated: x > 1e8 } }) + } + Value::String(s) => { + let s = s.trim(); + if let Ok(x) = s.parse::() { + return parse_stamp(&json!(x), fps); + } + // ISO 8601: date [T| ] time [zone]. + if s.len() >= 10 && s.as_bytes()[4] == b'-' && s.as_bytes()[7] == b'-' { + let (y, m, d) = (s[0..4].parse::().ok()?, s[5..7].parse::().ok()?, s[8..10].parse::().ok()?); + let mut secs = days_from_civil(y, m, d) as f64 * 86_400.0; + let rest = s[10..].trim_start_matches(['T', 't', ' ']); + if !rest.is_empty() { + // Zone: Z or ±hh:mm / ±hhmm. + let (time, zone) = match rest.find(['Z', 'z', '+']).or_else(|| rest.rfind('-')) { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, ""), + }; + secs += clock(time, fps)?; + if zone.len() > 1 { + let sign = if zone.starts_with('-') { -1.0 } else { 1.0 }; + let z = zone[1..].replace(':', ""); + let (zh, zm) = (z.get(0..2)?.parse::().ok()?, z.get(2..4).and_then(|m| m.parse::().ok()).unwrap_or(0.0)); + secs -= sign * (zh * 3600.0 + zm * 60.0); + } + } + return Some(Stamp { secs, dated: true }); + } + clock(s, fps).map(|secs| Stamp { secs, dated: false }) + } + _ => None, + } +} + +const TIME_KEYS: &[&str] = &["time", "timestamp", "t", "date", "datetime", "utc", "gps_time", "gpstime", "seconds", "time_s", "times"]; + +/// Split a CSV/TSV line (quotes allowed). +fn split_row(line: &str, sep: char) -> Vec { + let mut out = vec![]; + let mut cur = String::new(); + let mut q = false; + let mut chars = line.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '"' if q && chars.peek() == Some(&'"') => { + cur.push('"'); + chars.next(); + } + '"' => q = !q, + c if c == sep && !q => out.push(std::mem::take(&mut cur)), + c => cur.push(c), + } + } + out.push(cur); + out.into_iter().map(|s| s.trim().to_string()).collect() +} + +/// The time column of data footage: (key used, timestamps in row order). +pub fn data_times(text: &str, format: &str, key: Option<&str>) -> std::result::Result<(String, Vec), String> { + let pick = |names: &[String]| -> Option { + if let Some(k) = key { + return names.iter().find(|n| n.eq_ignore_ascii_case(k)).cloned(); + } + TIME_KEYS.iter().find_map(|t| names.iter().find(|n| n.eq_ignore_ascii_case(t)).cloned()) + }; + if format == "json" { + let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).map_err(|e| format!("invalid JSON: {e}"))?; + let rows = match &v { + Value::Array(a) => a.clone(), + Value::Object(o) => { + o.values().find_map(|x| x.as_array().filter(|a| a.first().is_some_and(Value::is_object)).cloned()).ok_or("no array of samples in the JSON")? + } + _ => return Err("the JSON holds no samples".into()), + }; + let names: Vec = rows.first().and_then(Value::as_object).map(|o| o.keys().cloned().collect()).unwrap_or_default(); + let k = pick(&names) + .ok_or_else(|| format!("no time field (looked for {}); pass `key`", key.map(|k| k.to_string()).unwrap_or_else(|| TIME_KEYS.join(", "))))?; + let times = rows.iter().filter_map(|r| r.get(&k).cloned()).collect(); + return Ok((k, times)); + } + let sep = if format == "tsv" { '\t' } else { ',' }; + let mut lines = text.trim_start_matches('\u{feff}').lines().filter(|l| !l.trim().is_empty()); + let header = split_row(lines.next().ok_or("the file is empty")?, sep); + let k = pick(&header) + .ok_or_else(|| format!("no time column (looked for {}); pass `key`", key.map(|k| k.to_string()).unwrap_or_else(|| TIME_KEYS.join(", "))))?; + let col = header.iter().position(|h| *h == k).unwrap_or(0); + let times = lines.filter_map(|l| split_row(l, sep).get(col).map(|c| json!(c))).collect(); + Ok((k, times)) +} + +fn data_item(s: &Session, p: &Value) -> Result { + let c = "layer.alignVideoToData"; + let id = match p.get("data") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id), + _ => s.project.items.values().find(|i| matches!(&i.kind, ItemKind::Footage(f) if f.kind == FootageKind::Data)).map(|i| i.id), + } + .ok_or_else(|| bad(c, "no data footage: import a JSON/CSV/TSV file and pass `data`"))?; + Ok(id) +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.alignVideoToData"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let footage = match layer.source.item().and_then(|i| s.project.item(i)).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != FootageKind::Data => f.clone(), + _ => return Err(bad(c, "select a footage layer")), + }; + let fps = footage.frame_rate.as_f64(); + let did = data_item(s, p)?; + let Some(ItemKind::Footage(df)) = s.project.item(did).map(|i| &i.kind) else { return Err(bad(c, "not data footage")) }; + let text = df.data.clone().ok_or_else(|| bad(c, "the data item holds no data"))?; + let format = df.codec.to_ascii_lowercase(); + let (key, raw) = data_times(&text, &format, str_p(p, "key").filter(|k| !k.is_empty())).map_err(|e| bad(c, e))?; + let stamps: Vec = raw.iter().filter_map(|v| parse_stamp(v, fps)).collect(); + let first = *stamps.first().ok_or_else(|| bad(c, format!("no readable timestamps in `{key}`")))?; + let video = match p.get("videoStart").filter(|v| v.as_str() != Some("")) { + Some(v) => parse_stamp(v, fps).ok_or_else(|| bad(c, "videoStart: ISO 8601 date-time, hh:mm:ss, timecode or seconds"))?, + None => { + let (created, _, _) = crate::media_browser::file_dates(&footage.path); + Stamp { secs: created.ok_or_else(|| bad(c, "the footage has no recording time; pass `videoStart`"))? as f64, dated: true } + } + }; + // Compare like with like: times of day when the data has no dates. + let v0 = if video.dated && !first.dated { video.secs.rem_euclid(86_400.0) } else { video.secs }; + let data_start = f_p(p, "dataStart").unwrap_or(0.0); + let fr = comp.frame_rate; + let start = fr.snap_nearest(Tick::from_seconds_f64(data_start + v0 - first.secs)); + let shift = start - layer.start_time; + s.edit("Align Video to Data", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + l.start_time += shift; + l.in_point += shift; + l.out_point += shift; + Ok(()) + })?; + Ok(json!({ + "layer": lid.0, + "startTime": start.seconds(), + "shift": shift.seconds(), + "key": key, + "samples": stamps.len(), + "firstSample": first.secs, + "videoStart": v0, + "dataStart": data_start, + })) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.alignVideoToData", + "Align Video to Data", + ["Layer", "Time"], + None, + "{layer?, data?: data footage id|name, key?: time field, videoStart?: ISO date-time | hh:mm:ss | timecode | seconds (default: file creation time), dataStart? (comp s of the first sample, 0)}", + has_layers, + run + )] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_timestamps() { + let p = |v: Value| parse_stamp(&v, 25.0).unwrap(); + assert_eq!(p(json!("1970-01-02T00:00:10Z")).secs, 86_410.0); + assert_eq!(p(json!("2000-03-01T00:00:00Z")).secs, 951_868_800.0); + assert_eq!(p(json!("1970-01-01T02:00:00+02:00")).secs, 0.0); + assert_eq!(p(json!("1970-01-01 01:00:00.5-01:00")).secs, 7200.5); + assert_eq!(p(json!("10:00:01.25")), Stamp { secs: 36_001.25, dated: false }); + assert_eq!(p(json!("10:00:01:05")).secs, 36_001.2); + assert_eq!(p(json!(1_700_000_000_500u64)), Stamp { secs: 1_700_000_000.5, dated: true }); + assert_eq!(p(json!(12.5)), Stamp { secs: 12.5, dated: false }); + assert_eq!(p(json!("3.5")).secs, 3.5); + assert!(parse_stamp(&json!("soon"), 25.0).is_none()); + } + + #[test] + fn finds_time_columns() { + let (k, t) = data_times("lat,Time,speed\n1,\"10:00:00\",3\n2,10:00:01,4\n", "csv", None).unwrap(); + assert_eq!(k, "Time"); + assert_eq!(t, vec![json!("10:00:00"), json!("10:00:01")]); + let (k, t) = data_times("a\tts\n1\t5\n", "tsv", Some("ts")).unwrap(); + assert_eq!((k.as_str(), t), ("ts", vec![json!("5")])); + let (k, t) = data_times(r#"{"meta": 1, "samples": [{"timestamp": 3, "v": 1}, {"timestamp": 4}]}"#, "json", None).unwrap(); + assert_eq!((k.as_str(), t), ("timestamp", vec![json!(3), json!(4)])); + assert!(data_times("x,y\n1,2\n", "csv", None).is_err()); + } +} diff --git a/crates/engine/src/commands/autotrace.rs b/crates/engine/src/commands/autotrace.rs new file mode 100644 index 00000000..093468b6 --- /dev/null +++ b/crates/engine/src/commands/autotrace.rs @@ -0,0 +1,211 @@ +//! Layer ▸ Auto-trace…: masks from a layer's alpha, a colour channel or its luminance, at the +//! current frame or over the work area (one Mask Path key per frame). The tracing is +//! `effectcraft_path::trace` (marching squares, Douglas–Peucker, Bezier fitting). + +use effectcraft_color::Label; +use effectcraft_keyframe::{Interp, Keyframe}; +use effectcraft_path::trace::{TraceOpts, Traced, trace}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{ItemKind, LayerSource, MaskMode, Solid, Value as KV}; +use effectcraft_raster::Image; +use effectcraft_render::{RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Channel { + Alpha, + Red, + Green, + Blue, + Luminance, +} + +impl Channel { + fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "alpha" | "a" => Channel::Alpha, + "red" | "r" => Channel::Red, + "green" | "g" => Channel::Green, + "blue" | "b" => Channel::Blue, + "luminance" | "luma" | "l" => Channel::Luminance, + _ => return None, + }) + } +} + +/// The traced scalar field of a frame (straight channel values, 0…1). +pub fn channel_field(img: &Image, ch: Channel, invert: bool) -> Vec { + img.data + .iter() + .map(|p| { + let a = p[3]; + let s = |v: f32| if a > 1e-6 { v / a } else { 0.0 }; + let v = match ch { + Channel::Alpha => a, + Channel::Red => s(p[0]), + Channel::Green => s(p[1]), + Channel::Blue => s(p[2]), + Channel::Luminance => 0.2126 * s(p[0]) + 0.7152 * s(p[1]) + 0.0722 * s(p[2]), + } + .clamp(0.0, 1.0); + if invert { 1.0 - v } else { v } + }) + .collect() +} + +/// Trace one frame: outlines in layer space. +fn trace_frame(img: &Image, offset: [f64; 2], ch: Channel, invert: bool, blur: f64, o: &TraceOpts) -> Vec { + let src = if blur > 0.0 { effectcraft_raster::gaussian_blur(img, blur / 2.0, blur / 2.0, false) } else { img.clone() }; + let field = channel_field(&src, ch, invert); + let mut out = trace(&field, img.width as usize, img.height as usize, o); + for t in &mut out { + for v in &mut t.path.vertices { + v[0] += offset[0]; + v[1] += offset[1]; + } + } + out +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.autoTrace"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + if !layer.source.is_av() { + return Err(bad(c, "auto-trace needs a footage, solid, precomp, text or shape layer")); + } + let ch = match str_p(p, "channel") { + Some(x) => Channel::from_name(x).ok_or_else(|| bad(c, "channel: alpha|red|green|blue|luminance"))?, + None => Channel::Alpha, + }; + let invert = b_p(p, "invert").unwrap_or(false); + let blur = f_p(p, "blur").unwrap_or(1.0).max(0.0); + let o = TraceOpts { + threshold: (f_p(p, "threshold").unwrap_or(50.0) / 100.0).clamp(0.0, 1.0) as f32, + tolerance: f_p(p, "tolerance").unwrap_or(1.0).max(0.01), + min_area: f_p(p, "minimumArea").unwrap_or(10.0).max(0.0), + corner_roundness: (f_p(p, "cornerRoundness").unwrap_or(50.0) / 100.0).clamp(0.0, 1.0), + }; + let fr = comp.frame_rate; + let work_area = matches!(str_p(p, "timeSpan"), Some("workArea" | "work area" | "work_area")); + let times: Vec = if work_area { + let a = comp.work_area.0.max(layer.in_point); + let b = comp.work_area.1.min(layer.out_point); + let (fa, fb) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(b - Tick(1))); + (fa..=fb).map(|f| fr.tick_of(f)).collect() + } else { + vec![s.time()] + }; + if times.is_empty() { + return Err(bad(c, "the work area doesn't overlap the layer")); + } + // Trace every frame (in parallel). + let frames: Vec> = { + let mut r = Renderer::new(&s.project, s.footage.as_ref(), RenderOpts::default()); + r.expr = s.expr.as_deref(); + let r = &r; + let (project, expr) = (&s.project, s.expr.as_deref()); + use rayon::prelude::*; + times + .par_iter() + .map(|t| match crate::tracking::source_frame(r, project, cid, &comp, &layer, *t, expr) { + Some((img, off)) => trace_frame(&img, off, ch, invert, blur, &o), + None => vec![], + }) + .collect() + }; + let n_masks = frames.iter().map(Vec::len).max().unwrap_or(0); + if n_masks == 0 { + return Err(bad(c, "nothing to trace: no pixels pass the threshold")); + } + let new_layer = b_p(p, "applyToNewLayer").unwrap_or(false); + let (sw, sh) = effectcraft_render::source_size(&s.project, &layer); + let layer_times: Vec = times.iter().map(|t| layer.layer_time(*t)).collect(); + let target = s.edit("Auto-trace", None, |proj, st| { + let target = if new_layer { + // A solid the size of the layer, in the same place, carries the masks. + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let name = format!("Auto-traced {}", layer.name); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: sw.max(1), height: sh.max(1), pixel_aspect: comp.pixel_aspect }), + ); + let mut l = build::layer(proj, &comp, &name, LayerSource::Solid { item: sid }, (sw.max(1), sh.max(1)), None); + l.start_time = layer.start_time; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.stretch = layer.stretch; + l.parent = layer.parent; + if let (Some(dst), Some(src)) = (l.props.sub_mut("transform"), layer.transform()) { + let mut tr = src.clone(); + tr.reassign_uids(&mut proj.next_id); + *dst = tr; + } + let id = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = cm.layers.iter().position(|x| x.id == lid).unwrap_or(0); + cm.layers.insert(at, l); + id + } else { + lid + }; + let mut next = proj.next_id; + let l = super::layer_mut(proj, cid, target)?; + let masks = l.props.sub_mut("masks").ok_or_else(|| bad(c, "this layer can't have masks"))?; + let base = masks.children.len(); + for k in 0..n_masks { + // Depth decides the mode: outlines add, holes subtract. + let first = frames.iter().find_map(|f| f.get(k)).expect("mask"); + let mode = if first.depth % 2 == 0 { MaskMode::Add } else { MaskMode::Subtract }; + let mode = if k == 0 { MaskMode::Add } else { mode }; + let mut g = build::mask( + &mut Ids(&mut next), + &format!("Auto-trace {}", k + 1), + first.path.clone(), + mode, + build::MASK_COLORS[(base + k) % build::MASK_COLORS.len()], + ); + if frames.len() > 1 { + let path = g.get_mut("path").expect("path"); + for (f, lt) in frames.iter().zip(&layer_times) { + if let Some(tr) = f.get(k) { + effectcraft_keyframe::set_key(&mut path.keys, Keyframe::new(*lt, KV::Path(tr.path.clone()))); + } + } + let op = g.get_mut("opacity").expect("opacity"); + if frames.iter().any(|f| f.get(k).is_none()) { + for (f, lt) in frames.iter().zip(&layer_times) { + let mut key = Keyframe::new(*lt, KV::Scalar(if f.get(k).is_some() { 100.0 } else { 0.0 })); + key.in_interp = Interp::Hold; + key.out_interp = Interp::Hold; + effectcraft_keyframe::set_key(&mut op.keys, key); + } + } + } + masks.children.push(g.into()); + } + proj.next_id = next; + st.selected_layers = vec![target]; + Ok(target) + })?; + Ok(json!({"layer": target.0, "masks": n_masks, "frames": times.len()})) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.autoTrace", + "Auto-trace...", + ["Layer"], + None, + "{layer?, timeSpan?: currentFrame|workArea, channel?: alpha|red|green|blue|luminance, invert?, blur? (px, 1), tolerance? (px, 1), threshold? (%, 50), minimumArea? (px, 10), cornerRoundness? (%, 50), applyToNewLayer?}", + has_layers, + run + )] +} diff --git a/crates/engine/src/commands/content_fill.rs b/crates/engine/src/commands/content_fill.rs new file mode 100644 index 00000000..b030a991 --- /dev/null +++ b/crates/engine/src/commands/content_fill.rs @@ -0,0 +1,344 @@ +//! Content-Aware Fill (Window ▸ Content-Aware Fill, Layer ▸ New ▸ Content-Aware Fill Layer): +//! fill the transparent area of a layer (what its masks cut away) across the work area or the +//! whole layer, render the result as a PNG sequence and add it as a "Fill" layer above the +//! source, like After Effects. The algorithms are `effectcraft_raster::inpaint` (Object: +//! flow-guided propagation + PatchMatch; Surface: propagation + one synthesised fill carried by +//! the flow; Edge Blend: membrane fill). It runs as a background job (Window ▸ Progress). + +use std::sync::Arc; + +use effectcraft_color::Label; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource, build}; +use effectcraft_raster::Image; +use effectcraft_raster::inpaint::{FillInput, FillMethod, FillOpts, dilate, fill_sequence}; +use effectcraft_render::{EvalCtx, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::scene_detect::LayerFrames; +use super::{CommandSpec, always, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::jobs::Apply; +use crate::{EngineError, Result, Session, cmd}; + +/// Content-Aware Fill panel settings (serde, in [`crate::EditorState`]). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct FillSettings { + /// `object`, `surface` or `edgeBlend`. + pub method: String, + /// `workArea` or `entire` (the layer's duration). + pub range: String, + /// Grow the fill area by this many pixels. + pub alpha_expansion: f64, + /// `none`, `subtle`, `moderate`, `strong`. + pub lighting_correction: String, + /// Reference frame: a layer of the comp and the comp time of the frame to use. + pub reference_layer: Option, + pub reference_time: Option, +} + +impl Default for FillSettings { + fn default() -> Self { + FillSettings { + method: "object".into(), + range: "workArea".into(), + alpha_expansion: 0.0, + lighting_correction: "none".into(), + reference_layer: None, + reference_time: None, + } + } +} + +fn lighting(s: &str) -> Option { + Some(match s { + "none" | "off" => 0.0, + "subtle" => 0.35, + "moderate" => 0.65, + "strong" => 1.0, + _ => return None, + }) +} + +fn set(s: &mut Session, p: &Value) -> Result { + let c = "contentFill.set"; + let mut st = s.state.content_fill.clone(); + if let Some(m) = str_p(p, "method") { + st.method = FillMethod::from_name(m).ok_or_else(|| bad(c, "method: object|surface|edgeBlend"))?.name().into(); + } + if let Some(r) = str_p(p, "range") { + st.range = match r { + "workArea" | "work area" => "workArea".into(), + "entire" | "entireDuration" | "layer" => "entire".into(), + _ => return Err(bad(c, "range: workArea|entire")), + }; + } + if let Some(a) = f_p(p, "alphaExpansion") { + st.alpha_expansion = a.clamp(0.0, 100.0); + } + if let Some(l) = str_p(p, "lightingCorrection") { + lighting(l).ok_or_else(|| bad(c, "lightingCorrection: none|subtle|moderate|strong"))?; + st.lighting_correction = l.into(); + } + match p.get("referenceLayer") { + Some(Value::Null) => st.reference_layer = None, + Some(v) => { + let comp = s.active_comp().ok_or(EngineError::NoComp)?; + st.reference_layer = Some(super::resolve_layer(comp, v).ok_or_else(|| bad(c, format!("no layer {v}")))?); + st.reference_time = Some(f_p(p, "referenceTime").unwrap_or(s.time().seconds())); + } + None => { + if let Some(t) = f_p(p, "referenceTime") { + st.reference_time = Some(t); + } + } + } + s.state.content_fill = st.clone(); + Ok(serde_json::to_value(st).unwrap_or_default()) +} + +/// A layer's masked source at comp time `t` on a `w × h` canvas in layer pixels (straight colour +/// with alpha 1 where kept), and the hole (`true` where the alpha is under one half). +fn masked_frame(r: &Renderer, ctx: &EvalCtx, layer: &effectcraft_project::Layer, w: u32, h: u32) -> (Image, Vec) { + let mut img = Image::new(w, h); + let mut hole = vec![true; (w * h) as usize]; + if let Some(buf) = r.layer_input(ctx, layer, 0) { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for y in 0..h { + for x in 0..w { + let (bx, by) = ((x as f64 + 0.5) * sc - buf.offset[0], (y as f64 + 0.5) * sc - buf.offset[1]); + let p = if (sc - 1.0).abs() < 1e-9 { buf.img.get(bx.floor() as i64, by.floor() as i64) } else { buf.img.sample_bilinear(bx, by) }; + let i = (y * w + x) as usize; + if p[3] >= 0.5 { + img.data[i] = [p[0] / p[3], p[1] / p[3], p[2] / p[3], 1.0]; + hole[i] = false; + } + } + } + } + (img, hole) +} + +/// Encode a straight-alpha frame (alpha 1) as 8-bit PNG. +fn png(img: &Image) -> std::result::Result, String> { + let rgba: Vec = img.data.iter().flat_map(|p| [p[0], p[1], p[2], p[3]].map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8)).collect(); + super::comp_more::encode_png(&rgba, img.width, img.height) +} + +fn default_dir(s: &Session, name: &str) -> String { + let base = s + .path + .as_deref() + .and_then(|p| std::path::Path::new(p).parent().map(|d| d.join("Fill"))) + .unwrap_or_else(|| std::env::temp_dir().join("effectcraft-fill")); + let safe: String = name.chars().map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' }).collect(); + let mut k = 1; + loop { + let d = base.join(format!("{safe}_Fill_{k}")); + if !d.exists() { + return d.to_string_lossy().to_string(); + } + k += 1; + } +} + +fn generate(s: &mut Session, p: &Value) -> Result { + let c = "contentFill.generate"; + let (cid, lid) = layer_p(s, p, c)?; + // Parameters override the panel settings. + let mut st = s.state.content_fill.clone(); + { + let saved = s.state.content_fill.clone(); + let mut q = p.clone(); + if let Some(o) = q.as_object_mut() { + o.retain(|k, _| ["method", "range", "alphaExpansion", "lightingCorrection", "referenceLayer", "referenceTime"].contains(&k.as_str())); + } + set(s, &q)?; + std::mem::swap(&mut st, &mut s.state.content_fill); + s.state.content_fill = saved; + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + if !layer.source.is_av() || layer.source.item().is_none() { + return Err(bad(c, "Content-Aware Fill needs a footage, solid or precomp layer")); + } + let (w, h) = effectcraft_render::source_size(&s.project, &layer); + if w == 0 || h == 0 { + return Err(bad(c, "the layer has no pixels")); + } + let fr = comp.frame_rate; + let (a, b) = + if st.range == "entire" { (layer.in_point, layer.out_point) } else { (comp.work_area.0.max(layer.in_point), comp.work_area.1.min(layer.out_point)) }; + let (a, b) = (a.max(Tick::ZERO), b.min(comp.duration)); + if b <= a { + return Err(bad(c, "the range doesn't overlap the layer")); + } + let (fa, fb) = (fr.frame_at(fr.snap_nearest(a)), fr.frame_at(b - Tick(1))); + let times: Vec = (fa..=fb).map(|f| fr.tick_of(f)).collect(); + let opts = FillOpts { + method: FillMethod::from_name(&st.method).unwrap_or_default(), + lighting: lighting(&st.lighting_correction).unwrap_or(0.0), + ..Default::default() + }; + let expand = st.alpha_expansion.round().max(0.0) as usize; + let reference = st.reference_layer.filter(|r| comp.layer(*r).is_some()).map(|r| (r, Tick::from_seconds_f64(st.reference_time.unwrap_or(0.0)))); + let dir = str_p(p, "outputDir").map(str::to_string).unwrap_or_else(|| default_dir(s, &layer.name)); + let lf = LayerFrames::new(s, cid, lid); + let services = s.services.clone(); + let importer = s.importer.clone(); + let wait = b_p(p, "wait").unwrap_or(false); + let name = layer.name.clone(); + s.spawn_task("contentFill", format!("Content-Aware Fill: {name}"), wait, move |ctl| { + ctl.message("Reading frames"); + let project = lf.project.clone(); + let comp = project.comp(cid).ok_or("composition missing")?; + let layer = comp.layer(lid).ok_or("layer missing")?; + let mut r = Renderer::new(&project, lf.footage.as_ref(), RenderOpts::default()); + r.expr = lf.expr.as_deref(); + r.cache = Some(&lf.cache); + let ctx0 = EvalCtx { expr: lf.expr.as_deref(), ..EvalCtx::new(&project, cid, comp, Tick::ZERO) }; + let n = times.len() as u64; + let mut frames = Vec::with_capacity(times.len()); + let mut holes = Vec::with_capacity(times.len()); + for (k, t) in times.iter().enumerate() { + let (img, hole) = masked_frame(&r, &ctx0.at(*t), layer, w, h); + holes.push(dilate(&hole, w as usize, h as usize, expand)); + frames.push(img); + if !ctl.progress(k as u64 + 1, n * 3) { + return Err(crate::render_queue::CANCELLED.into()); + } + } + if !holes.iter().any(|h| h.iter().any(|b| *b)) { + return Err("the layer has no transparent area to fill: draw a mask (Subtract) around what to remove".into()); + } + let mut references = vec![]; + if let Some((rl, rt)) = reference + && let Some(l) = comp.layer(rl) + { + let (img, hole) = masked_frame(&r, &ctx0.at(rt), l, w, h); + if hole.iter().all(|b| !*b) { + let k = times.iter().enumerate().min_by_key(|(_, t)| (t.0 - rt.0).abs()).map(|(k, _)| k).unwrap_or(0); + references.push((k, img)); + } + } + ctl.message("Filling"); + let input = FillInput { frames, holes, references }; + let out = fill_sequence(&input, &opts, &mut |d, t| ctl.progress(n + (d as u64 * n / t.max(1) as u64), n * 3)).ok_or(crate::render_queue::CANCELLED)?; + ctl.message("Writing the fill layer"); + std::fs::create_dir_all(&dir).map_err(|e| format!("{dir}: {e}"))?; + let mut paths = vec![]; + for (k, img) in out.iter().enumerate() { + let path = std::path::Path::new(&dir).join(format!("fill_{k:05}.png")).to_string_lossy().to_string(); + services.write_file(&path, &png(img)?).map_err(|e| format!("{path}: {e}"))?; + paths.push(path); + if !ctl.progress(2 * n + k as u64 + 1, n * 3) { + return Err(crate::render_queue::CANCELLED.into()); + } + } + let start = times[0]; + let count = times.len(); + let apply: Apply = Box::new(move |s: &mut Session| add_fill_layer(s, cid, lid, &name, paths, (w, h), start, count, importer)); + Ok(apply) + }) +} + +/// Import the rendered sequence and put it in a layer above the source. +#[allow(clippy::too_many_arguments)] +fn add_fill_layer( + s: &mut Session, + cid: ItemId, + lid: LayerId, + name: &str, + paths: Vec, + size: (u32, u32), + start: Tick, + count: usize, + importer: Option>, +) -> Result { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or_else(|| bad("contentFill.generate", "the source layer is gone"))?.clone(); + let fr = comp.frame_rate; + let dur = fr.tick_of(count as i64); + let mut footage = importer.and_then(|i| i.probe(&paths[0]).ok()).filter(|f| f.has_video).unwrap_or_else(|| Footage { + path: paths[0].clone(), + kind: FootageKind::Sequence, + width: size.0, + height: size.1, + has_video: true, + codec: "PNG".into(), + ..Default::default() + }); + footage.kind = FootageKind::Sequence; + footage.sequence = paths.clone(); + footage.frame_rate = fr; + footage.duration = dur; + let (lname, item, layer) = s.edit("Content-Aware Fill", None, |proj, st| { + let folder = proj.folder_named("Fill").unwrap_or_else(|| proj.add_item("Fill", Label::Yellow, None, ItemKind::Folder)); + let n = proj.items.values().filter(|i| i.parent == Some(folder)).count() + 1; + let lname = format!("Fill {n} [{name}]"); + let iid = proj.add_item(&lname, Label::Lavender, Some(folder), ItemKind::Footage(footage)); + let mut l = build::layer(proj, &comp, &lname, LayerSource::Footage { item: iid }, size, Some(dur)); + l.start_time = start; + l.in_point = start; + l.out_point = (start + dur).min(comp.duration); + l.parent = src.parent; + l.switches.three_d = src.switches.three_d; + if let (Some(dst), Some(t)) = (l.props.sub_mut("transform"), src.transform()) { + let mut tr = t.clone(); + tr.reassign_uids(&mut proj.next_id); + *dst = tr; + } + let id = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = cm.layers.iter().position(|x| x.id == lid).unwrap_or(0); + cm.layers.insert(at, l); + st.selected_layers = vec![id]; + Ok((lname, iid, id)) + })?; + Ok(json!({"layer": layer.0, "item": item.0, "name": lname, "frames": count, "files": paths})) +} + +fn open_panel(s: &mut Session, p: &Value) -> Result { + // Parameters given: generate straight away (agents); else show the panel. + if p.as_object().is_some_and(|o| !o.is_empty()) { + return generate(s, p); + } + s.events.push(crate::Event::Frontend { command: "window.panel".into(), params: json!({"panel": "contentAwareFill"}) }); + Ok(json!({"frontend": "window.panel"})) +} + +pub fn specs() -> Vec { + const P: &str = "{layer?, method?: object|surface|edgeBlend, range?: workArea|entire, alphaExpansion? (px), lightingCorrection?: none|subtle|moderate|strong, referenceLayer?, referenceTime? (s), outputDir?, wait?: bool}"; + vec![ + cmd!( + "contentFill.set", + "Content-Aware Fill Settings", + [], + None, + "{method?: object|surface|edgeBlend, range?: workArea|entire, alphaExpansion? (px), lightingCorrection?: none|subtle|moderate|strong, referenceLayer?: layer|null, referenceTime? (s)}", + always, + set + ), + CommandSpec { + id: "contentFill.generate", + label: "Generate Fill Layer", + menu: &[], + shortcut: None, + params: P, + enabled: has_layers, + run: generate, + journal: true, + }, + CommandSpec { + id: "layer.newContentAwareFill", + label: "Content-Aware Fill Layer...", + menu: &["Layer", "New"], + shortcut: None, + params: P, + enabled: always, + run: open_panel, + journal: true, + }, + ] +} diff --git a/crates/engine/src/commands/footage_panel.rs b/crates/engine/src/commands/footage_panel.rs new file mode 100644 index 00000000..4d11a35a --- /dev/null +++ b/crates/engine/src/commands/footage_panel.rs @@ -0,0 +1,224 @@ +//! The Footage panel (double-click footage in the Project panel): plays a footage item on its own +//! time ruler, marks In and Out points in source time, and edits the marked range into the active +//! composition with **Overlay Edit** (a new layer at the current time, on top) or **Ripple Insert +//! Edit** (the same, after pushing later layers — and the later part of layers crossing the +//! current time — back by the clip's duration). + +use effectcraft_project::{ItemId, ItemKind, LayerSource, build}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, comp_id, f_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// What the Footage panel shows (source time). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FootageView { + pub item: ItemId, + pub time: Tick, + #[serde(default)] + pub in_point: Option, + #[serde(default)] + pub out_point: Option, +} + +fn footage_item(s: &Session, p: &Value, cmd: &str) -> Result { + let id = match p.get("item") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id), + _ => s.state.footage_panel.as_ref().map(|f| f.item).or_else(|| s.state.project_selection.first().copied()), + } + .ok_or_else(|| bad(cmd, "no footage: pass `item` or open one in the Footage panel"))?; + match s.project.item(id).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != effectcraft_project::FootageKind::Data => Ok(id), + Some(ItemKind::Solid(_)) => Ok(id), + _ => Err(bad(cmd, format!("item {} is not footage", id.0))), + } +} + +/// Source duration and frame rate of a footage / solid item (stills: `None`). +fn source_timing(s: &Session, item: ItemId) -> (Option, effectcraft_time::FrameRate) { + match s.project.item(item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.kind != effectcraft_project::FootageKind::Still && f.duration > Tick::ZERO => (Some(f.duration), f.frame_rate), + Some(ItemKind::Footage(f)) => (None, f.frame_rate), + _ => (None, s.active_comp().map(|c| c.frame_rate).unwrap_or(effectcraft_time::FrameRate::FPS_30)), + } +} + +fn source_time(s: &Session, p: &Value, item: ItemId) -> Option { + let (_, fr) = source_timing(s, item); + if let Some(t) = f_p(p, "time") { + return Some(fr.snap_nearest(Tick::from_seconds_f64(t.max(0.0)))); + } + p.get("frame").and_then(Value::as_i64).map(|f| fr.tick_of(f.max(0))) +} + +fn open(s: &mut Session, p: &Value) -> Result { + let item = footage_item(s, p, "footage.open")?; + if s.state.footage_panel.as_ref().is_none_or(|f| f.item != item) { + s.state.footage_panel = Some(FootageView { item, time: Tick::ZERO, in_point: None, out_point: None }); + } + s.events.push(crate::Event::Frontend { command: "window.panel".into(), params: json!({"panel": "footage"}) }); + Ok(json!(s.state.footage_panel)) +} + +fn view_mut<'a>(s: &'a mut Session, cmd: &str) -> Result<&'a mut FootageView> { + s.state.footage_panel.as_mut().ok_or_else(|| bad(cmd, "the Footage panel shows no footage (footage.open)")) +} + +fn clamp_time(s: &Session, item: ItemId, t: Tick) -> Tick { + let (dur, fr) = source_timing(s, item); + match dur { + Some(d) => t.clamp(Tick::ZERO, (d - fr.frame_duration()).max(Tick::ZERO)), + None => t.max(Tick::ZERO), + } +} + +fn set_time(s: &mut Session, p: &Value) -> Result { + let item = view_mut(s, "footage.setTime")?.item; + let t = source_time(s, p, item).ok_or_else(|| bad("footage.setTime", "missing `time` or `frame`"))?; + let t = clamp_time(s, item, t); + view_mut(s, "footage.setTime")?.time = t; + Ok(json!(t.seconds())) +} + +fn set_mark(s: &mut Session, p: &Value, out: bool) -> Result { + let c = if out { "footage.setOut" } else { "footage.setIn" }; + let v = view_mut(s, c)?.clone(); + let t = source_time(s, p, v.item).unwrap_or(v.time); + let t = clamp_time(s, v.item, t); + let (_, fr) = source_timing(s, v.item); + let view = view_mut(s, c)?; + if out { + // The Out point marks the last frame shown: the range ends after it. + let end = t + fr.frame_duration(); + view.out_point = Some(end); + if view.in_point.is_some_and(|i| i >= end) { + view.in_point = None; + } + } else { + view.in_point = Some(t); + if view.out_point.is_some_and(|o| o <= t) { + view.out_point = None; + } + } + Ok(json!(s.state.footage_panel)) +} + +fn clear_marks(s: &mut Session, _: &Value) -> Result { + let v = view_mut(s, "footage.clearInOut")?; + v.in_point = None; + v.out_point = None; + Ok(Value::Null) +} + +/// Overlay / Ripple Insert Edit of the marked range at the comp's current time. +fn edit_into_comp(s: &mut Session, p: &Value, ripple: bool) -> Result { + let c = if ripple { "footage.rippleInsertEdit" } else { "footage.overlayEdit" }; + let item = footage_item(s, p, c)?; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let view = s.state.footage_panel.clone().filter(|v| v.item == item); + let (dur, _) = source_timing(s, item); + let src_in = f_p(p, "in").map(Tick::from_seconds_f64).or(view.as_ref().and_then(|v| v.in_point)).unwrap_or(Tick::ZERO); + let fr = comp.frame_rate; + let at = fr.snap_nearest(f_p(p, "at").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid))).max(Tick::ZERO); + let src_out = f_p(p, "out").map(Tick::from_seconds_f64).or(view.as_ref().and_then(|v| v.out_point)).or(dur).unwrap_or(src_in + (comp.duration - at)); + if src_out <= src_in { + return Err(bad(c, "the Out point must be after the In point")); + } + let len = fr.snap_nearest(src_out - src_in).max(fr.frame_duration()); + if at >= comp.duration { + return Err(bad(c, "the current time is past the end of the composition")); + } + let it = s.project.item(item).cloned().ok_or_else(|| bad(c, "no such item"))?; + let (src, size) = match &it.kind { + ItemKind::Footage(f) => (LayerSource::Footage { item }, (f.width, f.height)), + ItemKind::Solid(so) => (LayerSource::Solid { item }, (so.width, so.height)), + _ => return Err(bad(c, "not footage")), + }; + let label = if ripple { "Ripple Insert Edit" } else { "Overlay Edit" }; + let id = s.edit(label, None, |proj, st| { + let mut next = proj.next_id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + if ripple { + let mut added = vec![]; + for i in 0..cm.layers.len() { + let l = &mut cm.layers[i]; + if l.in_point >= at { + l.start_time += len; + l.in_point += len; + l.out_point += len; + } else if l.out_point > at { + // Split: the part after the current time moves back. + let mut b = l.clone(); + super::edit::reid(&mut b, &mut next); + l.out_point = at; + b.in_point = at + len; + b.out_point += len; + b.start_time += len; + added.push((i, b)); + } + } + for (i, b) in added.into_iter().rev() { + cm.layers.insert(i, b); + } + } + proj.next_id = next; + let mut l = build::layer(proj, &comp, &it.name, src, size, Some(len)); + l.start_time = at - src_in; + l.in_point = at; + l.out_point = (at + len).min(comp.duration); + let lid = l.id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + cm.layers.insert(0, l); + st.selected_layers = vec![lid]; + st.selected_props.clear(); + st.selected_keys.clear(); + Ok(lid) + })?; + Ok(json!({"layer": id.0, "in": at.seconds(), "duration": len.seconds()})) +} + +fn has_view_or_footage(s: &Session) -> std::result::Result<(), String> { + if s.state.footage_panel.is_some() { Ok(()) } else { Err("open footage in the Footage panel first".into()) } +} + +fn can_edit(s: &Session) -> std::result::Result<(), String> { + has_view_or_footage(s)?; + super::has_comp(s) +} + +fn info(s: &mut Session, _: &Value) -> Result { + let Some(v) = &s.state.footage_panel else { return Ok(Value::Null) }; + let (dur, fr) = source_timing(s, v.item); + let name = s.project.item(v.item).map(|i| i.name.clone()).unwrap_or_default(); + Ok( + json!({"item": v.item.0, "name": name, "time": v.time.seconds(), "frame": fr.frame_at(v.time), "in": v.in_point.map(|t| t.seconds()), "out": v.out_point.map(|t| t.seconds()), "duration": dur.map(|d| d.seconds()), "frameRate": fr.as_f64()}), + ) +} + +pub fn specs() -> Vec { + vec![ + cmd!("footage.open", "Open in Footage Panel", [], None, "{item: id|name}", always, open), + crate::query!("footage.info", "Footage Panel State", "{}", info), + cmd!("footage.setTime", "Footage Panel Time", [], None, "{time? (s, source) | frame?}", has_view_or_footage, set_time), + cmd!("footage.setIn", "Set In Point", [], None, "{time? | frame? (default: the panel's time)}", has_view_or_footage, |s, p| set_mark(s, p, false)), + cmd!("footage.setOut", "Set Out Point", [], None, "{time? | frame? (default: the panel's time)}", has_view_or_footage, |s, p| set_mark(s, p, true)), + cmd!("footage.clearInOut", "Clear In and Out", [], None, "{}", has_view_or_footage, clear_marks), + cmd!("footage.overlayEdit", "Overlay Edit", [], None, "{item?, comp?, in? (s), out? (s), at? (comp s; default the current time)}", can_edit, |s, p| { + edit_into_comp(s, p, false) + }), + cmd!( + "footage.rippleInsertEdit", + "Ripple Insert Edit", + [], + None, + "{item?, comp?, in? (s), out? (s), at? (comp s; default the current time)}", + can_edit, + |s, p| edit_into_comp(s, p, true) + ), + ] +} diff --git a/crates/engine/src/commands/frontend.rs b/crates/engine/src/commands/frontend.rs index 168e77a5..7abd2923 100644 --- a/crates/engine/src/commands/frontend.rs +++ b/crates/engine/src/commands/frontend.rs @@ -63,7 +63,7 @@ pub fn specs() -> Vec { "Show Panel", [], None, - "{panel: project|effectControls|composition|layer|timeline|info|audio|preview|effectsPresets|properties|character|paragraph|align|tracker|wiggler|smoother|motionSketch|paint|brushes|renderQueue|flowchart|history|markers|tools}", + "{panel: project|effectControls|composition|layer|timeline|info|audio|preview|effectsPresets|properties|character|paragraph|align|tracker|wiggler|smoother|motionSketch|paint|brushes|renderQueue|flowchart|history|markers|tools|lumetriScopes|footage|mediaBrowser|metadata|progress|contentAwareFill}", always ), fe!( diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 39fa4234..12324cf4 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -1,12 +1,15 @@ //! The command registry. Ids follow After Effects' menu structure; every menu item, panel button, //! shortcut and viewer/timeline gesture maps to one of these. +mod align_data; mod anim; pub mod anim_tools; mod animation; +mod autotrace; mod camera_cmds; mod comp; -mod comp_more; +pub(crate) mod comp_more; +pub mod content_fill; mod create; mod edit; mod effect; @@ -15,6 +18,7 @@ pub mod expr_tools; mod file; pub(crate) mod file_more; mod focus; +pub mod footage_panel; mod frontend; mod help; mod key_labels; @@ -30,6 +34,7 @@ mod mask; pub mod mask_interp; pub(crate) mod model3d; pub mod paint; +pub mod panels_cmds; mod paths; mod project_items; mod prop; @@ -39,6 +44,7 @@ pub mod puppet; mod query; mod render_queue; pub mod roto_cmds; +mod scene_detect; mod settings; mod shape_stroke; mod stubs; @@ -156,6 +162,12 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(essential::specs()); v.extend(expr_tools::specs()); v.extend(proxy::specs()); + v.extend(panels_cmds::specs()); + v.extend(footage_panel::specs()); + v.extend(autotrace::specs()); + v.extend(scene_detect::specs()); + v.extend(align_data::specs()); + v.extend(content_fill::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/panels_cmds.rs b/crates/engine/src/commands/panels_cmds.rs new file mode 100644 index 00000000..eb056878 --- /dev/null +++ b/crates/engine/src/commands/panels_cmds.rs @@ -0,0 +1,214 @@ +//! Commands behind the Progress, Media Browser, Metadata and Lumetri Scopes panels. + +use effectcraft_project::ItemId; +use effectcraft_raster::Image; +use effectcraft_raster::scopes::{self, ColorStandard, ScopeKind, ScopeOpts}; +use effectcraft_render::RenderOpts; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, f_p, str_p}; +use crate::{EngineError, Result, Session, cmd, media_browser as mb, query}; + +// ---------------------------------------------------------------- Progress + +fn jobs_list(s: &mut Session, _: &Value) -> Result { + s.poll_jobs(); + Ok(json!({"running": s.jobs(), "finished": s.job_log})) +} + +fn jobs_cancel(s: &mut Session, p: &Value) -> Result { + let id = str_p(p, "job").unwrap_or("all"); + Ok(json!({"cancelled": s.cancel_job(id)})) +} + +fn jobs_wait(s: &mut Session, _: &Value) -> Result { + s.wait_jobs(); + Ok(json!({"finished": s.job_log})) +} + +// ---------------------------------------------------------------- Media Browser + +fn browser_dir(s: &Session, p: &Value) -> String { + str_p(p, "path").map(str::to_string).or_else(|| s.state.media_browser.folder.clone()).unwrap_or_else(mb::home) +} + +fn browser_list(s: &mut Session, p: &Value) -> Result { + let dir = browser_dir(s, p); + let only = b_p(p, "importableOnly").unwrap_or(s.state.media_browser.importable_only); + let entries = mb::list(&dir, only).map_err(EngineError::Other)?; + Ok(json!({"path": dir, "parent": mb::parent(&dir), "entries": entries, "favorites": s.state.media_browser.favorites})) +} + +fn browser_enabled(_: &Session) -> std::result::Result<(), String> { + if mb::available() { Ok(()) } else { Err("the Media Browser needs the desktop app".into()) } +} + +fn browser_go(s: &mut Session, p: &Value) -> Result { + let dir = match str_p(p, "path") { + Some("..") => s.state.media_browser.folder.clone().and_then(|d| mb::parent(&d)).unwrap_or_else(mb::home), + Some(d) => d.to_string(), + None => mb::home(), + }; + if let Some(b) = b_p(p, "importableOnly") { + s.state.media_browser.importable_only = b; + } + // Validate before switching. + mb::list(&dir, false).map_err(EngineError::Other)?; + s.state.media_browser.folder = Some(dir.clone()); + browser_list(s, &json!({"path": dir})) +} + +fn browser_favorite(s: &mut Session, p: &Value, add: bool) -> Result { + let dir = browser_dir(s, p); + let favs = &mut s.state.media_browser.favorites; + favs.retain(|f| f != &dir); + if add { + favs.push(dir); + } + Ok(json!({"favorites": favs})) +} + +fn browser_import(s: &mut Session, p: &Value) -> Result { + let paths = p.get("paths").or(p.get("path")).cloned().ok_or_else(|| bad("mediaBrowser.import", "missing `paths`"))?; + let r = s.execute("file.import", json!({"paths": paths}))?; + // Add to the active comp (drag into the timeline). + if b_p(p, "addToComp").unwrap_or(false) + && s.active_comp_id().is_some() + && let Some(items) = r.get("items").and_then(Value::as_array) + { + for it in items { + s.execute("layer.addItem", json!({"item": it}))?; + } + } + Ok(r) +} + +fn file_info(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("mediaBrowser.fileInfo", "missing `path`"))?; + let (created, modified, size) = mb::file_dates(path); + let mut v = json!({"path": path, "kind": mb::kind_of(path), "size": size, "created": created.map(mb::iso_date), "modified": modified.map(mb::iso_date)}); + if let Some(imp) = s.importer.clone() + && matches!(mb::kind_of(path), Some("video" | "image" | "audio" | "model")) + && let Ok(f) = imp.probe(path) + { + v["footage"] = mb::footage_metadata(&f); + } + Ok(v) +} + +// ---------------------------------------------------------------- Metadata + +fn item_ref(s: &Session, p: &Value) -> Option { + match p.get("item") { + Some(Value::Number(n)) => n.as_u64().map(ItemId), + Some(Value::String(name)) => s.project.items.values().find(|i| &i.name == name).map(|i| i.id), + _ => s.state.project_selection.first().copied(), + } +} + +fn metadata(s: &mut Session, p: &Value) -> Result { + let project = mb::project_metadata(&s.project, s.path.as_deref()); + let item = match item_ref(s, p) { + Some(id) => Some(mb::item_metadata(&s.project, s.project.item(id).ok_or_else(|| bad("item.metadata", format!("no item {}", id.0)))?)), + None => None, + }; + Ok(json!({"project": project, "item": item})) +} + +fn project_comment(s: &mut Session, p: &Value) -> Result { + let text = str_p(p, "comment").ok_or_else(|| bad("project.setProjectComment", "missing `comment`"))?.to_string(); + s.edit("Project Comment", super::merge_p(p), |proj, _| { + proj.settings.comment = text; + Ok(()) + })?; + Ok(Value::Null) +} + +// ---------------------------------------------------------------- Lumetri Scopes + +/// The comp frame the scopes read (the viewer's image: over the comp background), at most +/// `max_side` pixels on the long side. +pub fn scope_frame(s: &Session, comp: ItemId, t: Tick, max_side: u32) -> Option { + let c = s.project.comp(comp)?; + let long = c.width.max(c.height).max(1) as f64; + let scale = (max_side as f64 / long).min(1.0); + let mut img = s.render(comp, t, RenderOpts { scale, ..Default::default() }); + let bg = c.background; + for p in &mut img.data { + let k = 1.0 - p[3].clamp(0.0, 1.0); + *p = [p[0] + bg[0] * k, p[1] + bg[1] * k, p[2] + bg[2] * k, 1.0]; + } + Some(img) +} + +/// Scope options from parameters. +pub fn scope_opts(p: &Value) -> std::result::Result<(ScopeKind, ScopeOpts), String> { + let kind = match str_p(p, "scope") { + Some(k) => ScopeKind::from_name(k).ok_or_else(|| format!("unknown scope `{k}`; one of {}", ScopeKind::ALL.map(|k| k.name()).join(", ")))?, + None => ScopeKind::WaveformRgb, + }; + let standard = match str_p(p, "standard") { + Some(x) => ColorStandard::from_name(x).ok_or_else(|| format!("standard: rec601|rec709|rec2020, not `{x}`"))?, + None => ColorStandard::Rec709, + }; + let size = f_p(p, "size").unwrap_or(256.0).clamp(16.0, 1024.0) as u32; + Ok((kind, ScopeOpts { standard, float: b_p(p, "float").unwrap_or(false), clamp: b_p(p, "clamp").unwrap_or(true), width: size, height: size })) +} + +fn scopes_analyze(s: &mut Session, p: &Value) -> Result { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = super::time_p(s, p, Some(comp)); + let (kind, o) = scope_opts(p).map_err(|e| bad("scopes.analyze", e))?; + let img = scope_frame(s, cid, t, 512).ok_or(EngineError::NoComp)?; + let sc = scopes::compute(&img, kind, &o); + // Signal statistics of the frame. + let n = img.data.len().max(1) as f64; + let mut mean = [0.0f64; 3]; + let (mut ymin, mut ymax, mut ysum) = (f64::MAX, f64::MIN, 0.0); + for px in &img.data { + for c in 0..3 { + mean[c] += px[c] as f64 / n; + } + let y = o.standard.ycbcr([px[0], px[1], px[2]])[0] as f64; + ymin = ymin.min(y); + ymax = ymax.max(y); + ysum += y; + } + let mut v = json!({ + "scope": kind.name(), + "width": sc.width, + "height": sc.height, + "peaks": [sc.peak(0), sc.peak(1), sc.peak(2)], + "totals": [sc.total(0), sc.total(1), sc.total(2)], + "mean": mean, + "luma": {"min": ymin, "max": ymax, "mean": ysum / n}, + }); + if kind == ScopeKind::Histogram { + v["histogram"] = json!(scopes::histogram_bars(&sc)); + } + Ok(v) +} + +pub fn specs() -> Vec { + vec![ + query!("jobs.list", "Background Jobs", "{}", jobs_list), + cmd!("jobs.cancel", "Cancel Job", [], None, "{job?: render|track|maskTrack|warp|camera|roto|task:|all}", always, jobs_cancel), + query!("jobs.wait", "Wait for Background Jobs", "{}", jobs_wait), + query!("mediaBrowser.list", "List Folder", "{path?, importableOnly?}", browser_list), + cmd!("mediaBrowser.go", "Go to Folder", [], None, "{path?: folder | \"..\", importableOnly?}", browser_enabled, browser_go), + cmd!("mediaBrowser.addFavorite", "Add to Favorites", [], None, "{path?}", browser_enabled, |s, p| browser_favorite(s, p, true)), + cmd!("mediaBrowser.removeFavorite", "Remove from Favorites", [], None, "{path?}", browser_enabled, |s, p| browser_favorite(s, p, false)), + cmd!("mediaBrowser.import", "Import", [], None, "{paths, addToComp?}", browser_enabled, browser_import), + query!("mediaBrowser.fileInfo", "File Info", "{path}", file_info), + query!("item.metadata", "Metadata", "{item?: id|name}", metadata), + cmd!("project.setProjectComment", "Project Comment", [], None, "{comment}", always, project_comment), + query!( + "scopes.analyze", + "Lumetri Scopes", + "{comp?, time?|frame?, scope?: waveformRgb|waveformLuma|waveformYc|vectorscopeYuv|vectorscopeHls|histogram|paradeRgb|paradeYuv, standard?: rec601|rec709|rec2020, float?, clamp?, size?}", + scopes_analyze + ), + ] +} diff --git a/crates/engine/src/commands/scene_detect.rs b/crates/engine/src/commands/scene_detect.rs new file mode 100644 index 00000000..8d523e21 --- /dev/null +++ b/crates/engine/src/commands/scene_detect.rs @@ -0,0 +1,182 @@ +//! Layer ▸ Scene Edit Detection…: find the cuts in a footage (or precomp) layer and Create +//! Markers, Split Layers, or Split and Precompose at them. The analysis +//! (`effectcraft_raster::cuts`: colour histograms + motion-compensated difference, adaptive peak +//! threshold) runs as a background job (Window ▸ Progress). + +use std::sync::Arc; + +use effectcraft_project::{ItemId, LayerId, Marker, Project}; +use effectcraft_raster::cuts::{self, CutOpts}; +use effectcraft_render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_layers, layer_p, str_p}; +use crate::jobs::Apply; +use crate::{EngineError, Result, Session, cmd}; + +/// What a background job needs to render a layer's frames. +#[derive(Clone)] +pub(crate) struct LayerFrames { + pub project: Arc, + pub footage: Arc, + pub expr: Option>, + pub cache: Arc, + pub comp: ItemId, + pub layer: LayerId, +} + +impl LayerFrames { + pub fn new(s: &Session, comp: ItemId, layer: LayerId) -> LayerFrames { + LayerFrames { project: s.project.clone(), footage: s.footage.clone(), expr: s.expr.clone(), cache: s.layer_cache.clone(), comp, layer } + } + /// Run `f` with a frame getter: comp time → (layer source pixels, offset in layer space). + pub fn with(&self, f: impl FnOnce(&(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync)) -> R) -> Option { + let comp = self.project.comp(self.comp)?; + let layer = comp.layer(self.layer)?; + let mut r = Renderer::new(&self.project, self.footage.as_ref(), RenderOpts::default()); + r.expr = self.expr.as_deref(); + r.cache = Some(&self.cache); + let r = &r; + let get = |t: Tick| crate::tracking::source_frame(r, &self.project, self.comp, comp, layer, t, self.expr.as_deref()); + Some(f(&get)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + Markers, + Split, + SplitPrecompose, +} + +/// Cut times (comp time) of the layer's frames `times` (first frame never a cut). +pub(crate) fn analyse( + lf: &LayerFrames, + times: &[Tick], + o: &CutOpts, + progress: &dyn Fn(u64, u64) -> bool, +) -> std::result::Result<(Vec, Vec), String> { + lf.with(|get| { + let mut sigs = Vec::with_capacity(times.len()); + // Signatures in batches (parallel), with progress and cancel between batches. + for (k, chunk) in times.chunks(8).enumerate() { + use rayon::prelude::*; + let batch: Vec = chunk.par_iter().map(|t| get(*t).map(|(img, _)| cuts::signature(&img)).unwrap_or_default()).collect(); + sigs.extend(batch); + if !progress(((k + 1) * 8).min(times.len()) as u64, times.len() as u64) { + return Err(crate::render_queue::CANCELLED.to_string()); + } + } + let mut scores = vec![0.0f32]; + for i in 1..sigs.len() { + scores.push(cuts::dissimilarity(&sigs[i - 1], &sigs[i])); + } + let cuts = cuts::detect(&scores, o); + Ok((cuts.into_iter().map(|i| times[i]).collect(), scores)) + }) + .ok_or_else(|| "layer missing".to_string())? +} + +fn run(s: &mut Session, p: &Value) -> Result { + let c = "layer.sceneEditDetection"; + let (cid, lid) = layer_p(s, p, c)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + if layer.source.item().is_none() { + return Err(bad(c, "scene edit detection needs a footage or precomp layer")); + } + let mode = match str_p(p, "mode").unwrap_or("markers") { + "markers" | "createMarkers" => Mode::Markers, + "split" | "splitLayers" => Mode::Split, + "splitPrecompose" | "splitAndPrecompose" => Mode::SplitPrecompose, + x => return Err(bad(c, format!("mode: markers|split|splitPrecompose, not `{x}`"))), + }; + let fr = comp.frame_rate; + let (a, b) = (fr.frame_at(fr.snap_nearest(layer.in_point.max(Tick::ZERO))), fr.frame_at(layer.out_point.min(comp.duration) - Tick(1))); + let times: Vec = (a..=b).map(|f| fr.tick_of(f)).collect(); + if times.len() < 2 { + return Err(bad(c, "the layer is too short")); + } + let o = CutOpts { threshold: f_p(p, "threshold").map(|t| (t as f32).clamp(0.01, 1.0)).unwrap_or(CutOpts::default().threshold), ..Default::default() }; + let lf = LayerFrames::new(s, cid, lid); + let name = layer.name.clone(); + let wait = b_p(p, "wait").unwrap_or(false); + s.spawn_task("sceneDetect", format!("Scene Edit Detection: {name}"), wait, move |ctl| { + let (cut_times, _) = analyse(&lf, ×, &o, &|d, t| ctl.progress(d, t))?; + let apply: Apply = Box::new(move |s: &mut Session| apply(s, cid, lid, mode, &cut_times)); + Ok(apply) + }) +} + +fn apply(s: &mut Session, cid: ItemId, lid: LayerId, mode: Mode, cuts: &[Tick]) -> Result { + let secs: Vec = cuts.iter().map(|t| t.seconds()).collect(); + if cuts.is_empty() { + return Ok(json!({"cuts": secs, "layers": [lid.0]})); + } + if mode == Mode::Markers { + s.edit("Scene Edit Detection", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + for (i, t) in cuts.iter().enumerate() { + let lt = l.layer_time(*t); + l.markers.retain(|m| m.time != lt); + l.markers.push(Marker { time: lt, comment: format!("Scene {}", i + 2), ..Default::default() }); + } + l.markers.sort_by_key(|m| m.time); + Ok(()) + })?; + return Ok(json!({"cuts": secs, "layers": [lid.0]})); + } + let snap = s.project.clone(); + // Split from the source layer: each piece ends at the next cut (pieces stacked like Split + // Layer: later pieces above). + let pieces = s.edit("Scene Edit Detection", None, |proj, st| { + let mut next = proj.next_id; + let cm = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let i = cm.layers.iter().position(|l| l.id == lid).ok_or(EngineError::NoComp)?; + let mut ids = vec![lid]; + for t in cuts { + let cur = cm.layers.iter().position(|l| l.id == *ids.last().expect("piece")).unwrap_or(i); + let l = &mut cm.layers[cur]; + if !(*t > l.in_point && *t < l.out_point) { + continue; + } + let mut b = l.clone(); + super::edit::reid(&mut b, &mut next); + b.in_point = *t; + l.out_point = *t; + ids.push(b.id); + cm.layers.insert(cur, b); + } + proj.next_id = next; + st.selected_layers = ids.clone(); + Ok(ids) + })?; + let mut out = pieces.clone(); + if mode == Mode::SplitPrecompose { + let base = s.project.comp(cid).and_then(|c| c.layer(lid)).map(|l| l.name.clone()).unwrap_or_default(); + out.clear(); + for (k, id) in pieces.iter().enumerate() { + let r = s.execute( + "layer.precompose", + json!({"comp": cid.0, "layers": [id.0], "mode": "move", "adjustDuration": true, "name": format!("{base} Scene {}", k + 1)}), + )?; + let new = r.get("layer").and_then(Value::as_u64).map(LayerId).unwrap_or(*id); + out.push(new); + } + s.collapse_undo("Scene Edit Detection", &snap); + } + Ok(json!({"cuts": secs, "layers": out.iter().map(|l| l.0).collect::>()})) +} + +pub fn specs() -> Vec { + vec![cmd!( + "layer.sceneEditDetection", + "Scene Edit Detection...", + ["Layer"], + None, + "{layer?, mode?: markers|split|splitPrecompose, threshold? (0…1, 0.25), wait?: bool}", + has_layers, + run + )] +} diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 273ecf99..440464c0 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -17,13 +17,7 @@ macro_rules! stub { pub fn specs() -> Vec { vec![ - // Layer ▸ New (3D / content-aware fill). - stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), // Mask options not in the mask model yet; pen / vertex editing (timeline milestone). - // Time (timeline milestone). - stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), - stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), - stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). stub!("camera.stereoRig", "Create Stereo 3D Rig", ["Layer", "Camera"], "{}"), stub!("camera.orbitNull", "Create Orbit Null", ["Layer", "Camera"], "{}"), @@ -39,7 +33,5 @@ pub fn specs() -> Vec { // Keyframes / text / tracking. stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), - // Workspaces and panels that don't exist yet. - stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] } diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs new file mode 100644 index 00000000..39a16915 --- /dev/null +++ b/crates/engine/src/jobs.rs @@ -0,0 +1,320 @@ +//! Background jobs: one list (Window ▸ Progress, `jobs.list`, `jobs.cancel`) over the render +//! queue, the analyses (tracker, mask tracker, Warp Stabilizer, 3D Camera Tracker, Roto Brush) and +//! the generic background tasks started here (Content-Aware Fill, Scene Edit Detection). +//! +//! A generic task runs a closure on a background thread (inline on wasm32 or when the caller +//! waits). The closure reports progress through [`TaskCtl`] and returns an [`Apply`]: a function +//! the session runs on the UI thread when [`Session::poll_jobs`] sees the task finish (that is +//! where the project is edited, as one undo step). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use serde::Serialize; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tracking::lock; + +/// Applied to the session when a task finishes successfully. +pub type Apply = Box crate::Result + Send>; + +/// Progress reporting and cancellation for a running task. +#[derive(Default)] +pub struct TaskCtl { + cancel: AtomicBool, + state: Mutex, +} + +#[derive(Clone, Debug, Default)] +struct TaskState { + done: u64, + total: u64, + message: String, + finished: bool, +} + +impl TaskCtl { + /// Report progress; returns `false` once the task was cancelled. + pub fn progress(&self, done: u64, total: u64) -> bool { + { + let mut s = lock(&self.state); + s.done = done; + s.total = total; + } + crate::offload::report(done, total); + !self.cancelled() + } + pub fn message(&self, m: impl Into) { + lock(&self.state).message = m.into(); + } + pub fn cancelled(&self) -> bool { + self.cancel.load(Ordering::Relaxed) + } +} + +type Work = Box Result + Send>; + +/// A generic background task. +pub struct Task { + pub id: u64, + pub kind: &'static str, + pub label: String, + pub ctl: Arc, + started: web_time::Instant, + result: Arc>>>, + thread: Option>, +} + +/// A finished job (kept for the Progress panel and agents). +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobRecord { + pub id: String, + pub kind: String, + pub label: String, + /// `done`, `cancelled` or `failed`. + pub status: String, + pub message: String, + pub result: Value, + pub seconds: f64, +} + +/// One row of the Progress panel. +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct JobInfo { + /// `render`, `track`, `maskTrack`, `warp`, `camera`, `roto`, or `task:`. + pub id: String, + pub kind: String, + pub label: String, + pub done: u64, + pub total: u64, + /// 0…1 (`None` when the total is not known yet). + pub fraction: Option, + pub running: bool, + pub message: String, + pub cancellable: bool, +} + +fn info(id: &str, kind: &str, label: String, done: u64, total: u64, running: bool, message: String) -> JobInfo { + JobInfo { + id: id.into(), + kind: kind.into(), + label, + done, + total, + fraction: (total > 0).then(|| (done as f64 / total as f64).clamp(0.0, 1.0)), + running, + message, + cancellable: running, + } +} + +impl Session { + /// Start a background task. `wait` (and wasm32) runs it to completion and applies it before + /// returning; the result is then the apply's value. + pub fn spawn_task( + &mut self, + kind: &'static str, + label: impl Into, + wait: bool, + work: impl FnOnce(&TaskCtl) -> Result + Send + 'static, + ) -> crate::Result { + let id = self.next_task_id; + self.next_task_id += 1; + let ctl = Arc::new(TaskCtl::default()); + let result: Arc>>> = Arc::new(Mutex::new(None)); + let label = label.into(); + let wait = wait || cfg!(target_arch = "wasm32"); + let work: Work = Box::new(work); + let run = { + let (ctl, result) = (ctl.clone(), result.clone()); + move || { + let r = work(&ctl); + *lock(&result) = Some(r); + lock(&ctl.state).finished = true; + } + }; + let thread = if wait { + run(); + None + } else { + Some(std::thread::Builder::new().name(format!("task-{kind}")).spawn(run).map_err(|e| crate::EngineError::Other(e.to_string()))?) + }; + self.tasks.push(Task { id, kind, label, ctl, started: web_time::Instant::now(), result, thread }); + if wait { + let rec = self.finish_task(self.tasks.len() - 1); + return match rec.status.as_str() { + "done" => Ok(rec.result), + _ => Err(crate::EngineError::Other(if rec.message.is_empty() { rec.status } else { rec.message })), + }; + } + Ok(json!({"job": format!("task:{id}"), "started": true})) + } + + fn finish_task(&mut self, i: usize) -> JobRecord { + let mut t = self.tasks.remove(i); + if let Some(th) = t.thread.take() { + let _ = th.join(); + } + let r = lock(&t.result).take(); + let cancelled = t.ctl.cancelled(); + let (status, message, result) = match r { + Some(Ok(apply)) if !cancelled => match apply(self) { + Ok(v) => ("done", String::new(), v), + Err(e) => ("failed", e.to_string(), Value::Null), + }, + Some(Ok(_)) => ("cancelled", String::new(), Value::Null), + Some(Err(e)) if cancelled || e == crate::render_queue::CANCELLED => ("cancelled", String::new(), Value::Null), + Some(Err(e)) => ("failed", e, Value::Null), + None => ("failed", "the task stopped".into(), Value::Null), + }; + let rec = JobRecord { + id: format!("task:{}", t.id), + kind: t.kind.into(), + label: t.label.clone(), + status: status.into(), + message: message.clone(), + result, + seconds: t.started.elapsed().as_secs_f64(), + }; + let toast = match status { + "done" => format!("{} finished", t.label), + "cancelled" => format!("{} cancelled", t.label), + _ => format!("{} failed: {message}", t.label), + }; + self.events.push(crate::Event::Toast { message: toast, error: status == "failed" }); + self.job_log.push(rec.clone()); + if self.job_log.len() > 50 { + self.job_log.remove(0); + } + rec + } + + /// Apply finished tasks. Frontends call this every frame while [`Session::tasks`] is not empty. + pub fn poll_jobs(&mut self) -> bool { + let mut any = false; + while let Some(i) = self.tasks.iter().position(|t| lock(&t.ctl.state).finished) { + self.finish_task(i); + any = true; + } + any + } + + /// Wait for every running task (CLI, tests). + pub fn wait_jobs(&mut self) { + while !self.tasks.is_empty() { + if let Some(th) = self.tasks.iter_mut().find_map(|t| t.thread.take()) { + let _ = th.join(); + } + self.poll_jobs(); + if self.tasks.iter().all(|t| t.thread.is_none() && !lock(&t.ctl.state).finished) { + break; + } + } + } + + /// Every running background job. + pub fn jobs(&self) -> Vec { + let mut out = vec![]; + if let Some(st) = self.render_progress() + && !st.finished + { + let name = st + .current + .and_then(|id| self.project.render_queue.iter().find(|i| i.id == id)) + .and_then(|i| self.project.comp(i.comp).map(|_| i.comp)) + .and_then(|c| self.project.item(c)) + .map(|i| i.name.clone()); + out.push(info( + "render", + "render", + format!("Render Queue{}", name.map(|n| format!(": {n}")).unwrap_or_default()), + st.done, + st.total, + true, + format!("item {} of {}", (st.items_done + 1).min(st.items_total.max(1)), st.items_total), + )); + } + if let Some(p) = self.track_progress() + && !p.finished + { + out.push(info("track", "track", "Track Motion".into(), p.done, p.total, true, String::new())); + } + if let Some(p) = self.mask_track_progress() + && !p.finished + { + out.push(info("maskTrack", "maskTrack", "Track Mask".into(), p.done, p.total, true, String::new())); + } + if let Some(p) = self.warp_progress() + && !p.finished + { + out.push(info("warp", "warp", "Warp Stabilizer analysis".into(), p.done, p.total, true, format!("step {}", p.step))); + } + if let Some(p) = self.camera_progress() + && !p.finished + { + out.push(info("camera", "camera", "3D Camera Tracker analysis".into(), p.done, p.total, true, format!("step {}", p.step))); + } + if let Some(p) = self.roto_progress() + && !p.finished + { + out.push(info("roto", "roto", "Roto Brush propagation".into(), p.done, p.total, true, String::new())); + } + for t in &self.tasks { + let s = lock(&t.ctl.state).clone(); + out.push(info(&format!("task:{}", t.id), t.kind, t.label.clone(), s.done, s.total, !s.finished, s.message)); + } + out + } + + /// Cancel a job by id (`render`, `track`, …, `task:`; `all` cancels everything). + pub fn cancel_job(&mut self, id: &str) -> bool { + match id { + "render" => self.stop_render(), + "track" => self.stop_track(), + "maskTrack" => self.stop_mask_track(), + "warp" => self.stop_warp(), + "camera" => self.stop_camera(), + "roto" => self.stop_roto(), + "all" => { + let ids: Vec = self.jobs().into_iter().map(|j| j.id).collect(); + ids.iter().fold(false, |a, j| self.cancel_job(j) | a) + } + _ => { + let Some(n) = id.strip_prefix("task:").and_then(|n| n.parse::().ok()) else { return false }; + match self.tasks.iter().find(|t| t.id == n) { + Some(t) => { + t.ctl.cancel.store(true, Ordering::Relaxed); + true + } + None => false, + } + } + } + } + + /// Fold the undo steps recorded since the project was `snap` into one step called `name` + /// (commands built from several commands). + pub fn collapse_undo(&mut self, name: &str, snap: &Arc) { + let h = &mut self.history; + let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { return }; + let first = h.undo[start].1.clone(); + h.undo.truncate(start); + h.undo.push((name.to_string(), first)); + h.merge_key = None; + } + + /// Cancel and drop every task (new/open project). + pub(crate) fn drop_tasks(&mut self) { + for t in &self.tasks { + t.ctl.cancel.store(true, Ordering::Relaxed); + } + for mut t in std::mem::take(&mut self.tasks) { + if let Some(th) = t.thread.take() { + let _ = th.join(); + } + } + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 508fb72e..c8efb428 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,8 +13,10 @@ pub mod camera_track; pub mod commands; pub mod config; pub mod demo; +pub mod jobs; pub mod links; pub mod mask_track; +pub mod media_browser; pub mod menus; pub mod offload; pub mod prefs; @@ -43,6 +45,7 @@ pub use effectcraft_effects as effects; pub use effectcraft_geom as geom; pub use effectcraft_keyframe as keyframe; pub use effectcraft_project as project; +pub use effectcraft_raster as raster; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; @@ -216,6 +219,15 @@ pub struct EditorState { /// Essential Graphics can expose). #[serde(default)] pub essential_solo: bool, + /// The Footage panel: the footage shown, its time and In/Out marks (source time). + #[serde(default)] + pub footage_panel: Option, + /// Media Browser: the folder shown and the favourites. + #[serde(default)] + pub media_browser: media_browser::BrowserState, + /// Content-Aware Fill panel settings. + #[serde(default)] + pub content_fill: commands::content_fill::FillSettings, } fn one_view() -> u8 { @@ -322,6 +334,11 @@ pub struct Session { pub offload: Option>, /// Analyses running in the [`Session::offload`] worker. pub offloaded: Vec, + /// Generic background tasks (Content-Aware Fill, Scene Edit Detection), see [`jobs`]. + pub tasks: Vec, + /// Finished tasks, newest last (Progress panel, `jobs.list`). + pub job_log: Vec, + pub next_task_id: u64, } /// A script to run (see [`Session::script`]). @@ -378,6 +395,9 @@ impl Default for Session { script: None, offload: None, offloaded: vec![], + tasks: vec![], + job_log: vec![], + next_task_id: 1, } } } @@ -634,6 +654,7 @@ impl Session { self.stop_roto(); self.roto_job = None; self.roto_pending.clear(); + self.drop_tasks(); p.fix_next_id(); upgrade_effects(&mut p); self.project = Arc::new(p); @@ -743,4 +764,6 @@ pub fn text_families() -> Vec { #[cfg(test)] mod tests_paint; #[cfg(test)] +mod tests_panels; +#[cfg(test)] mod tests_roto; diff --git a/crates/engine/src/media_browser.rs b/crates/engine/src/media_browser.rs new file mode 100644 index 00000000..afc7dc34 --- /dev/null +++ b/crates/engine/src/media_browser.rs @@ -0,0 +1,270 @@ +//! Media Browser (Window ▸ Media Browser): browse the local file system for footage, keep +//! favourite folders, read file metadata, and import (`file.import`). Desktop only: the web build +//! has no file system to browse ([`available`] is false there). +//! +//! Also the metadata the Metadata panel shows for project items and files (codec, size, rate, +//! duration, colour profile, dates). + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use effectcraft_project::{Footage, FootageKind, Item, ItemKind, Project}; + +/// Media Browser state (in [`crate::EditorState`], serde for agents). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BrowserState { + /// The folder shown (`None` = the home folder). + #[serde(default)] + pub folder: Option, + /// Favourite folders. + #[serde(default)] + pub favorites: Vec, + /// Show only importable files. + #[serde(default)] + pub importable_only: bool, +} + +/// Whether this build can browse files. +pub fn available() -> bool { + !cfg!(target_arch = "wasm32") +} + +/// One directory entry. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Entry { + pub name: String, + pub path: String, + pub is_dir: bool, + pub size: u64, + /// Seconds since the Unix epoch. + pub modified: Option, + /// Footage kind guessed from the extension (`video`, `image`, `audio`, `data`, `model`, + /// `project`), `None` for other files and folders. + pub kind: Option<&'static str>, +} + +const VIDEO: &[&str] = &["mp4", "m4v", "mov", "webm", "mkv", "avi", "mxf", "gif"]; +const IMAGE: &[&str] = &["png", "jpg", "jpeg", "tif", "tiff", "exr", "bmp", "webp", "psd", "psb", "svg", "tga", "dpx", "hdr"]; +const AUDIO: &[&str] = &["wav", "aif", "aiff", "mp3", "m4a", "aac", "flac", "ogg", "opus"]; +const DATA: &[&str] = &["json", "csv", "tsv", "mgjson"]; +const MODEL: &[&str] = &["gltf", "glb", "obj"]; + +/// Kind of file from its extension. +pub fn kind_of(path: &str) -> Option<&'static str> { + let ext = std::path::Path::new(path).extension()?.to_str()?.to_ascii_lowercase(); + let e = ext.as_str(); + if VIDEO.contains(&e) { + Some("video") + } else if IMAGE.contains(&e) { + Some("image") + } else if AUDIO.contains(&e) { + Some("audio") + } else if DATA.contains(&e) { + Some("data") + } else if MODEL.contains(&e) { + Some("model") + } else if matches!(e, "ecproj" | "ectemplate" | "json5") { + Some("project") + } else { + None + } +} + +/// The user's home folder (or `/`). +pub fn home() -> String { + std::env::var("HOME").or_else(|_| std::env::var("USERPROFILE")).unwrap_or_else(|_| "/".into()) +} + +fn unix(t: std::io::Result) -> Option { + t.ok()?.duration_since(std::time::UNIX_EPOCH).ok().map(|d| d.as_secs()) +} + +/// List a folder: folders first, then files, by name (hidden entries skipped). +pub fn list(dir: &str, importable_only: bool) -> Result, String> { + if !available() { + return Err("the Media Browser needs the desktop app".into()); + } + let rd = std::fs::read_dir(dir).map_err(|e| format!("{dir}: {e}"))?; + let mut out: Vec = rd + .filter_map(|e| e.ok()) + .filter_map(|e| { + let name = e.file_name().to_string_lossy().to_string(); + if name.starts_with('.') { + return None; + } + let md = e.metadata().ok()?; + let path = e.path().to_string_lossy().to_string(); + let kind = if md.is_dir() { None } else { kind_of(&path) }; + if importable_only && !md.is_dir() && kind.is_none() { + return None; + } + Some(Entry { name, path, is_dir: md.is_dir(), size: if md.is_dir() { 0 } else { md.len() }, modified: unix(md.modified()), kind }) + }) + .collect(); + out.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.to_lowercase().cmp(&b.name.to_lowercase()))); + Ok(out) +} + +/// The parent folder of `dir`. +pub fn parent(dir: &str) -> Option { + std::path::Path::new(dir).parent().map(|p| p.to_string_lossy().to_string()).filter(|p| !p.is_empty()) +} + +/// Format seconds since the epoch as an ISO 8601 UTC date-time. +pub fn iso_date(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let rem = secs % 86_400; + // Civil-from-days (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +/// File dates and size: (created, modified, size). +pub fn file_dates(path: &str) -> (Option, Option, Option) { + if !available() || path.is_empty() { + return (None, None, None); + } + match std::fs::metadata(path) { + Ok(md) => (unix(md.created()), unix(md.modified()), Some(md.len())), + Err(_) => (None, None, None), + } +} + +fn kind_name(k: FootageKind) -> &'static str { + match k { + FootageKind::Video => "video", + FootageKind::Still => "still", + FootageKind::Sequence => "sequence", + FootageKind::Audio => "audio", + FootageKind::Model => "model", + FootageKind::Data => "data", + } +} + +/// Metadata of footage (the fields the Metadata panel lists). +pub fn footage_metadata(f: &Footage) -> Value { + let (created, modified, size) = file_dates(f.sequence.first().map(String::as_str).unwrap_or(&f.path)); + json!({ + "path": f.path, + "kind": kind_name(f.kind), + "codec": f.codec, + "width": f.width, + "height": f.height, + "pixelAspect": f.pixel_aspect, + "frameRate": f.frame_rate.as_f64(), + "duration": f.duration.seconds(), + "frames": if f.has_video && f.kind != FootageKind::Still { Some(f.frame_rate.frame_at(f.duration)) } else { None }, + "hasVideo": f.has_video, + "hasAudio": f.has_audio, + "alpha": format!("{:?}", f.alpha), + "colorProfile": f.color_profile.map(|c| format!("{c:?}")).unwrap_or_else(|| "sRGB (default)".into()), + "linearLight": f.linear_light, + "fields": format!("{:?}", f.fields), + "sequenceFiles": f.sequence.len(), + "dataRows": f.data.as_ref().map(|d| d.lines().count().saturating_sub(1)), + "created": created.map(iso_date), + "modified": modified.map(iso_date), + "fileSize": size, + "missing": f.missing, + }) +} + +/// Metadata of a project item. +pub fn item_metadata(p: &Project, it: &Item) -> Value { + let mut v = json!({ + "id": it.id.0, + "name": it.name, + "type": it.type_name(), + "comment": it.comment, + "label": format!("{:?}", it.label), + }); + let extra = match &it.kind { + ItemKind::Footage(f) => footage_metadata(f), + ItemKind::Comp(c) => json!({ + "width": c.width, + "height": c.height, + "pixelAspect": c.pixel_aspect, + "frameRate": c.frame_rate.as_f64(), + "duration": c.duration.seconds(), + "layers": c.layers.len(), + "workArea": [c.work_area.0.seconds(), c.work_area.1.seconds()], + }), + ItemKind::Solid(s) => json!({"width": s.width, "height": s.height, "color": s.color}), + ItemKind::Folder => json!({"items": p.items.values().filter(|i| i.parent == Some(it.id)).count()}), + }; + if let (Value::Object(a), Value::Object(b)) = (&mut v, extra) { + a.extend(b); + } + if let Some(px) = &it.proxy { + v["proxy"] = json!(px.footage.path); + } + v +} + +/// Project-level metadata. +pub fn project_metadata(p: &Project, path: Option<&str>) -> Value { + let st = &p.settings; + let count = |f: &dyn Fn(&ItemKind) -> bool| p.items.values().filter(|i| f(&i.kind)).count(); + json!({ + "path": path, + "comment": st.comment, + "bitDepth": st.bit_depth.label(), + "workingSpace": st.working_space.map(|c| format!("{c:?}")), + "linearize": st.linearize, + "timeDisplay": format!("{:?}", st.time_display), + "audioSampleRate": st.audio_sample_rate, + "items": p.items.len(), + "compositions": count(&|k| matches!(k, ItemKind::Comp(_))), + "footage": count(&|k| matches!(k, ItemKind::Footage(_))), + "solids": count(&|k| matches!(k, ItemKind::Solid(_))), + "folders": count(&|k| matches!(k, ItemKind::Folder)), + "renderQueue": p.render_queue.len(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lists_a_folder() { + let dir = std::env::temp_dir().join(format!("ec-mb-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("Shots")).unwrap(); + std::fs::write(dir.join("b.png"), b"x").unwrap(); + std::fs::write(dir.join("A.mov"), b"xy").unwrap(); + std::fs::write(dir.join("notes.txt"), b"xyz").unwrap(); + std::fs::write(dir.join(".hidden"), b"").unwrap(); + let d = dir.to_string_lossy().to_string(); + let all = list(&d, false).unwrap(); + let names: Vec<&str> = all.iter().map(|e| e.name.as_str()).collect(); + assert_eq!(names, ["Shots", "A.mov", "b.png", "notes.txt"]); + assert!(all[0].is_dir); + assert_eq!(all[1].kind, Some("video")); + assert_eq!(all[2].kind, Some("image")); + assert_eq!(all[1].size, 2); + let media = list(&d, true).unwrap(); + assert_eq!(media.len(), 3); + assert_eq!(parent(&dir.join("Shots").to_string_lossy()).as_deref(), Some(d.as_str())); + assert!(list(&dir.join("nope").to_string_lossy(), false).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn iso_dates() { + assert_eq!(iso_date(0), "1970-01-01T00:00:00Z"); + assert_eq!(iso_date(1_700_000_000), "2023-11-14T22:13:20Z"); + assert_eq!(iso_date(951_782_400), "2000-02-29T00:00:00Z"); + } +} diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 0470625c..2ae54aee 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -942,19 +942,19 @@ Window Audio | window.panel {"panel":"audio"} | Cmd+4 Brushes | window.panel {"panel":"brushes"} | Cmd+9 Character | window.panel {"panel":"character"} | Cmd+6 - Content-Aware Fill | window.unavailablePanel {"panel":"contentAwareFill"} + Content-Aware Fill | window.panel {"panel":"contentAwareFill"} Effects & Presets | window.panel {"panel":"effectsPresets"} | Cmd+5 Essential Graphics | window.panel {"panel":"essentialGraphics"} Info | window.panel {"panel":"info"} | Cmd+2 - Lumetri Scopes | window.unavailablePanel {"panel":"lumetriScopes"} + Lumetri Scopes | window.panel {"panel":"lumetriScopes"} Mask Interpolation | window.panel {"panel":"maskInterpolation"} - Media Browser | window.unavailablePanel {"panel":"mediaBrowser"} - Metadata | window.unavailablePanel {"panel":"metadata"} + Media Browser | window.panel {"panel":"mediaBrowser"} + Metadata | window.panel {"panel":"metadata"} Motion Sketch | window.panel {"panel":"motionSketch"} Paint | window.panel {"panel":"paint"} | Cmd+8 Paragraph | window.panel {"panel":"paragraph"} | Cmd+7 Preview | window.panel {"panel":"preview"} | Cmd+3 - Progress | window.unavailablePanel {"panel":"progress"} + Progress | window.panel {"panel":"progress"} Properties | window.panel {"panel":"properties"} Script Console | window.panel {"panel":"scriptConsole"} Smoother | window.panel {"panel":"smoother"} @@ -965,7 +965,7 @@ Window Composition | window.panel {"panel":"composition"} Effect Controls | window.panel {"panel":"effectControls"} | F3 Flowchart | window.panel {"panel":"flowchart"} - Footage | window.unavailablePanel {"panel":"footage"} + Footage | window.panel {"panel":"footage"} Layer | window.panel {"panel":"layer"} Project | window.panel {"panel":"project"} | Cmd+0 Render Queue | window.panel {"panel":"renderQueue"} | Cmd+Alt+0 diff --git a/crates/engine/src/tests_panels.rs b/crates/engine/src/tests_panels.rs new file mode 100644 index 00000000..d989b28e --- /dev/null +++ b/crates/engine/src/tests_panels.rs @@ -0,0 +1,518 @@ +//! M13.6 panels and tools through the commands: Progress (jobs), Media Browser, Metadata, +//! Lumetri Scopes, the Footage panel's Overlay / Ripple Insert edits, Content-Aware Fill, Scene +//! Edit Detection, Auto-trace and Align Video to Data. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource, build}; +use effectcraft_raster::Image; +use effectcraft_raster::inpaint::psnr; +use effectcraft_render::FootageSource; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::{Value, json}; + +use crate::Session; + +const FPS: u32 = 25; + +/// Footage computed per frame. +struct Gen { + make: Box Image + Send + Sync>, + cache: Mutex>>, +} + +impl FootageSource for Gen { + fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option> { + let f = (t.seconds() * FPS as f64).round().max(0.0) as u32; + let mut c = self.cache.lock().unwrap(); + Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone()) + } +} + +/// A comp `w × h`, `secs` long, with one footage layer drawn by `make` (returns session, comp, +/// layer, footage item). +fn setup(w: u32, h: u32, secs: f64, make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, ItemId, LayerId, ItemId) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Panels", "width": w, "height": h, "frameRate": FPS, "duration": secs})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let (lid, item) = s + .edit("clip", None, |p, _| { + let footage = Footage { + path: "synthetic.mov".into(), + kind: FootageKind::Video, + width: w, + height: h, + frame_rate: FrameRate::new(FPS as i64, 1), + duration: Tick::from_seconds_f64(secs), + has_video: true, + codec: "SYN".into(), + ..Default::default() + }; + let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); + let comp = p.comp(cid).unwrap().clone(); + let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (w, h), None); + let id = l.id; + p.comp_mut(cid).unwrap().layers.insert(0, l); + Ok((id, item)) + }) + .unwrap(); + s.footage = Arc::new(Gen { make: Box::new(make), cache: Mutex::default() }); + s.execute("layer.select", json!({"layers": [lid.0]})).unwrap(); + (s, cid, lid, item) +} + +fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("ec-panels-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +// ---------------------------------------------------------------- Progress + +#[test] +fn progress_lists_and_cancels_jobs() { + let mut s = Session::default(); + let r = s + .spawn_task("test", "Endless job", false, |ctl| { + let mut k = 0; + while ctl.progress(k % 100, 100) { + std::thread::sleep(std::time::Duration::from_millis(2)); + k += 1; + } + Err(crate::render_queue::CANCELLED.into()) + }) + .unwrap(); + let id = r["job"].as_str().unwrap().to_string(); + let listed = s.execute("jobs.list", json!({})).unwrap(); + let running = listed["running"].as_array().unwrap(); + assert_eq!(running.len(), 1); + assert_eq!(running[0]["id"], json!(id)); + assert_eq!(running[0]["label"], json!("Endless job")); + assert_eq!(running[0]["cancellable"], json!(true)); + assert_eq!(s.execute("jobs.cancel", json!({"job": id})).unwrap()["cancelled"], json!(true)); + s.execute("jobs.wait", json!({})).unwrap(); + assert!(s.jobs().is_empty()); + assert_eq!(s.job_log.last().unwrap().status, "cancelled"); + // A finished job applies its edit (one undo step) and reports its result. + let v = s + .spawn_task("test", "Quick job", true, |ctl| { + ctl.progress(1, 1); + let apply: crate::jobs::Apply = Box::new(|s: &mut Session| { + s.edit("Quick", None, |p, _| { + p.settings.comment = "done".into(); + Ok(()) + })?; + Ok(json!(42)) + }); + Ok(apply) + }) + .unwrap(); + assert_eq!(v, json!(42)); + assert_eq!(s.project.settings.comment, "done"); + assert_eq!(s.job_log.last().unwrap().status, "done"); + assert!(s.undo()); + assert_eq!(s.project.settings.comment, ""); +} + +// ---------------------------------------------------------------- Media Browser & Metadata + +#[test] +fn media_browser_and_metadata() { + let dir = tmp("browser"); + std::fs::create_dir_all(dir.join("Footage")).unwrap(); + std::fs::write(dir.join("shot.mov"), b"0123").unwrap(); + std::fs::write(dir.join("data.csv"), "time,v\n0,1\n").unwrap(); + std::fs::write(dir.join("readme.txt"), "x").unwrap(); + let d = dir.to_string_lossy().to_string(); + let mut s = Session::default(); + let r = s.execute("mediaBrowser.go", json!({"path": d})).unwrap(); + let names: Vec<&str> = r["entries"].as_array().unwrap().iter().map(|e| e["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Footage", "data.csv", "readme.txt", "shot.mov"]); + assert_eq!(s.state.media_browser.folder.as_deref(), Some(d.as_str())); + let r = s.execute("mediaBrowser.list", json!({"importableOnly": true})).unwrap(); + assert_eq!(r["entries"].as_array().unwrap().len(), 3); + s.execute("mediaBrowser.addFavorite", json!({})).unwrap(); + assert_eq!(s.state.media_browser.favorites, vec![d.clone()]); + s.execute("mediaBrowser.go", json!({"path": dir.join("Footage").to_string_lossy()})).unwrap(); + s.execute("mediaBrowser.go", json!({"path": ".."})).unwrap(); + assert_eq!(s.state.media_browser.folder.as_deref(), Some(d.as_str())); + s.execute("mediaBrowser.removeFavorite", json!({"path": d})).unwrap(); + assert!(s.state.media_browser.favorites.is_empty()); + // Import a data file from the browser; its metadata reads back. + let r = s.execute("mediaBrowser.import", json!({"paths": [dir.join("data.csv").to_string_lossy()]})).unwrap(); + let id = r["items"][0].as_u64().unwrap(); + let m = s.execute("item.metadata", json!({"item": id})).unwrap(); + assert_eq!(m["item"]["type"], json!("Data")); + assert_eq!(m["item"]["codec"], json!("CSV")); + assert_eq!(m["item"]["dataRows"], json!(1)); + assert!(m["item"]["modified"].as_str().unwrap().ends_with('Z')); + assert_eq!(m["project"]["footage"], json!(1)); + // Editable comments: item (undoable) and project. + s.execute("project.setComment", json!({"items": [id], "comment": "telemetry"})).unwrap(); + s.execute("project.setProjectComment", json!({"comment": "client cut"})).unwrap(); + let m = s.execute("item.metadata", json!({"item": id})).unwrap(); + assert_eq!(m["item"]["comment"], json!("telemetry")); + assert_eq!(m["project"]["comment"], json!("client cut")); + s.undo(); + assert_eq!(s.project.settings.comment, ""); + let fi = s.execute("mediaBrowser.fileInfo", json!({"path": dir.join("shot.mov").to_string_lossy()})).unwrap(); + assert_eq!(fi["size"], json!(4)); + assert_eq!(fi["kind"], json!("video")); + std::fs::remove_dir_all(&dir).unwrap(); +} + +// ---------------------------------------------------------------- Lumetri Scopes + +#[test] +fn scopes_read_the_comp_frame() { + let mut s = Session::default(); + s.execute("comp.new", json!({"width": 64, "height": 48, "frameRate": 25, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + let v = s.execute("scopes.analyze", json!({"scope": "vectorscopeYuv", "size": 101})).unwrap(); + let st = effectcraft_raster::scopes::ColorStandard::Rec709; + let [_, cb, cr] = st.ycbcr([1.0, 0.0, 0.0]); + assert_eq!(v["peaks"][0], json!([((cb + 0.5) * 100.0).round() as u32, ((0.5 - cr) * 100.0).round() as u32])); + let h = s.execute("scopes.analyze", json!({"scope": "histogram", "standard": "rec601"})).unwrap(); + let bars = &h["histogram"]; + assert_eq!(bars[0][255], json!(64 * 48)); + assert_eq!(bars[1][0], json!(64 * 48)); + assert!((h["luma"]["mean"].as_f64().unwrap() - 0.299).abs() < 1e-3); + assert!(s.execute("scopes.analyze", json!({"scope": "bogus"})).is_err()); +} + +// ---------------------------------------------------------------- Footage panel + +#[test] +fn footage_panel_overlay_and_ripple_insert() { + let (mut s, cid, clip, item) = setup(32, 24, 8.0, |_| Image::filled(32, 24, [0.5, 0.5, 0.5, 1.0])); + s.execute("footage.open", json!({"item": item.0})).unwrap(); + assert!( + s.drain_events().iter().any(|e| matches!(e, crate::Event::Frontend { command, params } if command == "window.panel" && params["panel"] == "footage")) + ); + s.execute("footage.setTime", json!({"time": 1.0})).unwrap(); + s.execute("footage.setIn", json!({})).unwrap(); + s.execute("footage.setOut", json!({"frame": 49})).unwrap(); + let info = s.execute("footage.info", json!({})).unwrap(); + assert_eq!(info["in"], json!(1.0)); + assert_eq!(info["out"], json!(2.0)); + // Overlay Edit at 0.6 s: a 1 s layer on top showing source 1–2 s. + s.set_time(Tick::from_seconds_f64(0.6)); + let r = s.execute("footage.overlayEdit", json!({})).unwrap(); + let ov = LayerId(r["layer"].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers[0].id, ov); + let l = comp.layer(ov).unwrap(); + assert_eq!((l.in_point.seconds(), l.out_point.seconds(), l.start_time.seconds()), (0.6, 1.6, -0.4)); + assert_eq!(comp.layer(clip).unwrap().out_point.seconds(), 8.0, "overlay leaves other layers alone"); + // Ripple Insert at 3 s: layers crossing 3 s are split, the rest moves back by 1 s. + s.set_time(Tick::from_seconds_f64(3.0)); + let r = s.execute("footage.rippleInsertEdit", json!({})).unwrap(); + let ins = LayerId(r["layer"].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers.len(), 4, "overlay, insert, clip split in two"); + let l = comp.layer(ins).unwrap(); + assert_eq!((l.in_point.seconds(), l.out_point.seconds()), (3.0, 4.0)); + let head = comp.layer(clip).unwrap(); + assert_eq!((head.in_point.seconds(), head.out_point.seconds()), (0.0, 3.0)); + let tail = comp.layers.iter().find(|x| x.id != clip && x.id != ins && x.id != ov).unwrap(); + assert_eq!((tail.in_point.seconds(), tail.out_point.seconds(), tail.start_time.seconds()), (4.0, 9.0, 1.0)); + // Source time continuity: the tail shows source 3 s at comp 4 s. + assert_eq!(tail.layer_time(Tick::from_seconds_f64(4.0)).seconds(), 3.0); + // The overlay layer (in before 3 s, out before 3 s) didn't move. + assert_eq!(comp.layer(ov).unwrap().in_point.seconds(), 0.6); + // One undo step each. + assert!(s.undo()); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 2); + assert!(s.execute("footage.setOut", json!({"time": 0.5})).is_ok()); + assert_eq!(s.state.footage_panel.as_ref().unwrap().in_point, None, "an Out before the In clears the In"); +} + +// ---------------------------------------------------------------- Content-Aware Fill + +fn bg(x: f32, y: f32) -> [f32; 4] { + let s = 0.5 + 0.3 * (x * std::f32::consts::TAU / 10.0).sin(); + let n = effectcraft_raster::hash_noise(((x.floor() as i64).rem_euclid(20) / 4) as u32, ((y as u32) % 20) / 4, 3); + [s, 0.3 + 0.4 * n, 0.6 - 0.3 * s * n, 1.0] +} + +/// Panning textured background with a red "object" walking through the masked region. +fn caf_frame(f: u32, with_object: bool) -> Image { + let (w, h) = (96u32, 64u32); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let mut p = bg(x as f32 + 2.0 * f as f32, y as f32); + let ox = 40 + (f % 4); + if with_object && (ox..ox + 10).contains(&x) && (26..36).contains(&y) { + p = [0.9, 0.1, 0.1, 1.0]; + } + img.set(x, y, p); + } + } + img +} + +#[test] +fn content_aware_fill_removes_an_object() { + let (mut s, cid, clip, _) = setup(96, 64, 0.48, |f| caf_frame(f, true)); + // Cut the object out with a Subtract mask; fill the work area (12 frames). + s.execute("layer.addMask", json!({"rect": [36.0, 22.0, 20.0, 18.0], "mode": "subtract"})).unwrap(); + let out = tmp("caf"); + s.execute("contentFill.set", json!({"method": "object", "range": "workArea", "alphaExpansion": 1})).unwrap(); + let r = s.execute("contentFill.generate", json!({"outputDir": out.to_string_lossy(), "wait": true})).unwrap(); + let fill = LayerId(r["layer"].as_u64().unwrap()); + let files: Vec = r["files"].as_array().unwrap().iter().map(|v| v.as_str().unwrap().to_string()).collect(); + assert_eq!(files.len(), 12); + // The Fill layer sits right above the source with the source's timing. + let comp = s.project.comp(cid).unwrap(); + let i = comp.layers.iter().position(|l| l.id == fill).unwrap(); + assert_eq!(comp.layers[i + 1].id, clip); + let fl = comp.layer(fill).unwrap(); + assert_eq!((fl.in_point, fl.out_point.seconds()), (Tick::ZERO, 0.48)); + let LayerSource::Footage { item } = fl.source else { panic!("footage") }; + let Some(ItemKind::Footage(f)) = s.project.item(item).map(|i| &i.kind) else { panic!() }; + assert_eq!((f.kind, f.sequence.len()), (FootageKind::Sequence, 12)); + assert!(s.project.item(item).unwrap().name.starts_with("Fill 1")); + // The filled frames match the clean background inside the mask, and stay steady over time. + let region: Vec = (0..96 * 64).map(|i| (36..56).contains(&(i % 96)) && (22..40).contains(&(i / 96))).collect(); + let mut prev: Option = None; + for (k, path) in files.iter().enumerate() { + let png = image::open(path).unwrap().to_rgba8(); + let img = Image::from_rgba8(96, 64, png.as_raw()); + let truth = caf_frame(k as u32, false); + let db = psnr(&img, &truth, Some(®ion)); + assert!(db > 25.0, "frame {k}: {db:.1} dB"); + // Outside the hole the frame is the source itself. + let outside: Vec = region.iter().map(|r| !r).collect(); + assert!(psnr(&img, &caf_frame(k as u32, true), Some(&outside)) > 40.0); + if let Some(p) = &prev { + // Temporal consistency: the change in the hole follows the 2 px/frame pan (compare + // against the previous fill shifted). + let mut shifted = p.clone(); + for y in 0..64 { + for x in 0..94 { + shifted.data[y * 96 + x] = p.data[y * 96 + x + 2]; + } + } + let inner: Vec = (0..96 * 64).map(|i| (38..54).contains(&(i % 96)) && (24..38).contains(&(i / 96))).collect(); + assert!(psnr(&img, &shifted, Some(&inner)) > 24.0, "frame {k} jumps"); + } + prev = Some(img); + } + // One undo step removes the fill layer. + assert!(s.undo()); + assert!(s.project.comp(cid).unwrap().layer(fill).is_none()); + std::fs::remove_dir_all(&out).unwrap(); + // Without a hole there is nothing to fill. + let (mut s2, _, _, _) = setup(32, 24, 0.2, |_| Image::filled(32, 24, [0.5, 0.5, 0.5, 1.0])); + assert!(s2.execute("contentFill.generate", json!({"wait": true, "outputDir": tmp("caf2").to_string_lossy()})).is_err()); +} + +#[test] +fn content_aware_fill_runs_in_the_background() { + let (mut s, _, _, _) = setup(48, 32, 0.2, |f| { + let mut img = Image::new(48, 32); + for y in 0..32 { + for x in 0..48 { + img.set(x, y, bg(x as f32 + f as f32, y as f32)); + } + } + img + }); + s.execute("layer.addMask", json!({"rect": [20.0, 10.0, 8.0, 8.0], "mode": "subtract"})).unwrap(); + let out = tmp("cafbg"); + let r = s.execute("contentFill.generate", json!({"method": "edgeBlend", "outputDir": out.to_string_lossy()})).unwrap(); + assert!(r["job"].as_str().unwrap().starts_with("task:")); + assert_eq!(s.jobs().len(), 1); + assert_eq!(s.jobs()[0].kind, "contentFill"); + s.wait_jobs(); + let rec = s.job_log.last().unwrap(); + assert_eq!(rec.status, "done", "{}", rec.message); + assert_eq!(rec.result["frames"], json!(5)); + std::fs::remove_dir_all(&out).unwrap(); +} + +// ---------------------------------------------------------------- Scene Edit Detection + +fn shot_frame(f: u32) -> Image { + // Shots: frames 0–9, 10–17, 18–24. + let (kind, k) = if f < 10 { + (0, f) + } else if f < 18 { + (1, f - 10) + } else { + (2, f - 18) + }; + let mut img = Image::new(80, 48); + for y in 0..48 { + for x in 0..80 { + let fx = x as f32 + 2.0 * k as f32; + let v = match kind { + 0 => 0.5 + 0.4 * (fx * 0.15).sin() * (y as f32 * 0.11).cos(), + 1 => { + if ((fx / 10.0).floor() as i32 + y as i32 / 10) % 2 == 0 { + 0.85 + } else { + 0.15 + } + } + _ => effectcraft_raster::hash_noise((fx.floor() as u32) / 4, y / 4, 5), + }; + let c = match kind { + 0 => [v, v * 0.6, 0.2], + 1 => [0.1, v * 0.8, v], + _ => [v * 0.9, v * 0.9, 0.5 + v * 0.4], + }; + img.set(x, y, [c[0], c[1], c[2], 1.0]); + } + } + img +} + +#[test] +fn scene_edit_detection_finds_cuts() { + let (mut s, cid, clip, _) = setup(80, 48, 1.0, shot_frame); + // Markers. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "markers", "wait": true})).unwrap(); + assert_eq!(r["cuts"], json!([0.4, 0.72])); + let l = s.project.comp(cid).unwrap().layer(clip).unwrap(); + assert_eq!(l.markers.iter().map(|m| m.time.seconds()).collect::>(), vec![0.4, 0.72]); + s.undo(); + // Split Layers. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "split", "wait": true})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect(); + assert_eq!(ids.len(), 3); + let comp = s.project.comp(cid).unwrap(); + let spans: Vec<(f64, f64)> = ids.iter().map(|i| comp.layer(LayerId(*i)).unwrap()).map(|l| (l.in_point.seconds(), l.out_point.seconds())).collect(); + assert_eq!(spans, vec![(0.0, 0.4), (0.4, 0.72), (0.72, 1.0)]); + s.undo(); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 1); + // Split and Precompose: three precomp layers, one undo step. + let r = s.execute("layer.sceneEditDetection", json!({"mode": "splitPrecompose", "wait": true})).unwrap(); + let comp = s.project.comp(cid).unwrap(); + assert_eq!(comp.layers.len(), 3); + for v in r["layers"].as_array().unwrap() { + let l = comp.layer(LayerId(v.as_u64().unwrap())).unwrap(); + assert!(matches!(l.source, LayerSource::Comp { .. })); + } + assert_eq!(s.history.undo.last().unwrap().0, "Scene Edit Detection"); + s.undo(); + assert_eq!(s.project.comp(cid).unwrap().layers.len(), 1); +} + +// ---------------------------------------------------------------- Auto-trace + +fn disc_frame(f: u32) -> Image { + let mut img = Image::new(80, 60); + let (cx, cy) = (30.0 + 2.0 * f as f64, 30.0); + for y in 0..60 { + for x in 0..80 { + let d = ((x as f64 + 0.5 - cx).powi(2) + (y as f64 + 0.5 - cy).powi(2)).sqrt(); + // Anti-aliased disc of radius 15 plus a 12 × 12 square, on transparency. + let a = (15.5 - d).clamp(0.0, 1.0) as f32; + let sq = (60..72).contains(&x) && (8..20).contains(&y); + let a = if sq { 1.0 } else { a }; + img.set(x, y, [a, a, a, a]); + } + } + img +} + +fn iou_with(path_set: &[effectcraft_keyframe::ShapePath], truth: &Image) -> f64 { + let paths: Vec<_> = path_set.iter().map(effectcraft_path::to_kurbo).collect(); + let cov = effectcraft_path::fill_coverage(&paths, &effectcraft_geom::Mat3::IDENTITY, truth.width, truth.height, effectcraft_path::FillRule::EvenOdd); + let (mut i, mut u) = (0.0, 0.0); + for (c, p) in cov.data.iter().zip(&truth.data) { + let (a, b) = (*c >= 0.5, p[3] >= 0.5); + i += (a && b) as u8 as f64; + u += (a || b) as u8 as f64; + } + i / u +} + +fn mask_paths(s: &Session, cid: ItemId, lid: LayerId, lt: Tick) -> Vec { + let l = s.project.comp(cid).unwrap().layer(lid).unwrap(); + l.masks() + .unwrap() + .groups() + .filter(|g| g.get("opacity").unwrap().value_at(lt).as_f64() > 0.0) + .map(|g| match g.get("path").unwrap().value_at(lt) { + effectcraft_project::Value::Path(p) => p, + _ => panic!("path"), + }) + .collect() +} + +#[test] +fn auto_trace_matches_shapes() { + let (mut s, cid, clip, _) = setup(80, 60, 0.4, disc_frame); + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "channel": "alpha", "tolerance": 0.5, "cornerRoundness": 30})).unwrap(); + assert_eq!(r["masks"], json!(2)); + let iou = iou_with(&mask_paths(&s, cid, clip, Tick::ZERO), &disc_frame(0)); + assert!(iou > 0.95, "IoU {iou}"); + s.undo(); + // Work area onto a new layer: one key per frame following the moving disc. + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "timeSpan": "workArea", "applyToNewLayer": true, "minimumArea": 20})).unwrap(); + let nl = LayerId(r["layer"].as_u64().unwrap()); + assert_ne!(nl, clip); + assert_eq!(r["frames"], json!(10)); + for f in [0u32, 5, 9] { + let t = FrameRate::new(FPS as i64, 1).tick_of(f as i64); + let iou = iou_with(&mask_paths(&s, cid, nl, t), &disc_frame(f)); + assert!(iou > 0.93, "frame {f}: IoU {iou}"); + } + // Luminance with Invert traces the background instead (one outline with holes). + s.undo(); + let r = s.execute("layer.autoTrace", json!({"layer": clip.0, "channel": "luminance", "invert": true, "threshold": 50})).unwrap(); + assert!(r["masks"].as_u64().unwrap() >= 1); +} + +// ---------------------------------------------------------------- Align Video to Data + +#[test] +fn align_video_to_data() { + let (mut s, cid, clip, _) = setup(16, 16, 20.0, |_| Image::filled(16, 16, [0.2, 0.2, 0.2, 1.0])); + let add_data = |s: &mut Session, name: &str, ext: &str, text: &str| -> u64 { + s.edit("data", None, |p, _| { + let f = + Footage { path: format!("{name}.{ext}"), kind: FootageKind::Data, codec: ext.to_uppercase(), data: Some(text.into()), ..Default::default() }; + Ok(p.add_item(name, Label::Sandstone, None, ItemKind::Footage(f)).0) + }) + .unwrap() + }; + let gps = add_data(&mut s, "gps", "csv", "lat,lon,timestamp\n1,2,2026-05-01T10:00:00Z\n1,2,2026-05-01T10:00:01Z\n"); + // The video started 5 s after the first sample: it moves 5 s later. + let r = s.execute("layer.alignVideoToData", json!({"data": gps, "videoStart": "2026-05-01T10:00:05Z"})).unwrap(); + assert_eq!(r["startTime"], json!(5.0)); + let l = s.project.comp(cid).unwrap().layer(clip).unwrap(); + assert_eq!((l.start_time.seconds(), l.in_point.seconds()), (5.0, 5.0)); + s.undo(); + // Data placed at 2 s, timestamps as times of day, video start as timecode. + let log = add_data(&mut s, "log", "json", r#"{"samples": [{"t": "09:59:58.0", "v": 1}, {"t": "09:59:59.0", "v": 2}]}"#); + let r = s.execute("layer.alignVideoToData", json!({"data": "log", "videoStart": "10:00:00:00", "dataStart": 2.0})).unwrap(); + assert_eq!(r["key"], json!("t")); + assert_eq!(r["startTime"], json!(4.0)); + // A video that started before the data moves earlier (negative start). + let r = s.execute("layer.alignVideoToData", json!({"data": log, "videoStart": "2026-05-01T09:59:57Z"})).unwrap(); + assert_eq!(r["startTime"], json!(-1.0)); + // Errors: unknown key, no data. + assert!(s.execute("layer.alignVideoToData", json!({"data": gps, "key": "nope", "videoStart": 0})).is_err()); + let _: Value = Value::Null; +} + +#[test] +fn panels_commands_are_wired() { + // The Window menu opens real panels now, and the former stubs are enabled. + let menus = crate::menus::entries(); + for panel in ["lumetriScopes", "footage", "mediaBrowser", "metadata", "progress", "contentAwareFill"] { + assert!(menus.iter().any(|(_, e)| e.command == "window.panel" && e.params["panel"] == panel), "{panel}"); + } + assert!(crate::find_command("window.unavailablePanel").is_none()); + let (s, _, _, _) = setup(8, 8, 1.0, |_| Image::filled(8, 8, [0.0; 4])); + for id in ["layer.autoTrace", "layer.sceneEditDetection", "layer.alignVideoToData", "layer.newContentAwareFill", "contentFill.generate"] { + assert!(s.is_enabled(id), "{id}"); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index e9a641e8..824bd929 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -118,6 +118,9 @@ pub struct ProjectSettings { /// when a GPU adapter exists) or Mercury Software Only (`false`, the CPU compositor). #[serde(default = "yes")] pub gpu_acceleration: bool, + /// The project's comment (Metadata panel). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub comment: String, } impl Default for ProjectSettings { @@ -131,6 +134,7 @@ impl Default for ProjectSettings { frame_start: 0, audio_sample_rate: 48_000, gpu_acceleration: true, + comment: String::new(), } } } diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 91d7c335..298f6652 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -36,10 +36,16 @@ pub enum PanelKind { MaskInterpolation, ScriptConsole, EssentialGraphics, + LumetriScopes, + Footage, + MediaBrowser, + Metadata, + Progress, + ContentAwareFill, } impl PanelKind { - pub const ALL: [PanelKind; 26] = [ + pub const ALL: [PanelKind; 32] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -66,6 +72,12 @@ impl PanelKind { PanelKind::MaskInterpolation, PanelKind::ScriptConsole, PanelKind::EssentialGraphics, + PanelKind::LumetriScopes, + PanelKind::Footage, + PanelKind::MediaBrowser, + PanelKind::Metadata, + PanelKind::Progress, + PanelKind::ContentAwareFill, ]; pub fn title(self) -> &'static str { match self { @@ -95,6 +107,12 @@ impl PanelKind { PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", + PanelKind::LumetriScopes => "Lumetri Scopes", + PanelKind::Footage => "Footage", + PanelKind::MediaBrowser => "Media Browser", + PanelKind::Metadata => "Metadata", + PanelKind::Progress => "Progress", + PanelKind::ContentAwareFill => "Content-Aware Fill", } } pub fn id(self) -> String { @@ -102,7 +120,7 @@ impl PanelKind { } pub fn from_name(s: &str) -> Option { let n = s.to_ascii_lowercase().replace([' ', '_', '-', '&'], ""); - Self::ALL.iter().copied().find(|p| format!("{p:?}").to_ascii_lowercase() == n || p.title().to_ascii_lowercase().replace([' ', '&'], "") == n) + Self::ALL.iter().copied().find(|p| format!("{p:?}").to_ascii_lowercase() == n || p.title().to_ascii_lowercase().replace([' ', '&', '-'], "") == n) } /// Window-menu shortcut. pub fn window_shortcut(self) -> Option<&'static str> { @@ -295,7 +313,7 @@ pub fn workspace(name: &str) -> DockNode { hsplit( FixedA(360.0), tabs(&[EffectControls, Project], 0), - hsplit(FixedB(300.0), tabs(&[Composition, Layer], 0), right(&[Info, Preview], &[EffectsPresets])), + hsplit(FixedB(340.0), tabs(&[Composition, Layer], 0), right(&[LumetriScopes, Info, Preview], &[EffectsPresets])), ), tabs(&[Timeline], 0), ), @@ -315,17 +333,17 @@ pub fn workspace(name: &str) -> DockNode { Ratio(0.55), hsplit( FixedA(300.0), - tabs(&[Project, EffectControls, Flowchart, History], 0), + tabs(&[Project, EffectControls, Flowchart, History, MediaBrowser, Metadata], 0), hsplit( FixedB(300.0), - tabs(&[Composition, Layer], 0), + tabs(&[Composition, Layer, Footage], 0), right( - &[Info, Preview, Audio, Align, Character, Paragraph], - &[EffectsPresets, Properties, Tracker, Wiggler, Smoother, MotionSketch, Paint, Brushes, Markers], + &[Info, Preview, Audio, Align, Character, Paragraph, LumetriScopes], + &[EffectsPresets, Properties, Tracker, Wiggler, Smoother, MotionSketch, Paint, Brushes, Markers, ContentAwareFill], ), ), ), - tabs(&[Timeline, RenderQueue], 0), + tabs(&[Timeline, RenderQueue, Progress], 0), ), "Standard" => vsplit( Ratio(0.56), diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..9df1d6ee 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -787,6 +787,11 @@ impl EffectcraftApp { self.session.poll_mask_track(); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } + // Background tasks (Content-Aware Fill, Scene Edit Detection). + if !self.session.tasks.is_empty() { + self.session.poll_jobs(); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // Analyses running in a worker (the web app). if !self.session.offloaded.is_empty() { self.session.poll_offload(); diff --git a/crates/ui-egui/src/panels/content_fill_panel.rs b/crates/ui-egui/src/panels/content_fill_panel.rs new file mode 100644 index 00000000..5d534e59 --- /dev/null +++ b/crates/ui-egui/src/panels/content_fill_panel.rs @@ -0,0 +1,138 @@ +//! Content-Aware Fill panel (Window ▸ Content-Aware Fill): Fill Target, Alpha Expansion, Fill +//! Method (Object / Surface / Edge Blend), Lighting Correction, Range (Work Area / Entire +//! Duration), a reference frame from a layer of the comp, and Generate Fill Layer. The settings +//! live in `EditorState::content_fill` (`contentFill.set`); the button runs +//! `contentFill.generate` as a background job (Window ▸ Progress). + +use egui::{Align2, Rect, pos2, vec2}; +use serde_json::json; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +const ROW: f32 = 28.0; + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let x0 = rect.min.x + 12.0; + let xv = x0 + 130.0; + let w = (rect.max.x - xv - 12.0).clamp(90.0, 220.0); + let mut y = rect.min.y + 10.0; + let st = app.session.state.content_fill.clone(); + let comp = app.session.active_comp().cloned(); + let layer = comp.as_ref().and_then(|c| app.session.state.selected_layers.first().and_then(|l| c.layer(*l)).cloned()); + // Fill Target. + p.text(pos2(x0, y + 9.0), Align2::LEFT_CENTER, "Fill Target", Tokens::semibold(12.0), t.text); + y += ROW - 6.0; + let target = match &layer { + Some(l) => { + let masks = l + .masks() + .map(|m| { + m.groups() + .filter(|g| { + matches!( + g.kind, + effectcraft_engine::project::GroupKind::Mask { mode: effectcraft_engine::project::MaskMode::Subtract, .. } + | effectcraft_engine::project::GroupKind::Mask { inverted: true, .. } + ) + }) + .count() + }) + .unwrap_or(0); + format!("{} — {masks} subtracting mask(s)", l.name) + } + None => "Select a layer with transparent areas (e.g. a Subtract mask)".into(), + }; + p.text(pos2(x0, y + 9.0), Align2::LEFT_CENTER, &target, Tokens::ui(11.5), if layer.is_some() { t.text_dim } else { t.text_faint }); + app.auto.add("contentFill.target", Rect::from_min_size(pos2(x0, y), vec2(rect.width() - 24.0, 18.0)), &target); + y += ROW; + let mut set = serde_json::Map::new(); + // Alpha Expansion. + kit::label(ui, pos2(x0, y), "Alpha Expansion:", &t); + let a = kit::number(app, ui, pos2(xv, y), "contentFill.alphaExpansion", st.alpha_expansion, 0.2, (0.0, 100.0), 0, " px"); + if (a - st.alpha_expansion).abs() > 1e-9 { + set.insert("alphaExpansion".into(), json!(a.round())); + } + y += ROW; + // Fill Method. + kit::label(ui, pos2(x0, y), "Fill Method:", &t); + let methods = ["object", "surface", "edgeBlend"]; + let cur = methods.iter().position(|m| *m == st.method).unwrap_or(0); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.method", &["Object", "Surface", "Edge Blend"], cur) { + set.insert("method".into(), json!(methods[i])); + } + y += ROW; + // Lighting Correction (Object / Surface). + kit::label(ui, pos2(x0, y), "Lighting Correction:", &t); + let lights = ["none", "subtle", "moderate", "strong"]; + let cur = lights.iter().position(|m| *m == st.lighting_correction).unwrap_or(0); + if st.method != "edgeBlend" + && let Some(i) = + kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.lighting", &["None", "Subtle", "Moderate", "Strong"], cur) + { + set.insert("lightingCorrection".into(), json!(lights[i])); + } + y += ROW; + // Range. + kit::label(ui, pos2(x0, y), "Range:", &t); + let cur = usize::from(st.range == "entire"); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.range", &["Work Area", "Entire Duration"], cur) { + set.insert("range".into(), json!(if i == 1 { "entire" } else { "workArea" })); + } + y += ROW; + // Reference frame: a layer of the comp at a time. + kit::label(ui, pos2(x0, y), "Reference Frame:", &t); + if let Some(c) = &comp { + let mut names = vec!["None".to_string()]; + names.extend(c.layers.iter().map(|l| l.name.clone())); + let cur = st.reference_layer.and_then(|r| c.layers.iter().position(|l| l.id == r)).map(|i| i + 1).unwrap_or(0); + let opts: Vec<&str> = names.iter().map(String::as_str).collect(); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(xv, y), vec2(w, 20.0)), "contentFill.referenceLayer", &opts, cur) { + set.insert("referenceLayer".into(), if i == 0 { json!(null) } else { json!(c.layers[i - 1].id.0) }); + } + y += ROW; + if st.reference_layer.is_some() { + let at = st.reference_time.unwrap_or(0.0); + p.text(pos2(xv, y + 9.0), Align2::LEFT_CENTER, format!("at {}", kit::seconds(at)), Tokens::ui(11.5), t.text_dim); + if kit::button(app, ui, Rect::from_min_size(pos2(xv + 80.0, y), vec2(120.0, 20.0)), "contentFill.referenceHere", "Use Current Time", false) { + set.insert("referenceTime".into(), json!(app.session.time().seconds())); + } + y += ROW; + } + } else { + y += ROW; + } + if !set.is_empty() { + kit::exec(app, "contentFill.set", serde_json::Value::Object(set)); + } + // Generate, or the running job. + y += 6.0; + let running = app.session.jobs().into_iter().find(|j| j.kind == "contentFill"); + match running { + Some(j) => { + let frac = j.fraction.unwrap_or(0.0) as f32; + let bar = Rect::from_min_size(pos2(x0, y + 7.0), vec2((rect.width() - 110.0).max(60.0), 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * frac, 6.0)), 3.0, t.accent); + p.text(pos2(x0, y + 24.0), Align2::LEFT_CENTER, format!("{} {:.0} %", j.message, frac * 100.0), Tokens::ui(11.0), t.text_dim); + if kit::button(app, ui, Rect::from_min_size(pos2(bar.max.x + 10.0, y), vec2(70.0, 20.0)), "contentFill.cancel", "Cancel", false) { + kit::exec(app, "jobs.cancel", json!({"job": j.id})); + } + ui.ctx().request_repaint_after(std::time::Duration::from_millis(150)); + } + None => { + if kit::button(app, ui, Rect::from_min_size(pos2(x0, y), vec2(160.0, 24.0)), "contentFill.generate", "Generate Fill Layer", true) { + match &layer { + Some(l) => { + kit::exec(app, "contentFill.generate", json!({"layer": l.id.0})); + } + None => app.ui.status = "Select the layer to fill first".into(), + } + } + } + } +} diff --git a/crates/ui-egui/src/panels/footage_panel.rs b/crates/ui-egui/src/panels/footage_panel.rs new file mode 100644 index 00000000..cfc69825 --- /dev/null +++ b/crates/ui-egui/src/panels/footage_panel.rs @@ -0,0 +1,150 @@ +//! The Footage panel (double-click footage in the Project panel): the source on its own time +//! ruler, play / pause, In and Out marks, and the Edit Target buttons — Overlay Edit and Ripple +//! Insert Edit — that cut the marked range into the active composition at its current time. +//! Everything runs `footage.*` commands (state in `EditorState::footage_panel`). + +use effectcraft_engine::project::{FootageKind, ItemKind}; +use effectcraft_engine::time::Tick; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.pasteboard); + let Some(view) = app.session.state.footage_panel.clone() else { + p.text(rect.center(), Align2::CENTER_CENTER, "Double-click footage in the Project panel to open it here", Tokens::ui(12.0), t.text_faint); + app.auto.add("footage.empty", rect, "Footage panel (empty)"); + return; + }; + let Some(item) = app.session.project.item(view.item).cloned() else { + app.session.state.footage_panel = None; + return; + }; + let footage = match &item.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }; + let (dur, fr) = match &footage { + Some(f) if f.kind != FootageKind::Still && f.duration > Tick::ZERO => (f.duration, f.frame_rate), + Some(f) => (Tick::from_seconds_f64(10.0), f.frame_rate), + None => (Tick::from_seconds_f64(10.0), effectcraft_engine::time::FrameRate::FPS_30), + }; + // Title strip. + let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); + p.rect_filled(top, 0.0, t.panel_bg); + p.text(pos2(top.min.x + 10.0, top.center().y), Align2::LEFT_CENTER, format!("Footage: {}", item.name), Tokens::semibold(12.0), t.text); + app.auto.add("footage.title", top, &item.name); + // Bottom: ruler and controls. + // Rows, bottom up: Edit Target buttons, transport, ruler. + let edit_row = Rect::from_min_max(pos2(rect.min.x, rect.max.y - 30.0), rect.max); + let bar = Rect::from_min_max(pos2(rect.min.x, edit_row.min.y - 28.0), pos2(rect.max.x, edit_row.min.y)); + let ruler = Rect::from_min_max(pos2(rect.min.x + 10.0, bar.min.y - 26.0), pos2(rect.max.x - 10.0, bar.min.y - 4.0)); + p.rect_filled(Rect::from_min_max(pos2(rect.min.x, ruler.min.y - 4.0), rect.max), 0.0, t.panel_bg); + // Canvas. + let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, ruler.min.y - 4.0)); + if let Some(f) = &footage + && f.has_video + { + let key = (view.item.0, view.time.0, app.session.revision); + let tid = egui::Id::new("footage-panel-tex"); + let cached: Option<((u64, i64, u64), egui::TextureHandle, [u32; 2])> = ctx.data(|d| d.get_temp(tid)); + let tex = match cached { + Some((k, tex, size)) if k == key => Some((tex, size)), + _ => app.session.footage.frame(view.item, f, view.time).map(|img| { + let ci = egui::ColorImage::from_rgba_unmultiplied([img.width as usize, img.height as usize], &img.to_rgba8()); + let tex = ctx.load_texture("footage-panel", ci, egui::TextureOptions::LINEAR); + ctx.data_mut(|d| d.insert_temp(tid, (key, tex.clone(), [img.width, img.height]))); + (tex, [img.width, img.height]) + }), + }; + if let Some((tex, [w, h])) = tex { + let s = ((area.width() - 20.0) / w as f32).min((area.height() - 20.0) / h as f32).max(0.01); + let r = Rect::from_center_size(area.center(), vec2(w as f32 * s, h as f32 * s)); + super::viewer::checker(&p.with_clip_rect(area), r); + p.image(tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } + } else if let ItemKind::Solid(so) = &item.kind { + let c = so.color; + p.rect_filled(area.shrink(20.0), 0.0, Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8)); + } else { + p.text(area.center(), Align2::CENTER_CENTER, "No video", Tokens::ui(12.0), t.text_faint); + } + // Ruler: ticks, In/Out span, CTI; click or drag to scrub. + let x_of = |tt: Tick| ruler.min.x + (tt.seconds() / dur.seconds().max(1e-9)) as f32 * ruler.width(); + p.rect_filled(ruler, 2.0, t.tl_ruler_bg); + let secs = dur.seconds().max(0.001); + let step = [0.04, 0.2, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0].into_iter().find(|s| (secs / s) as f32 * 50.0 < ruler.width()).unwrap_or(600.0); + let mut k = 0.0; + while k <= secs { + let x = ruler.min.x + (k / secs) as f32 * ruler.width(); + p.line_segment([pos2(x, ruler.max.y - 6.0), pos2(x, ruler.max.y)], Stroke::new(1.0, t.tl_ruler_tick)); + p.text(pos2(x + 2.0, ruler.min.y + 1.0), Align2::LEFT_TOP, kit::seconds(k), Tokens::ui(9.0), t.tl_ruler_text); + k += step; + } + let (a, b) = (view.in_point.unwrap_or(Tick::ZERO), view.out_point.unwrap_or(dur)); + p.rect_filled(Rect::from_min_max(pos2(x_of(a), ruler.max.y - 4.0), pos2(x_of(b), ruler.max.y)), 0.0, t.work_area); + let cx = x_of(view.time); + p.line_segment([pos2(cx, ruler.min.y), pos2(cx, ruler.max.y)], Stroke::new(1.5, t.cti)); + let resp = ui.interact(ruler, egui::Id::new("footage-ruler"), Sense::click_and_drag()); + app.auto.add("footage.ruler", ruler, "Time ruler"); + if (resp.clicked() || resp.dragged()) + && let Some(pt) = resp.interact_pointer_pos() + { + let s = ((pt.x - ruler.min.x) / ruler.width()).clamp(0.0, 1.0) as f64 * dur.seconds(); + kit::exec(app, "footage.setTime", json!({"time": s})); + } + // Playback. + let play_id = egui::Id::new("footage-playing"); + let playing: bool = ctx.data(|d| d.get_temp(play_id).unwrap_or(false)); + if playing { + let dt = ctx.input(|i| i.stable_dt).min(0.1) as f64; + let next = view.time.seconds() + dt; + let next = if next >= dur.seconds() - fr.frame_duration().seconds() { 0.0 } else { next }; + let _ = app.session.execute("footage.setTime", json!({"time": next})); + ctx.request_repaint(); + } + // Controls. + let y = bar.min.y + 5.0; + let mut x = bar.min.x + 10.0; + p.text( + pos2(x, bar.center().y), + Align2::LEFT_CENTER, + effectcraft_engine::time::format_timecode_ae(fr.frame_at(view.time), fr, false), + Tokens::mono(12.0), + t.timecode, + ); + x += 92.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(54.0, 20.0)), "footage.play", if playing { "Pause" } else { "Play" }, false) { + ctx.data_mut(|d| d.insert_temp(play_id, !playing)); + } + x += 60.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(28.0, 20.0)), "footage.setIn", "{", false) { + kit::exec(app, "footage.setIn", json!({})); + } + x += 32.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(28.0, 20.0)), "footage.setOut", "}", false) { + kit::exec(app, "footage.setOut", json!({})); + } + x += 36.0; + let span = format!("Δ {}", kit::seconds((b - a).seconds())); + p.text(pos2(x, bar.center().y), Align2::LEFT_CENTER, span, Tokens::ui(11.0), t.text_dim); + let comp_ok = app.session.active_comp_id().is_some(); + let target = app.session.active_comp_id().and_then(|c| app.session.project.item(c)).map(|i| i.name.clone()).unwrap_or_default(); + let ey = edit_row.min.y + 5.0; + let bx = (edit_row.max.x - 238.0).max(edit_row.min.x + 10.0); + if !target.is_empty() && bx > edit_row.min.x + 140.0 { + p.text(pos2(bx - 8.0, edit_row.center().y), Align2::RIGHT_CENTER, format!("Edit Target: {target}"), Tokens::ui(11.0), t.text_dim); + } + if kit::button(app, ui, Rect::from_min_size(pos2(bx, ey), vec2(110.0, 20.0)), "footage.rippleInsertEdit", "Ripple Insert Edit", false) && comp_ok { + kit::exec(app, "footage.rippleInsertEdit", json!({})); + } + if kit::button(app, ui, Rect::from_min_size(pos2(bx + 116.0, ey), vec2(110.0, 20.0)), "footage.overlayEdit", "Overlay Edit", true) && comp_ok { + kit::exec(app, "footage.overlayEdit", json!({})); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 3e051948..8d63d170 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -343,6 +343,60 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ], ) } + "layer.autoTrace" if !has(p, &["channel", "timeSpan", "threshold"]) => ( + "Auto-trace".into(), + vec![ + Field::choice("timeSpan", "Time Span", &[("Current Frame", json!("currentFrame")), ("Work Area", json!("workArea"))], 0), + Field::choice( + "channel", + "Channel", + &[("Alpha", json!("alpha")), ("Red", json!("red")), ("Green", json!("green")), ("Blue", json!("blue")), ("Luminance", json!("luminance"))], + 0, + ), + Field::bool("invert", "Invert", false), + Field::num("blur", "Blur (pixels before auto-trace)", 1.0), + Field::num("tolerance", "Tolerance (pixels)", 1.0), + Field::num("threshold", "Threshold (%)", 50.0), + Field::num("minimumArea", "Minimum Area (pixels)", 10.0), + Field::num("cornerRoundness", "Corner Roundness (%)", 50.0), + Field::bool("applyToNewLayer", "Apply To New Layer", false), + ], + ), + "layer.sceneEditDetection" if !has(p, &["mode"]) => ( + "Scene Edit Detection".into(), + vec![ + Field::choice( + "mode", + "Action", + &[("Create Markers", json!("markers")), ("Split Layers", json!("split")), ("Split and Precompose", json!("splitPrecompose"))], + 0, + ), + Field::num("threshold", "Sensitivity threshold (0–1, lower finds more)", 0.25), + ], + ), + "layer.alignVideoToData" if !has(p, &["data", "videoStart"]) => { + let data: Vec<(String, Value)> = s + .project + .items + .values() + .filter(|i| matches!(&i.kind, effectcraft_engine::project::ItemKind::Footage(f) if f.kind == effectcraft_engine::project::FootageKind::Data)) + .map(|i| (i.name.clone(), json!(i.id.0))) + .collect(); + if data.is_empty() { + info(app, "Align Video to Data", "Import a data file (JSON, CSV or TSV with a time column) first: File ▸ Import."); + return true; + } + let opts: Vec<(&str, Value)> = data.iter().map(|(n, v)| (n.as_str(), v.clone())).collect(); + ( + "Align Video to Data".into(), + vec![ + Field::choice("data", "Data", &opts, 0), + Field::text("key", "Time field (empty: automatic)", ""), + Field::text("videoStart", "Video start (ISO date-time, hh:mm:ss, timecode; empty: file date)", ""), + Field::num("dataStart", "First sample at (comp seconds)", 0.0), + ], + ) + } "file.importPlaceholder" | "file.replaceWithPlaceholder" if p.as_object().is_none_or(|m| m.is_empty()) => ( "New Placeholder".into(), vec![ diff --git a/crates/ui-egui/src/panels/media_panels.rs b/crates/ui-egui/src/panels/media_panels.rs new file mode 100644 index 00000000..b1a3230e --- /dev/null +++ b/crates/ui-egui/src/panels/media_panels.rs @@ -0,0 +1,366 @@ +//! Media Browser, Metadata and Progress panels (Window menu). +//! +//! - **Media Browser**: favourites and the folder tree on the left, the folder's files on the +//! right with thumbnails (images and video, decoded through the media layer's probe), double- +//! click to open a folder or import a file, drag files into the Project panel or the timeline. +//! Desktop only (the web build has no file system to browse). +//! - **Metadata**: what we read from the selected item's file (codec, size, rate, duration, +//! colour profile, dates) and the project's, with editable item and project comments. +//! - **Progress**: every background job (render queue, track / mask track / Warp Stabilizer / +//! 3D Camera Tracker / Roto Brush analyses, Content-Aware Fill, Scene Edit Detection) with a +//! progress bar and a cancel button, and the recently finished ones. + +use effectcraft_engine::media_browser as mb; +use effectcraft_engine::project::ItemId; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::{Value, json}; + +use super::DragPayload; +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +// ---------------------------------------------------------------- Media Browser + +type Thumb = Option<(egui::TextureHandle, [u32; 2])>; + +/// Thumbnail of a file (cached per path; `None` while not decoded or not decodable). +fn thumbnail(app: &EffectcraftApp, ctx: &egui::Context, path: &str, budget: &mut u32) -> Thumb { + let id = egui::Id::new(("mb-thumb", path)); + if let Some(t) = ctx.data(|d| d.get_temp::(id)) { + return t; + } + if *budget == 0 { + return None; + } + *budget -= 1; + let thumb = (|| { + let imp = app.session.importer.clone()?; + let f = imp.probe(path).ok()?; + if !f.has_video { + return None; + } + // A synthetic item id keyed by the path (the frame cache keys on the path). + let hid = path.bytes().fold(0xcbf2_9ce4_8422_2325u64, |h, b| (h ^ b as u64).wrapping_mul(0x100_0000_01b3)) | (1 << 62); + let img = app.session.footage.frame(ItemId(hid), &f, effectcraft_engine::time::Tick::ZERO)?; + let step = (img.width.max(img.height) as f32 / 128.0).ceil().max(1.0) as u32; + let (w, h) = (img.width.div_ceil(step), img.height.div_ceil(step)); + let full = img.to_rgba8(); + let mut px = Vec::with_capacity((w * h * 4) as usize); + for y in 0..h { + for x in 0..w { + let i = (((y * step) * img.width + x * step) * 4) as usize; + px.extend_from_slice(&full[i..i + 4]); + } + } + let tex = ctx.load_texture(format!("mb-{path}"), egui::ColorImage::from_rgba_unmultiplied([w as usize, h as usize], &px), egui::TextureOptions::LINEAR); + Some((tex, [w, h])) + })(); + ctx.data_mut(|d| d.insert_temp(id, thumb.clone())); + thumb +} + +pub fn media_browser(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + if !mb::available() { + p.text(rect.center(), Align2::CENTER_CENTER, "The Media Browser needs the desktop app (use File ▸ Import here)", Tokens::ui(12.0), t.text_faint); + app.auto.add("mediaBrowser.unavailable", rect, "Media Browser unavailable"); + return; + } + let st = app.session.state.media_browser.clone(); + let dir = st.folder.clone().unwrap_or_else(mb::home); + // Listing cached per folder for a second. + let lid = egui::Id::new(("mb-list", dir.clone(), st.importable_only)); + let now = ctx.input(|i| i.time); + let listing: Option<(f64, Result, String>)> = ctx.data(|d| d.get_temp(lid)); + let entries = match listing { + Some((at, l)) if now - at < 1.0 => l, + _ => { + let l = mb::list(&dir, st.importable_only); + ctx.data_mut(|d| d.insert_temp(lid, (now, l.clone()))); + l + } + }; + // Toolbar: up, home, path, favourite, importable only. + let y = rect.min.y + 6.0; + let mut x = rect.min.x + 8.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(30.0, 20.0)), "mediaBrowser.up", "↑", false) { + kit::exec(app, "mediaBrowser.go", json!({"path": ".."})); + } + x += 34.0; + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(48.0, 20.0)), "mediaBrowser.home", "Home", false) { + kit::exec(app, "mediaBrowser.go", json!({})); + } + x += 54.0; + let fav = st.favorites.contains(&dir); + if kit::button(app, ui, Rect::from_min_size(pos2(x, y), vec2(24.0, 20.0)), "mediaBrowser.favorite", if fav { "★" } else { "☆" }, false) { + kit::exec(app, if fav { "mediaBrowser.removeFavorite" } else { "mediaBrowser.addFavorite" }, json!({"path": dir})); + } + x += 30.0; + let only = kit::checkbox(app, ui, pos2(x, y + 2.0), "mediaBrowser.importableOnly", "Media only", st.importable_only); + if only != st.importable_only { + kit::exec(app, "mediaBrowser.go", json!({"path": dir, "importableOnly": only})); + } + x += 100.0; + p.text(pos2(x, y + 10.0), Align2::LEFT_CENTER, &dir, Tokens::ui(11.5), t.text_dim); + app.auto.add("mediaBrowser.path", Rect::from_min_max(pos2(x, y), pos2(rect.max.x - 8.0, y + 20.0)), &dir); + // Left column: favourites. + let side = Rect::from_min_max(pos2(rect.min.x, rect.min.y + 32.0), pos2(rect.min.x + 160.0_f32.min(rect.width() * 0.3), rect.max.y)); + p.line_segment([side.right_top(), side.right_bottom()], Stroke::new(1.0, t.separator)); + p.text(pos2(side.min.x + 10.0, side.min.y + 10.0), Align2::LEFT_CENTER, "Favorites", Tokens::semibold(11.5), t.text_dim); + let mut fy = side.min.y + 22.0; + let mut places: Vec<(String, String)> = vec![("Home".into(), mb::home())]; + places.extend( + st.favorites.iter().map(|f| (std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()), f.clone())), + ); + for (i, (name, path)) in places.iter().enumerate() { + let r = Rect::from_min_size(pos2(side.min.x, fy), vec2(side.width(), 20.0)); + let resp = ui.interact(r, egui::Id::new(("mb-fav", i)), Sense::click()); + if *path == dir { + p.rect_filled(r, 0.0, t.row_selected); + } else if resp.hovered() { + p.rect_filled(r, 0.0, t.hover); + } + p.text(pos2(r.min.x + 14.0, r.center().y), Align2::LEFT_CENTER, name, Tokens::ui(12.0), t.text); + app.auto.add(&format!("mediaBrowser.place.{i}"), r, name); + if resp.clicked() { + kit::exec(app, "mediaBrowser.go", json!({"path": path})); + } + fy += 20.0; + } + // Files. + let list = Rect::from_min_max(pos2(side.max.x + 1.0, side.min.y), rect.max); + let entries = match entries { + Ok(e) => e, + Err(e) => { + p.text(list.center(), Align2::CENTER_CENTER, e, Tokens::ui(12.0), t.danger); + return; + } + }; + let tile = vec2(140.0, 112.0); + let cols = ((list.width() - 8.0) / tile.x).floor().max(1.0) as usize; + let mut budget = 2u32; + let scroll_id = egui::Id::new("mb-scroll"); + let mut scroll: f32 = ctx.data(|d| d.get_temp(scroll_id).unwrap_or(0.0)); + if ui.rect_contains_pointer(list) { + scroll -= ctx.input(|i| i.smooth_scroll_delta.y); + } + let rows = entries.len().div_ceil(cols); + scroll = scroll.clamp(0.0, (rows as f32 * tile.y - list.height() + 8.0).max(0.0)); + ctx.data_mut(|d| d.insert_temp(scroll_id, scroll)); + let lp = p.with_clip_rect(list); + let mut actions: Vec<(&str, Value)> = vec![]; + for (i, e) in entries.iter().enumerate() { + let r = Rect::from_min_size( + pos2(list.min.x + 8.0 + (i % cols) as f32 * tile.x, list.min.y + 8.0 + (i / cols) as f32 * tile.y - scroll), + tile - vec2(8.0, 8.0), + ); + if !r.intersects(list) { + continue; + } + let resp = ui.interact(r.intersect(list), egui::Id::new(("mb-entry", &e.path)), Sense::click_and_drag()); + app.auto.add(&format!("mediaBrowser.entry.{}", e.name), r, &e.name); + lp.rect_filled(r, 4.0, if resp.hovered() { t.hover } else { t.row }); + let thumb = Rect::from_min_max(r.min + vec2(6.0, 6.0), pos2(r.max.x - 6.0, r.max.y - 26.0)); + lp.rect_filled(thumb, 3.0, Color32::from_rgb(0x18, 0x18, 0x18)); + let kind = if e.is_dir { "Folder" } else { e.kind.unwrap_or("File") }; + match (e.is_dir, e.kind) { + (false, Some("image" | "video")) => match thumbnail(app, &ctx, &e.path, &mut budget) { + Some((tex, [w, h])) => { + let s = (thumb.width() / w as f32).min(thumb.height() / h as f32); + lp.image( + tex.id(), + Rect::from_center_size(thumb.center(), vec2(w as f32 * s, h as f32 * s)), + Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), + Color32::WHITE, + ); + } + None => { + lp.text(thumb.center(), Align2::CENTER_CENTER, kind, Tokens::ui(11.0), t.text_faint); + } + }, + _ => { + lp.text(thumb.center(), Align2::CENTER_CENTER, if e.is_dir { "▣" } else { kind }, Tokens::ui(if e.is_dir { 28.0 } else { 11.0 }), t.text_faint); + } + } + let name = if e.name.chars().count() > 20 { format!("{}…", e.name.chars().take(19).collect::()) } else { e.name.clone() }; + lp.text(pos2(r.min.x + 8.0, r.max.y - 12.0), Align2::LEFT_CENTER, name, Tokens::ui(11.5), t.text); + if resp.double_clicked() { + if e.is_dir { + actions.push(("mediaBrowser.go", json!({"path": e.path}))); + } else if e.kind.is_some() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path]}))); + } + } + if resp.drag_started() && !e.is_dir && e.kind.is_some() { + egui::DragAndDrop::set_payload(&ctx, DragPayload::Files(vec![e.path.clone()])); + } + resp.context_menu(|ui| { + if !e.is_dir && ui.button("Import").clicked() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path]}))); + ui.close(); + } + if !e.is_dir && ui.button("Import and Add to Composition").clicked() { + actions.push(("mediaBrowser.import", json!({"paths": [e.path], "addToComp": true}))); + ui.close(); + } + if e.is_dir && ui.button("Add to Favorites").clicked() { + actions.push(("mediaBrowser.addFavorite", json!({"path": e.path}))); + ui.close(); + } + }); + } + if entries.is_empty() { + lp.text(list.center(), Align2::CENTER_CENTER, "Empty folder", Tokens::ui(12.0), t.text_faint); + } + if budget == 0 { + ctx.request_repaint(); + } + for (id, p) in actions { + kit::exec(app, id, p); + } +} + +// ---------------------------------------------------------------- Metadata + +fn rows_of(v: &Value) -> Vec<(String, String)> { + let Some(o) = v.as_object() else { return vec![] }; + o.iter() + .filter(|(k, v)| !v.is_null() && !matches!(k.as_str(), "comment" | "id")) + .map(|(k, v)| { + let s = match v { + Value::String(s) => s.clone(), + Value::Number(n) => n.as_f64().map(|f| if f.fract() == 0.0 { format!("{f:.0}") } else { format!("{f:.3}") }).unwrap_or_default(), + other => other.to_string(), + }; + (k.clone(), s) + }) + .collect() +} + +/// An editable comment field; returns the edited text when it changes. +fn comment_field(ui: &mut egui::Ui, r: Rect, id: &str, text: &str) -> Option { + let key = egui::Id::new(("meta-comment", id)); + let mut buf: String = ui.data(|d| d.get_temp(key)).unwrap_or_else(|| text.to_string()); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(r)); + let resp = child.add(egui::TextEdit::multiline(&mut buf).desired_width(r.width()).desired_rows(2).hint_text("Add a comment")); + if resp.has_focus() || resp.changed() { + ui.data_mut(|d| d.insert_temp(key, buf.clone())); + } else { + ui.data_mut(|d| d.remove::(key)); + } + (resp.lost_focus() && buf != text).then_some(buf) +} + +pub fn metadata(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let Ok(v) = app.session.execute("item.metadata", json!({})) else { return }; + let mut y = rect.min.y + 8.0; + let x0 = rect.min.x + 12.0; + let xv = x0 + 120.0; + let section = |p: &egui::Painter, y: &mut f32, title: &str| { + p.text(pos2(x0, *y + 8.0), Align2::LEFT_CENTER, title, Tokens::semibold(12.0), t.text); + *y += 22.0; + }; + if let Some(item) = v.get("item").filter(|i| !i.is_null()) { + let name = item["name"].as_str().unwrap_or_default().to_string(); + let id = item["id"].as_u64().unwrap_or(0); + section(&p, &mut y, &format!("{name} ({})", item["type"].as_str().unwrap_or(""))); + kit::label(ui, pos2(x0, y), "Comment", &t); + let r = Rect::from_min_size(pos2(xv, y), vec2((rect.max.x - xv - 12.0).max(80.0), 40.0)); + app.auto.add("metadata.itemComment", r, item["comment"].as_str().unwrap_or("")); + if let Some(c) = comment_field(ui, r, &format!("item{id}"), item["comment"].as_str().unwrap_or("")) { + kit::exec(app, "project.setComment", json!({"items": [id], "comment": c})); + } + y += 48.0; + for (k, val) in rows_of(item) { + if matches!(k.as_str(), "name" | "type") { + continue; + } + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &k, Tokens::ui(11.5), t.text_dim); + p.text(pos2(xv, y + 8.0), Align2::LEFT_CENTER, &val, Tokens::ui(11.5), t.text); + app.auto.add(&format!("metadata.item.{k}"), Rect::from_min_size(pos2(xv, y), vec2(200.0, 16.0)), &val); + y += 18.0; + } + y += 10.0; + } else { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Select an item in the Project panel", Tokens::ui(12.0), t.text_faint); + y += 28.0; + } + let proj = v["project"].clone(); + section(&p, &mut y, "Project"); + kit::label(ui, pos2(x0, y), "Comment", &t); + let r = Rect::from_min_size(pos2(xv, y), vec2((rect.max.x - xv - 12.0).max(80.0), 40.0)); + app.auto.add("metadata.projectComment", r, proj["comment"].as_str().unwrap_or("")); + if let Some(c) = comment_field(ui, r, "project", proj["comment"].as_str().unwrap_or("")) { + kit::exec(app, "project.setProjectComment", json!({"comment": c})); + } + y += 48.0; + for (k, val) in rows_of(&proj) { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &k, Tokens::ui(11.5), t.text_dim); + p.text(pos2(xv, y + 8.0), Align2::LEFT_CENTER, &val, Tokens::ui(11.5), t.text); + y += 18.0; + } +} + +// ---------------------------------------------------------------- Progress + +pub fn progress(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + if !app.session.tasks.is_empty() { + app.session.poll_jobs(); + } + let jobs = app.session.jobs(); + let mut y = rect.min.y + 8.0; + let x0 = rect.min.x + 12.0; + let w = (rect.width() - 24.0).max(60.0); + if jobs.is_empty() { + p.text(pos2(x0, y + 10.0), Align2::LEFT_CENTER, "No background jobs running", Tokens::ui(12.0), t.text_faint); + y += 28.0; + } else { + ui.ctx().request_repaint_after(std::time::Duration::from_millis(150)); + } + for j in &jobs { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &j.label, Tokens::medium(12.0), t.text); + let frac = j.fraction.unwrap_or(0.0) as f32; + let pct = j.fraction.map(|f| format!("{:.0} %", f * 100.0)).unwrap_or_else(|| "…".into()); + p.text(pos2(x0 + w - 70.0, y + 8.0), Align2::RIGHT_CENTER, format!("{pct} {}", j.message), Tokens::ui(11.0), t.text_dim); + let bar = Rect::from_min_size(pos2(x0, y + 20.0), vec2(w - 76.0, 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * frac.clamp(0.0, 1.0), bar.height())), 3.0, t.accent); + app.auto.add(&format!("progress.job.{}", j.id), Rect::from_min_size(pos2(x0, y), vec2(w, 28.0)), &j.label); + if j.cancellable + && kit::button(app, ui, Rect::from_min_size(pos2(x0 + w - 64.0, y + 12.0), vec2(64.0, 18.0)), &format!("progress.cancel.{}", j.id), "Cancel", false) + { + kit::exec(app, "jobs.cancel", json!({"job": j.id})); + } + y += 38.0; + } + if !app.session.job_log.is_empty() { + p.line_segment([pos2(x0, y), pos2(x0 + w, y)], Stroke::new(1.0, t.separator)); + y += 6.0; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Finished", Tokens::semibold(11.5), t.text_dim); + y += 20.0; + for r in app.session.job_log.iter().rev().take(20) { + let col = match r.status.as_str() { + "done" => t.text, + "cancelled" => t.text_dim, + _ => t.danger, + }; + let msg = if r.message.is_empty() { format!("{} · {:.1} s", r.status, r.seconds) } else { format!("{} · {}", r.status, r.message) }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, &r.label, Tokens::ui(11.5), col); + p.text(pos2(x0 + w, y + 8.0), Align2::RIGHT_CENTER, msg, Tokens::ui(11.0), t.text_faint); + y += 18.0; + if y > rect.max.y { + break; + } + } + } +} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 2e6d03ae..31daf811 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -3,6 +3,7 @@ pub mod anim_tools; pub mod camera_tracker_ui; pub mod comp_settings; +pub mod content_fill_panel; pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; @@ -10,6 +11,7 @@ pub mod effects_presets; pub mod essential; pub mod expr_bar; pub mod flowchart; +pub mod footage_panel; pub mod forms; pub mod fx_widgets; pub mod graph; @@ -20,14 +22,17 @@ pub mod key_dialogs; pub mod layer_panel; pub mod layer_styles_dialog; pub mod markers_ui; +pub mod media_panels; pub mod misc; pub mod paint_panels; +pub mod panel_kit; pub mod precomp; pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; +pub mod scopes_panel; pub mod script_console; pub mod settings; pub mod shortcut_editor; @@ -57,6 +62,8 @@ pub enum DragPayload { Effect(String), /// A property dragged from the timeline (to the Essential Graphics panel). Property { layer: u64, prop: u64 }, + /// Files from the Media Browser (import; into the timeline: import and add). + Files(Vec), } /// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). @@ -103,6 +110,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), PanelKind::Flowchart => flowchart::show(app, ui, rect), + PanelKind::LumetriScopes => scopes_panel::show(app, ui, rect), + PanelKind::Footage => footage_panel::show(app, ui, rect), + PanelKind::MediaBrowser => media_panels::media_browser(app, ui, rect), + PanelKind::Metadata => media_panels::metadata(app, ui, rect), + PanelKind::Progress => media_panels::progress(app, ui, rect), + PanelKind::ContentAwareFill => content_fill_panel::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/panel_kit.rs b/crates/ui-egui/src/panels/panel_kit.rs new file mode 100644 index 00000000..cbf28c8a --- /dev/null +++ b/crates/ui-egui/src/panels/panel_kit.rs @@ -0,0 +1,70 @@ +//! Small immediate-mode controls shared by the Lumetri Scopes, Footage, Media Browser, Metadata, +//! Progress and Content-Aware Fill panels. Each registers its automation id. + +use egui::{Align2, Rect, pos2, vec2}; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// A dropdown; returns the newly chosen index. +pub fn dropdown(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, auto: &str, options: &[&str], cur: usize) -> Option { + let t = app.tokens; + let id = egui::Id::new(("kit-dd", auto)); + let text = options.get(cur).copied().unwrap_or(""); + if widgets::dropdown(ui, r, text, &t, id).clicked() { + widgets::open_popup(ui, id); + } + app.auto.add(auto, r, text); + let opts: Vec = options.iter().map(|s| s.to_string()).collect(); + widgets::popup_menu(ui, id, r.left_bottom(), &opts, Some(cur)).filter(|i| *i != cur) +} + +/// A checkbox with a label; returns the new state. +pub fn checkbox(app: &mut EffectcraftApp, ui: &mut egui::Ui, at: egui::Pos2, auto: &str, text: &str, on: bool) -> bool { + let t = app.tokens; + let r = Rect::from_min_size(at, vec2(16.0, 16.0)); + let clicked = widgets::checkbox(ui, r, on, &t, egui::Id::new(("kit-cb", auto))).clicked(); + ui.painter().text(pos2(r.max.x + 6.0, r.center().y), Align2::LEFT_CENTER, text, Tokens::ui(12.0), t.text); + let full = Rect::from_min_size(at, vec2(22.0 + text.len() as f32 * 6.5, 16.0)); + app.auto.add(auto, full, text); + if clicked { !on } else { on } +} + +/// A text button; true when clicked. +pub fn button(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, auto: &str, text: &str, primary: bool) -> bool { + let t = app.tokens; + let clicked = widgets::text_button(ui, r, text, primary, &t, egui::Id::new(("kit-btn", auto))).clicked(); + app.auto.add(auto, r, text); + clicked +} + +/// A dimmed label at (x, y-row). +pub fn label(ui: &egui::Ui, at: egui::Pos2, text: &str, t: &Tokens) { + ui.painter().text(pos2(at.x, at.y + 9.0), Align2::LEFT_CENTER, text, Tokens::ui(12.0), t.text_dim); +} + +/// A draggable number; returns the new value. +#[allow(clippy::too_many_arguments)] +pub fn number(app: &mut EffectcraftApp, ui: &mut egui::Ui, at: egui::Pos2, auto: &str, v: f64, speed: f64, range: (f64, f64), dec: usize, suffix: &str) -> f64 { + let t = app.tokens; + let (r, nv, _) = widgets::hot_number_at(ui, at, egui::Id::new(("kit-n", auto)), v, speed, range, dec, suffix, &t); + app.auto.add(auto, r, &format!("{v}")); + nv.unwrap_or(v) +} + +/// Run a command, reporting errors in the status bar. +pub fn exec(app: &mut EffectcraftApp, id: &str, p: serde_json::Value) -> Option { + match app.session.execute(id, p) { + Ok(v) => Some(v), + Err(e) => { + app.ui.status = e.to_string(); + None + } + } +} + +/// Format seconds as `m:ss.ff`. +pub fn seconds(s: f64) -> String { + let m = (s / 60.0).floor(); + format!("{}:{:05.2}", m as i64, s - m * 60.0) +} diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index ce750fac..f3898954 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -481,6 +481,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ItemKind::Folder if !app.ui.project_open_folders.remove(&id.0) => { app.ui.project_open_folders.insert(id.0); } + // Footage (not data) and solids open in the Footage panel. + ItemKind::Footage(f) if f.kind != effectcraft_engine::project::FootageKind::Data => { + actions.push(("footage.open".into(), json!({"item": id.0}))) + } + ItemKind::Solid(_) => actions.push(("footage.open".into(), json!({"item": id.0}))), _ => {} } } @@ -539,6 +544,16 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { egui::DragAndDrop::clear_payload(&ctx); } } + // Files dragged from the Media Browser: import. + if let Some(DragPayload::Files(paths)) = egui::DragAndDrop::payload::(&ctx).as_deref() + && hover_y.is_some() + { + lp.rect_stroke(list.shrink(1.0), 0.0, Stroke::new(1.0, t.accent.gamma_multiply(0.6)), egui::StrokeKind::Inside); + if ctx.input(|i| i.pointer.any_released()) { + actions.push(("mediaBrowser.import".into(), json!({"paths": paths}))); + egui::DragAndDrop::clear_payload(&ctx); + } + } // Drag ghost. if let Some(DragPayload::Item(iid)) = egui::DragAndDrop::payload::(&ctx).as_deref() && let Some(pos) = ctx.pointer_hover_pos() diff --git a/crates/ui-egui/src/panels/scopes_panel.rs b/crates/ui-egui/src/panels/scopes_panel.rs new file mode 100644 index 00000000..d220213b --- /dev/null +++ b/crates/ui-egui/src/panels/scopes_panel.rs @@ -0,0 +1,197 @@ +//! Lumetri Scopes panel (Window ▸ Lumetri Scopes): waveform (RGB / Luma / YC), vectorscope +//! (YUV / HLS), histogram and parade (RGB / YUV) of the active composition's frame at the current +//! time, with the colour standard (Rec. 601 / 709 / 2020), 8-bit or float scale and clamp. The +//! maths is `effectcraft_raster::scopes`; agents read the same numbers with `scopes.analyze`. + +use effectcraft_engine::commands::panels_cmds::scope_frame; +use effectcraft_engine::raster::scopes::{self, ColorStandard, Scope, ScopeKind, ScopeOpts}; +use egui::{Align2, Color32, Rect, Stroke, pos2, vec2}; +use serde::{Deserialize, Serialize}; + +use super::panel_kit as kit; +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// Panel options (serde in the UI state). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ScopesState { + pub scope: String, + pub standard: String, + pub float: bool, + pub clamp: bool, +} + +impl Default for ScopesState { + fn default() -> Self { + ScopesState { scope: "waveformRgb".into(), standard: "rec709".into(), float: false, clamp: true } + } +} + +const STANDARDS: [&str; 3] = ["Rec. 601", "Rec. 709", "Rec. 2020"]; + +#[derive(Clone)] +struct Cached { + key: (u64, u64, i64, String, u32), + scope: Scope, + tex: Option, +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, t.panel_bg); + let mut st = app.ui.scopes.clone(); + let kind = ScopeKind::from_name(&st.scope).unwrap_or(ScopeKind::WaveformRgb); + let standard = ColorStandard::from_name(&st.standard).unwrap_or_default(); + // Top bar: scope, standard, 8-bit / float, clamp. + let mut x = rect.min.x + 8.0; + let y = rect.min.y + 6.0; + let labels: Vec<&str> = ScopeKind::ALL.iter().map(|k| k.label()).collect(); + let cur = ScopeKind::ALL.iter().position(|k| *k == kind).unwrap_or(0); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(150.0, 20.0)), "scopes.kind", &labels, cur) { + st.scope = ScopeKind::ALL[i].name().into(); + } + x += 158.0; + let si = ColorStandard::ALL.iter().position(|c| *c == standard).unwrap_or(1); + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(90.0, 20.0)), "scopes.standard", &STANDARDS, si) { + st.standard = ColorStandard::ALL[i].name().into(); + } + x += 98.0; + if let Some(i) = kit::dropdown(app, ui, Rect::from_min_size(pos2(x, y), vec2(70.0, 20.0)), "scopes.scale", &["8 Bit", "Float"], usize::from(st.float)) { + st.float = i == 1; + } + x += 78.0; + st.clamp = kit::checkbox(app, ui, pos2(x, y + 2.0), "scopes.clamp", "Clamp Signal", st.clamp); + if st != app.ui.scopes { + app.ui.scopes = st.clone(); + } + let area = Rect::from_min_max(pos2(rect.min.x + 8.0, rect.min.y + 34.0), pos2(rect.max.x - 8.0, rect.max.y - 8.0)); + if area.width() < 40.0 || area.height() < 40.0 { + return; + } + p.rect_filled(area, 2.0, Color32::from_rgb(0x10, 0x10, 0x10)); + app.auto.add("scopes.plot", area, kind.label()); + let Some(cid) = app.session.active_comp_id() else { + p.text(area.center(), Align2::CENTER_CENTER, "Open a composition to see its scopes", Tokens::ui(12.0), t.text_faint); + return; + }; + // Plot rectangle: square for vectorscopes. + let plot = if kind.is_vectorscope() { + let s = area.width().min(area.height()) - 16.0; + Rect::from_center_size(area.center(), vec2(s, s)) + } else { + Rect::from_min_max(area.min + vec2(34.0, 8.0), area.max - vec2(8.0, 8.0)) + }; + let size = (plot.width().max(plot.height()) as u32).clamp(64, 512); + let o = ScopeOpts { + standard, + float: st.float, + clamp: st.clamp, + width: if kind.is_vectorscope() { size } else { (plot.width() as u32).clamp(64, 768) }, + height: if kind.is_vectorscope() { size } else { (plot.height() as u32).clamp(32, 512) }, + }; + let time = app.session.time(); + let key = (app.session.revision, cid.0, time.0, format!("{}{}{}{}{}x{}", st.scope, st.standard, st.float, st.clamp, o.width, o.height), 0); + let ctx = ui.ctx().clone(); + let id = egui::Id::new("scopes-cache"); + let cached: Option = ctx.data(|d| d.get_temp(id)); + let cached = match cached { + Some(c) if c.key == key => c, + _ => { + let Some(frame) = scope_frame(&app.session, cid, time, 480) else { return }; + let scope = scopes::compute(&frame, kind, &o); + let tex = (kind != ScopeKind::Histogram).then(|| { + let mono = matches!(kind, ScopeKind::WaveformLuma | ScopeKind::VectorscopeYuv | ScopeKind::VectorscopeHls); + let img = egui::ColorImage::from_rgba_unmultiplied([scope.width as usize, scope.height as usize], &tint(&scope, mono)); + ctx.load_texture("lumetri-scope", img, egui::TextureOptions::LINEAR) + }); + let c = Cached { key, scope, tex }; + ctx.data_mut(|d| d.insert_temp(id, c.clone())); + c + } + }; + graticule(&p, plot, kind, &o, &t); + if let Some(tex) = &cached.tex { + p.image(tex.id(), plot, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } else { + histogram(&p, plot, &cached.scope); + } +} + +/// Trace colours: green-tinted mono for luma/vectorscope, per-channel colours otherwise. +fn tint(s: &Scope, mono: bool) -> Vec { + let n = s.width as usize * s.height as usize; + let mut out = vec![0u8; n * 4]; + for i in 0..n { + let (r, g, b) = (s.planes[0][i], s.planes[1][i], s.planes[2][i]); + let (r, g, b) = if mono { + let m = r.max(g).max(b); + (m * 0.75, m, m * 0.75) + } else { + (r, g, b) + }; + let a = r.max(g).max(b).min(1.0); + out[i * 4] = (r.min(1.0) * 255.0) as u8; + out[i * 4 + 1] = (g.min(1.0) * 255.0) as u8; + out[i * 4 + 2] = (b.min(1.0) * 255.0) as u8; + out[i * 4 + 3] = (a * 255.0) as u8; + } + out +} + +fn graticule(p: &egui::Painter, plot: Rect, kind: ScopeKind, o: &ScopeOpts, t: &Tokens) { + let line = Stroke::new(1.0, Color32::from_rgba_unmultiplied(255, 255, 255, 28)); + if kind.is_vectorscope() { + let c = plot.center(); + for r in [0.25f32, 0.5, 0.75, 1.0] { + p.circle_stroke(c, plot.width() / 2.0 * r, line); + } + p.line_segment([pos2(plot.min.x, c.y), pos2(plot.max.x, c.y)], line); + p.line_segment([pos2(c.x, plot.min.y), pos2(c.x, plot.max.y)], line); + for (name, [u, v]) in scopes::vector_targets(o.standard, kind == ScopeKind::VectorscopeHls) { + let q = pos2(plot.min.x + (u + 0.5) * plot.width(), plot.min.y + (0.5 - v) * plot.height()); + p.rect_stroke(Rect::from_center_size(q, vec2(8.0, 8.0)), 0.0, Stroke::new(1.0, Color32::from_rgb(0xc8, 0x6a, 0x6a)), egui::StrokeKind::Middle); + p.text(q + vec2(7.0, -7.0), Align2::LEFT_BOTTOM, name, Tokens::ui(10.0), t.text_dim); + } + return; + } + if kind == ScopeKind::Histogram { + return; + } + let (lo, hi) = o.range(); + let marks: &[f32] = if o.float { &[-0.25, 0.0, 0.25, 0.5, 0.75, 1.0, 1.25] } else { &[0.0, 0.25, 0.5, 0.75, 1.0] }; + for v in marks { + let y = plot.max.y - (v - lo) / (hi - lo) * plot.height(); + p.line_segment([pos2(plot.min.x, y), pos2(plot.max.x, y)], line); + let label = if o.float { format!("{v:.2}") } else { format!("{}", (v * 255.0).round() as i32) }; + p.text(pos2(plot.min.x - 4.0, y), Align2::RIGHT_CENTER, label, Tokens::ui(9.5), t.text_faint); + } + if matches!(kind, ScopeKind::ParadeRgb | ScopeKind::ParadeYuv) { + for k in 1..3 { + let x = plot.min.x + plot.width() * k as f32 / 3.0; + p.line_segment([pos2(x, plot.min.y), pos2(x, plot.max.y)], line); + } + } +} + +fn histogram(p: &egui::Painter, plot: Rect, s: &Scope) { + let bars = scopes::histogram_bars(s); + let max = bars.iter().flat_map(|b| b.iter()).copied().max().unwrap_or(1).max(1) as f32; + let cols = [ + Color32::from_rgba_unmultiplied(230, 70, 70, 150), + Color32::from_rgba_unmultiplied(70, 220, 90, 150), + Color32::from_rgba_unmultiplied(80, 120, 255, 150), + ]; + let n = bars[0].len().max(1) as f32; + for (c, b) in bars.iter().enumerate() { + let pts: Vec = + b.iter().enumerate().map(|(i, v)| pos2(plot.min.x + i as f32 / n * plot.width(), plot.max.y - (*v as f32 / max).sqrt() * plot.height())).collect(); + for (i, q) in pts.iter().enumerate() { + p.line_segment([pos2(q.x, plot.max.y), *q], Stroke::new((plot.width() / n).max(1.0), cols[c].gamma_multiply(0.5))); + if i > 0 { + p.line_segment([pts[i - 1], *q], Stroke::new(1.0, cols[c])); + } + } + } +} diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 10ad62e6..72f39df8 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -1986,6 +1986,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effect.apply".into(), json!({"effect": e, "layers": [l.0]}))); } } + crate::panels::DragPayload::Files(paths) => actions.push(("mediaBrowser.import".into(), json!({"paths": paths, "addToComp": true}))), crate::panels::DragPayload::Property { .. } => {} } egui::DragAndDrop::clear_payload(&ctx); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index adffbbeb..07715d16 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -612,6 +612,9 @@ pub struct UiState { /// Flowchart panel options (layers, effects, solids, direction, root comp). #[serde(default)] pub flowchart: crate::panels::flowchart::FlowOptions, + /// Lumetri Scopes panel options. + #[serde(default)] + pub scopes: crate::panels::scopes_panel::ScopesState, } impl Default for UiState { @@ -665,6 +668,7 @@ impl Default for UiState { anim_tools: AnimToolsState::default(), mini_flowchart: None, flowchart: Default::default(), + scopes: Default::default(), } } } diff --git a/crates/ui-egui/tests/ui_panels.rs b/crates/ui-egui/tests/ui_panels.rs new file mode 100644 index 00000000..67d3db92 --- /dev/null +++ b/crates/ui-egui/tests/ui_panels.rs @@ -0,0 +1,104 @@ +//! Headless checks for the M13.6 panels: each opens from the Window menu as a real panel and +//! registers its automation ids; the Footage panel's buttons edit into the comp; the Progress +//! panel lists and cancels a job; Lumetri Scopes switch scope through their dropdown. + +use effectcraft_engine::Session; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui::{Event, Pos2, pos2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Panels", "width": 320, "height": 180, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn open(h: &mut Harness<'_, EffectcraftApp>, panel: &str) { + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "window.panel", json!({"panel": panel})).unwrap(); + h.run_steps(3); +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")).clone(); + let p: Pos2 = pos2(e.rect[0] + e.rect[2] / 2.0, e.rect[1] + e.rect[3] / 2.0); + h.input_mut().events.push(Event::PointerMoved(p)); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(2); +} + +#[test] +fn window_menu_opens_the_new_panels() { + let mut h = harness(); + for (name, kind, auto) in [ + ("lumetriScopes", PanelKind::LumetriScopes, "scopes.kind"), + ("footage", PanelKind::Footage, "footage.empty"), + ("mediaBrowser", PanelKind::MediaBrowser, "mediaBrowser.path"), + ("metadata", PanelKind::Metadata, "metadata.projectComment"), + ("progress", PanelKind::Progress, ""), + ("contentAwareFill", PanelKind::ContentAwareFill, "contentFill.method"), + ] { + open(&mut h, name); + assert!(h.state().ui.dock.contains(kind) || h.state().ui.floating.iter().any(|f| f.panels.contains(&kind)), "{name} shown"); + if !auto.is_empty() { + assert!(h.state().auto.find(auto).is_some(), "{name}: {auto}"); + } + } +} + +#[test] +fn scopes_switch_kind() { + let mut h = harness(); + open(&mut h, "lumetriScopes"); + assert!(h.state().auto.find("scopes.plot").is_some()); + h.state_mut().ui.scopes.scope = "vectorscopeYuv".into(); + h.run_steps(2); + assert_eq!(h.state().auto.find("scopes.plot").unwrap().label, "Vectorscope YUV"); + h.state_mut().ui.scopes.scope = "histogram".into(); + h.run_steps(2); + assert_eq!(h.state().auto.find("scopes.kind").unwrap().label, "Histogram"); +} + +#[test] +fn footage_panel_buttons_edit_into_the_comp() { + let mut h = harness(); + let item = h.state().session.project.items.values().find(|i| matches!(i.kind, effectcraft_engine::project::ItemKind::Solid(_))).unwrap().id.0; + h.state_mut().session.execute("footage.open", json!({"item": item})).unwrap(); + h.run_steps(4); + assert!(h.state().ui.dock.contains(PanelKind::Footage) || h.state().ui.floating.iter().any(|f| f.panels.contains(&PanelKind::Footage))); + assert!(h.state().auto.find("footage.overlayEdit").is_some()); + let before = h.state().session.active_comp().unwrap().layers.len(); + click(&mut h, "footage.overlayEdit"); + assert_eq!(h.state().session.active_comp().unwrap().layers.len(), before + 1); + click(&mut h, "footage.rippleInsertEdit"); + assert!(h.state().session.active_comp().unwrap().layers.len() >= before + 2); +} + +#[test] +fn progress_panel_cancels_a_job() { + let mut h = harness(); + h.state_mut() + .session + .spawn_task("test", "Long analysis", false, |ctl| { + while ctl.progress(1, 2) { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Err("cancelled".into()) + }) + .unwrap(); + open(&mut h, "progress"); + let id = h.state().session.jobs()[0].id.clone(); + assert!(h.state().auto.find(&format!("progress.job.{id}")).is_some()); + click(&mut h, &format!("progress.cancel.{id}")); + h.state_mut().session.wait_jobs(); + h.run_steps(2); + assert!(h.state().session.jobs().is_empty()); + assert_eq!(h.state().session.job_log.last().unwrap().status, "cancelled"); +} diff --git a/docs/architecture.md b/docs/architecture.md index 39acd396..f37f4e73 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -267,6 +267,42 @@ demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes in the effect. +**Content-Aware Fill** (`raster::inpaint`, `engine::commands::content_fill`): the hole is +where the layer's masked source (`Renderer::layer_input` with no effects) is under half +opacity, grown by Alpha Expansion. Optical flow between neighbouring membrane-filled frames is +*completed* inside the holes (membrane interpolation of the surrounding tiles' motion); each +missing pixel follows it forwards and backwards to the nearest frame (or reference frame) where +it is visible, the two candidates blended by temporal distance, with Lighting Correction adding +the membrane of the border mismatch (Poisson-style). What no frame shows is synthesised by +PatchMatch (Barnes et al. 2009) inside Wexler et al.'s (2007) multi-scale EM: Object refines each +frame starting from the previous result warped along the flow; Surface carries one synthesised +fill through the range; Edge Blend is the membrane fill alone. The frames are written as a PNG +sequence and imported into a "Fill" layer above the source. It runs as a generic **background +task** (`engine::jobs`: a closure with progress and cancel on a thread, inline on wasm32 or +with `wait`, whose result is applied on the UI thread as one undo step); the Progress panel and +`jobs.list` / `jobs.cancel` show these tasks together with the render queue and the analyses. + +**Scene Edit Detection** (`raster::cuts`) compares consecutive frames by colour-histogram +distance and motion-compensated (block-flow warped) luma error, takes their geometric mean and +marks frames that exceed a threshold and several times their neighbours' median; the cuts +become layer markers, split layers, or split-and-precomposed layers. **Auto-trace** +(`path::trace`) runs marching squares on the chosen channel, Douglas–Peucker simplification and +cubic fitting with Corner Roundness blending chord and smooth tangents; holes become Subtract +masks, and a work-area trace keys Mask Path per frame. + +**Align Video to Data** reads data footage (File ▸ Import of `.json`, `.csv`, `.tsv`, kept as +text in the project). Supported data: JSON as an array of objects or an object holding one +(`{"samples": [...]}`), CSV/TSV with a header row (quoted fields allowed). The time key is the +`key` parameter or the first field named `time`, `timestamp`, `t`, `date`, `datetime`, `utc`, +`gps_time`, `seconds` or `time_s`; values may be ISO 8601 date-times (with `Z` or `±hh:mm`), +times of day (`hh:mm:ss.sss`), timecode (`hh:mm:ss:ff` at the footage rate), Unix seconds or +Unix milliseconds. The layer's Start Time becomes `dataStart + videoStart − firstSample` +(`videoStart` given, or the file's creation date on the desktop; reduced to a time of day when +the data has no dates). + +**Lumetri Scopes** (`raster::scopes`) are density plots of the viewer frame (rendered over the +comp background) with BT.601 / BT.709 / BT.2020 luma and colour-difference coefficients. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index fbe12a5f..a38ed840 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -24,9 +24,8 @@ renders); stroke taper/wave and multi-segment dashes; variable mask feather poin rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur and blend modes; a few viewer and colour-management menu items; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and -the "better than After Effects" items (a plug-in API, branching history, GPU particles). +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; face +tracking; and the "better than After Effects" items (a plug-in API, branching history, GPU particles). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -167,6 +166,25 @@ style runs; the Layer Style dialog (LYR-9); Graph Editor snapping to markers and Reference Axes; Animate Text ▸ Variable Font Axes. Remaining in these rows: Lottie can't carry taper/wave, Advanced 3D's DOF has no iris shapes, and variable axes don't change advances. +## Update: M13.6 panels and content tools + +The last placeholder panels became real panels with automation ids: **Lumetri Scopes** +(waveform RGB / Luma / YC, vectorscope YUV / HLS, histogram, parade RGB / YUV; Rec. 601 / 709 / +2020, 8-bit or float scale, clamp; `scopes.analyze`), **Footage** (double-click footage: its own +time ruler, play, In/Out, Overlay Edit and Ripple Insert Edit; `footage.*`), **Media Browser** +(desktop file system, favourites, thumbnails, import and drag to the Project panel or timeline; +`mediaBrowser.*`), **Metadata** (codec, size, rate, duration, colour profile, file dates; item +and project comments; `item.metadata`) and **Progress** (every render and analysis with +progress and cancel; `jobs.*`). **Content-Aware Fill** (panel and Layer ▸ New ▸ Content-Aware +Fill Layer): Object / Surface / Edge Blend, Work Area / Entire Duration, Alpha Expansion, +Lighting Correction and a reference frame from a layer, rendered to a PNG sequence in a Fill +layer above the source as a background job. **Scene Edit Detection** (markers, split, split and +precompose), **Auto-trace** (alpha / RGB / luminance, current frame or work area, to the layer +or a new layer) and **Align Video to Data** (JSON / CSV / TSV time keys) replace their disabled +menu entries. Not yet: creating a reference frame by editing a still (After Effects hands it to +Photoshop), Content-Aware Fill's learned model (ours is PatchMatch + flow propagation), and Media +Browser in the web app. + ## Highest-value gaps, in order 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10). From e362cae4c75d8d0145f0f36e03ceebdfb97f6220 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:21:30 -0400 Subject: [PATCH 16/36] M13.6: panel polish (log-density scopes, narrow-panel layouts) and an offscreen panels snapshot test Co-Authored-By: Claude Opus 5.5 --- crates/raster/src/scopes.rs | 9 +++-- .../ui-egui/src/panels/content_fill_panel.rs | 2 +- crates/ui-egui/src/panels/media_panels.rs | 10 +++-- crates/ui-egui/src/panels/scopes_panel.rs | 6 ++- crates/ui-egui/tests/ui_panels.rs | 40 +++++++++++++++++++ 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/crates/raster/src/scopes.rs b/crates/raster/src/scopes.rs index befb26ef..6b803a12 100644 --- a/crates/raster/src/scopes.rs +++ b/crates/raster/src/scopes.rs @@ -5,7 +5,8 @@ //! ITU-R BT.601 / BT.709 / BT.2020 coefficients, colour differences `Cb = (B − Y) / (2(1 − Kb))` //! and `Cr = (R − Y) / (2(1 − Kr))` (each in −0.5…0.5). Values are read straight (un-premultiplied) //! from the frame. Every scope is a [`Scope`]: a `width × height` grid of three density planes -//! (one per trace colour) normalised so the busiest cell is 1, ready to be tinted and drawn. +//! (one per trace colour) on a logarithmic scale where the busiest cell is 1, ready to be tinted +//! and drawn. //! //! Scales: **8-bit** maps 0…1 to the plot (0…255 graticule); **float** maps −0.25…1.25 so //! super-white and sub-black values stay visible. **Clamp** limits the signal to 0…1 first. @@ -178,10 +179,12 @@ impl Scope { self } fn normalise(&mut self) { - // Square-root density so faint traces stay visible next to dense ones. + // Logarithmic density (like a phosphor trace) so faint traces stay visible next to + // dense ones. let max = self.counts.iter().flat_map(|p| p.iter()).copied().max().unwrap_or(0).max(1) as f32; + let k = 1.0 / (1.0 + max).ln(); for c in 0..3 { - self.planes[c] = self.counts[c].iter().map(|&n| (n as f32 / max).sqrt()).collect(); + self.planes[c] = self.counts[c].iter().map(|&n| if n == 0 { 0.0 } else { (0.25 + 0.75 * (1.0 + n as f32).ln() * k).min(1.0) }).collect(); } } /// Total count of plane `c`. diff --git a/crates/ui-egui/src/panels/content_fill_panel.rs b/crates/ui-egui/src/panels/content_fill_panel.rs index 5d534e59..1937b4cc 100644 --- a/crates/ui-egui/src/panels/content_fill_panel.rs +++ b/crates/ui-egui/src/panels/content_fill_panel.rs @@ -45,7 +45,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { .unwrap_or(0); format!("{} — {masks} subtracting mask(s)", l.name) } - None => "Select a layer with transparent areas (e.g. a Subtract mask)".into(), + None => "Select a layer with a Subtract mask".into(), }; p.text(pos2(x0, y + 9.0), Align2::LEFT_CENTER, &target, Tokens::ui(11.5), if layer.is_some() { t.text_dim } else { t.text_faint }); app.auto.add("contentFill.target", Rect::from_min_size(pos2(x0, y), vec2(rect.width() - 24.0, 18.0)), &target); diff --git a/crates/ui-egui/src/panels/media_panels.rs b/crates/ui-egui/src/panels/media_panels.rs index b1a3230e..4f6fd0ae 100644 --- a/crates/ui-egui/src/panels/media_panels.rs +++ b/crates/ui-egui/src/panels/media_panels.rs @@ -104,11 +104,13 @@ pub fn media_browser(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if only != st.importable_only { kit::exec(app, "mediaBrowser.go", json!({"path": dir, "importableOnly": only})); } - x += 100.0; - p.text(pos2(x, y + 10.0), Align2::LEFT_CENTER, &dir, Tokens::ui(11.5), t.text_dim); - app.auto.add("mediaBrowser.path", Rect::from_min_max(pos2(x, y), pos2(rect.max.x - 8.0, y + 20.0)), &dir); + // The path on its own row. + let _ = x; + let py = y + 24.0; + p.text(pos2(rect.min.x + 10.0, py + 9.0), Align2::LEFT_CENTER, &dir, Tokens::ui(11.5), t.text_dim); + app.auto.add("mediaBrowser.path", Rect::from_min_max(pos2(rect.min.x + 8.0, py), pos2(rect.max.x - 8.0, py + 18.0)), &dir); // Left column: favourites. - let side = Rect::from_min_max(pos2(rect.min.x, rect.min.y + 32.0), pos2(rect.min.x + 160.0_f32.min(rect.width() * 0.3), rect.max.y)); + let side = Rect::from_min_max(pos2(rect.min.x, rect.min.y + 54.0), pos2(rect.min.x + 160.0_f32.min(rect.width() * 0.3), rect.max.y)); p.line_segment([side.right_top(), side.right_bottom()], Stroke::new(1.0, t.separator)); p.text(pos2(side.min.x + 10.0, side.min.y + 10.0), Align2::LEFT_CENTER, "Favorites", Tokens::semibold(11.5), t.text_dim); let mut fy = side.min.y + 22.0; diff --git a/crates/ui-egui/src/panels/scopes_panel.rs b/crates/ui-egui/src/panels/scopes_panel.rs index d220213b..db5279d8 100644 --- a/crates/ui-egui/src/panels/scopes_panel.rs +++ b/crates/ui-egui/src/panels/scopes_panel.rs @@ -62,11 +62,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { st.float = i == 1; } x += 78.0; - st.clamp = kit::checkbox(app, ui, pos2(x, y + 2.0), "scopes.clamp", "Clamp Signal", st.clamp); + // Narrow panels: Clamp Signal on a second row. + let (cx, cy, top) = if x + 110.0 > rect.max.x { (rect.min.x + 8.0, y + 26.0, 60.0) } else { (x, y, 34.0) }; + st.clamp = kit::checkbox(app, ui, pos2(cx, cy + 2.0), "scopes.clamp", "Clamp Signal", st.clamp); if st != app.ui.scopes { app.ui.scopes = st.clone(); } - let area = Rect::from_min_max(pos2(rect.min.x + 8.0, rect.min.y + 34.0), pos2(rect.max.x - 8.0, rect.max.y - 8.0)); + let area = Rect::from_min_max(pos2(rect.min.x + 8.0, rect.min.y + top), pos2(rect.max.x - 8.0, rect.max.y - 8.0)); if area.width() < 40.0 || area.height() < 40.0 { return; } diff --git a/crates/ui-egui/tests/ui_panels.rs b/crates/ui-egui/tests/ui_panels.rs index 67d3db92..ee027daa 100644 --- a/crates/ui-egui/tests/ui_panels.rs +++ b/crates/ui-egui/tests/ui_panels.rs @@ -102,3 +102,43 @@ fn progress_panel_cancels_a_job() { assert!(h.state().session.jobs().is_empty()); assert_eq!(h.state().session.job_log.last().unwrap().status, "cancelled"); } + +/// Headless look at the panels (wgpu offscreen; needs a GPU adapter). Run with +/// `PANELS_SNAPSHOT=/abs/dir cargo test -p effectcraft-ui-egui --test ui_panels -- --ignored`. +#[test] +#[ignore] +fn panels_snapshot() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + let solid = s.project.items.values().find(|i| matches!(i.kind, effectcraft_engine::project::ItemKind::Solid(_))).map(|i| i.id.0); + let mut app = Some(EffectcraftApp::new(s)); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| app.take().expect("app")); + h.run_steps(3); + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "window.workspace", json!({"name": "All Panels"})).unwrap(); + h.run_steps(3); + let dir = std::env::var("PANELS_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out").into()); + std::fs::create_dir_all(&dir).unwrap(); + if let Some(id) = solid { + h.state_mut().session.execute("footage.open", json!({"item": id})).unwrap(); + } + for (panel, scope) in [ + ("lumetriScopes", "waveformRgb"), + ("lumetriScopes", "vectorscopeYuv"), + ("lumetriScopes", "histogram"), + ("lumetriScopes", "paradeRgb"), + ("contentAwareFill", ""), + ("metadata", ""), + ("progress", ""), + ("mediaBrowser", ""), + ("footage", ""), + ] { + if !scope.is_empty() { + h.state_mut().ui.scopes.scope = scope.into(); + } + open(&mut h, panel); + h.run_steps(3); + let img = h.render().expect("render"); + img.save(format!("{dir}/{panel}{scope}.png")).unwrap(); + } +} From 02c9a3f13c255c22249f3fa8c4d43578734bd1be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:38:33 -0400 Subject: [PATCH 17/36] M13.2: collapsed precomps project their 3D layers (and environment sky) with the parent comp's view size Co-Authored-By: Claude Opus 5.5 --- crates/render/src/tests_collapse.rs | 34 ++++++++++++++++++++++++++++ crates/render/src/three_d/compose.rs | 11 +++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/crates/render/src/tests_collapse.rs b/crates/render/src/tests_collapse.rs index d3cf954c..0da0288f 100644 --- a/crates/render/src/tests_collapse.rs +++ b/crates/render/src/tests_collapse.rs @@ -200,3 +200,37 @@ fn wireframe_quality_draws_the_layer_bounds() { let img = render_frame(&p, cid, Tick::ZERO, 1.0); assert!(img.data.iter().all(|q| q[3] == 0.0 || q[3] == 1.0)); } + +#[test] +fn collapsed_precomp_of_another_size_renders_through_the_parent_camera() { + // A 1000×600 scene comp with a blue 3D solid at its centre, collapsed into a 300×200 comp + // whose camera looks at the precomp layer (at the parent's centre). The nested layers must + // be projected with the parent comp's view (its centre and size), not the scene comp's. + let run = |three_d_precomp: bool, camera: bool| { + let mut p = project(); + let outer = Comp::new(300, 200, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let inner = Comp::new(1000, 600, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let mut blue = solid(&mut p, &inner, [0.0, 0.0, 1.0], (40, 40), [500.0, 300.0, 0.0]); + blue.switches.three_d = true; + let mut inner = inner; + inner.layers = vec![blue]; + let iid = p.add_item("Scene", Label::Sandstone, None, ItemKind::Comp(inner.into())); + let mut pre = build::layer(&mut p, &outer, "Scene", LayerSource::Comp { item: iid }, (1000, 600), None); + pre.switches.three_d = three_d_precomp; + pre.switches.collapse = true; + let mut c = outer.clone(); + c.layers = vec![pre]; + if camera { + let cam = build::layer(&mut p, &outer, "Camera", LayerSource::Camera, (300, 200), None); + c.layers.insert(0, cam); + } + let cid = p.add_item("Face", Label::Sandstone, None, ItemKind::Comp(c.into())); + render_frame(&p, cid, Tick::ZERO, 1.0) + }; + for (three_d, camera) in [(true, true), (true, false), (false, true)] { + let img = run(three_d, camera); + let c = img.get(150, 100); + assert!(c[2] > 0.99 && c[3] > 0.99, "3D precomp {three_d}, camera {camera}: centre {c:?}"); + assert!(img.get(100, 100)[3] < 0.01, "only the 40 px solid is drawn"); + } +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index b134c88f..c97b78bb 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -161,6 +161,13 @@ pub(crate) fn camera_for(r: &Renderer, ctx: &EvalCtx) -> CameraState { } } +/// The size of the comp the camera looks through: a collapsed precomp's layers are projected +/// into the outermost comp they collapse into (its centre and size), not their own comp's. +pub(crate) fn view_size(r: &Renderer, ctx: &EvalCtx) -> (f64, f64) { + let c = r.collapse3d.map_or(ctx.comp, |c| c.parent.comp); + (c.width as f64, c.height as f64) +} + /// Number of motion-blur sub-samples for a layer. fn mb_samples(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> usize { if r.opts.motion_blur && ctx.comp.enable_motion_blur && layer.switches.motion_blur { @@ -202,7 +209,7 @@ fn prepare_with<'a>( if buf.img.is_empty() { return None; } - let comp = (ctx.comp.width as f64, ctx.comp.height as f64); + let comp = view_size(r, ctx); let s = r.opts.scale; let cam = camera_for(r, ctx); let outer = r.collapse3d.map_or(Mat4::IDENTITY, |c| c.world); @@ -920,7 +927,7 @@ pub(crate) fn draw_sky(r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut let cam = camera_for(r, ctx); let (fwd, right, down) = (cam.forward(), cam.right(), cam.down()); let s = r.opts.scale.max(1e-9); - let (cw, ch) = (ctx.comp.width as f64, ctx.comp.height as f64); + let (cw, ch) = view_size(r, ctx); let (ox, oy) = r.roi_offset().map_or((0.0, 0.0), |(x, y, _, _)| (x, y)); let rot = environment_rotation(ctx); let (iw, ih) = (img.width as f64, img.height as f64); From 4210f003023c265fed531c0004eea8ad16183633 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:40:59 -0400 Subject: [PATCH 18/36] M14.5: fmt after merge Co-Authored-By: Claude Opus 5.5 --- crates/render/src/three_d/compose.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index c2dd4ec0..a4bf7f20 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -682,8 +682,7 @@ pub struct Plane3d { /// Prepare a run of 3D layers for an accelerator. `None` when the run must be drawn on the CPU /// (adjustment, wireframe or environment background layers, Advanced 3D). pub(crate) fn gpu_run(r: &Renderer, ctx: &EvalCtx, run: &[&Layer], out: (u32, u32)) -> Option { - if run.is_empty() || run.iter().any(|l| l.switches.adjustment || l.environment_background || r.quality(l) == effectcraft_project::Quality::Wireframe) - { + if run.is_empty() || run.iter().any(|l| l.switches.adjustment || l.environment_background || r.quality(l) == effectcraft_project::Quality::Wireframe) { return None; } if ctx.comp.renderer == effectcraft_project::Renderer::Advanced3D && r.collapse3d.is_none() { From b26fbb6247977158fb7022c303e0bba25333667d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:44:03 -0400 Subject: [PATCH 19/36] M10.2: VP9 inter frames (motion search, MV prediction, loop filter, forward probability updates, rate control) effectcraft-vp9enc gains inter frames: single LAST reference, 16x16/8x8 blocks, candidate + diamond search refined to quarter samples with the regular 8-tap filters, NEARESTMV/NEARMV/ZEROMV/NEWMV with spec MV prediction, skip blocks, the in-loop deblocking filter, per-frame forward probability updates, a key frame interval and target-bitrate rate control. Decoded output matches the encoder's reconstruction exactly through FilmCraft's decoder and ffmpeg. WebM export now encodes sequentially with a key frame every two seconds. Co-Authored-By: Claude Opus 5.5 --- crates/export/src/lib.rs | 2 +- crates/export/src/webm.rs | 72 ++- crates/export/tests/vp9_roundtrip.rs | 181 +++++- crates/vp9enc/Cargo.toml | 2 +- crates/vp9enc/README.md | 52 +- crates/vp9enc/src/bool.rs | 25 - crates/vp9enc/src/lf.rs | 124 ++++ crates/vp9enc/src/lib.rs | 934 ++++++++------------------- crates/vp9enc/src/mc.rs | 207 ++++++ crates/vp9enc/src/probs.rs | 426 ++++++++++++ crates/vp9enc/src/tables.rs | 73 +++ crates/vp9enc/src/tile.rs | 894 +++++++++++++++++++++++++ crates/vp9enc/src/transform.rs | 249 +++++++ docs/parity.md | 2 +- 14 files changed, 2511 insertions(+), 732 deletions(-) create mode 100644 crates/vp9enc/src/lf.rs create mode 100644 crates/vp9enc/src/mc.rs create mode 100644 crates/vp9enc/src/probs.rs create mode 100644 crates/vp9enc/src/tile.rs create mode 100644 crates/vp9enc/src/transform.rs diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 941a7723..c9196625 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,7 +7,7 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | -//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WebM | WebM (Matroska) | VP9 profile 0 (`effectcraft-vp9enc`: key + inter frames, a key frame every 2 s, loop filter), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | //! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index eb99cd22..65ca1189 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -5,8 +5,10 @@ //! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are //! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose //! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, -//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every -//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (key +//! frames every two seconds, inter frames between them: SimpleBlocks carry the key flag, inter +//! BlockGroups a ReferenceBlock; the alpha stream's key frames line up with the colour stream's), +//! Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, //! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit //! extended sample rate) follow their published file layouts. @@ -82,6 +84,10 @@ fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { struct Frame { ms: i64, track: u64, + /// A key frame (VP9) or any audio packet. + key: bool, + /// Timestamp of the previous video frame (ReferenceBlock of inter frames). + prev_ms: Option, data: Vec, alpha: Option>, } @@ -94,10 +100,15 @@ fn cluster(frames: &mut [Frame]) -> Vec { for f in frames.iter() { let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; match &f.alpha { - None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + None => el(&mut body, 0xA3, &block(f.track, rel, if f.key { 0x80 } else { 0x00 }, &f.data)), Some(a) => { let mut g = vec![]; el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + if let (false, Some(p)) = (f.key, f.prev_ms) { + // ReferenceBlock: a signed timestamp relative to this block (an inter frame). + let d = (p - f.ms).clamp(i16::MIN as i64, -1) as i16; + el(&mut g, 0xFB, &d.to_be_bytes()); + } let mut more = vec![]; el_uint(&mut more, 0xEE, 1); el(&mut more, 0xA5, a); @@ -117,7 +128,19 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; let quality = job.output.quality.clamp(1, 100); - let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let fps = rate.as_f64().max(1.0); + let cfg = effectcraft_vp9enc::EncoderConfig { + width: w, + height: h, + quality, + full_range: false, + keyframe_interval: (fps * 2.0).round().max(1.0) as u32, + target_kbps: None, + frame_rate: fps, + loop_filter: true, + }; + let mut color_enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let mut alpha_enc = alpha.then(|| effectcraft_vp9enc::Vp9Encoder::new(cfg.clone())); let with_audio = wants_audio(job); let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); let total = st.total; @@ -209,7 +232,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re while pcm_left.len() >= fs { let pkt = enc.encode_float(&pcm_left[..fs]); pcm_left.drain(..fs); - frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + frames.push(Frame { ms: packets as i64 * 20, track: 2, key: true, prev_ms: None, data: pkt, alpha: None }); packets += 1; } }; @@ -218,23 +241,28 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let mut i = 0; while i < total { let end = (i + batch).min(total); - let encoded: Vec<(Vec, Option>)> = (i..end) - .into_par_iter() - .map(|k| { - let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); - let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); - let color = enc.encode_yuv420(&y, &u, &vv); - let a = alpha.then(|| { - let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); - ea.encode_yuv420(&y, &u, &vv) - }); - (color, a) - }) - .collect(); - let mut frames: Vec = - encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and + // alpha side by side. + let pixels: Vec> = + (i..end).into_par_iter().map(|k| rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h)).collect(); + let mut frames: Vec = Vec::with_capacity(pixels.len()); + for (j, px) in pixels.iter().enumerate() { + let (color, a) = rayon::join( + || { + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(px, w, h, false); + color_enc.encode_frame(&y, &u, &vv, false) + }, + || { + alpha_enc.as_mut().map(|ea| { + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(px, w, h); + ea.encode_frame(&y, &u, &vv, false).data + }) + }, + ); + let k = i + j as u64; + frames.push(Frame { ms: frame_ms(k), track: 1, key: color.key, prev_ms: (k > 0).then(|| frame_ms(k - 1)), data: color.data, alpha: a }); + } + drop(pixels); let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; audio_until(t_end, &mut frames, end >= total); let c = cluster(&mut frames); diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index 8156d850..5bf46e0a 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -61,7 +61,7 @@ fn lossless_is_exact_for_odd_sizes() { for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { for kind in 0..3 { let (y, u, v) = source(w, h, kind); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert_eq!((p.width, p.height), (w as u32, h as u32)); @@ -80,7 +80,7 @@ fn lossy_quality_and_size() { let (y, u, v) = source(w, h, kind); let mut last = 0usize; for q in [30u8, 60, 80, 95] { - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); let got = crop(&plane_u8(&p.y), p.y_stride, w, h); @@ -103,7 +103,7 @@ fn wide_frames_use_tile_columns() { // 4160 px > 64 superblocks: two tile columns are required. let (w, h) = (4160, 16); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); @@ -113,7 +113,7 @@ fn wide_frames_use_tile_columns() { fn ffmpeg_decodes_our_stream() { let (w, h) = (96, 64); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let frame = e.encode_yuv420(&y, &u, &v); // IVF container (one frame). let mut ivf = Vec::new(); @@ -147,3 +147,176 @@ fn ffmpeg_decodes_our_stream() { assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); } + +// ---------------------------------------------------------------- inter frames + +/// Frame `t` of a moving sequence: a diagonal gradient with a sinusoidal texture and bright +/// squares, all translating by a fractional amount per frame (`kind` 1 adds noise that changes +/// every frame). +fn moving(w: usize, h: usize, t: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let (dx, dy) = (1.75 * t as f64, 0.6 * t as f64); + let mut s = 1234u32 + t as u32 * 77; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 28) as f64 + }; + let lum = |x: f64, y: f64| -> f64 { + let g = 40.0 + 120.0 * (x / w as f64) + 50.0 * (y / h as f64); + let tex = 18.0 * ((x * 0.31).sin() * (y * 0.23).cos()); + let sq = if (x - 20.0).rem_euclid(64.0) < 18.0 && (y - 10.0).rem_euclid(48.0) < 14.0 { 45.0 } else { 0.0 }; + g + tex + sq + }; + let mut y = vec![0u8; w * h]; + for yy in 0..h { + for x in 0..w { + let n = if kind == 1 { rnd() - 7.5 } else { 0.0 }; + y[yy * w + x] = (lum(x as f64 - dx, yy as f64 - dy) + n).round().clamp(0.0, 255.0) as u8; + } + } + let u: Vec = (0..cw * ch).map(|i| (100.0 + 40.0 * (((i % cw) as f64 * 2.0 - dx) / w as f64)).round() as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (150.0 - 30.0 * (((i / cw) as f64 * 2.0 - dy) / h as f64)).round() as u8).collect(); + (y, u, v) +} + +fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + [crop(&plane_u8(&p.y), p.y_stride, w, h), crop(&plane_u8(&p.u), p.uv_stride, cw, ch), crop(&plane_u8(&p.v), p.uv_stride, cw, ch)] +} + +/// Encode `n` frames and decode them with one decoder: every decoded frame must equal the +/// encoder's reconstruction. Returns (total bytes, mean luma PSNR vs the source). +fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { + let (w, h) = (cfg.width as usize, cfg.height as usize); + let mut e = Vp9Encoder::new(cfg); + let mut d = Decoder::new(); + let (mut bytes, mut db) = (0usize, 0.0); + for t in 0..n { + let (y, u, v) = moving(w, h, t, kind); + let f = e.encode_frame(&y, &u, &v, false); + bytes += f.data.len(); + let kind_s = if f.key { "key" } else { "inter" }; + let mut pics = d.decode(&f.data, t as i64).unwrap_or_else(|err| panic!("{w}x{h} frame {t} ({kind_s}): {err:?}")); + assert_eq!(pics.len(), 1); + let got = planes_of(&pics.remove(0), w, h); + let rec = e.reconstruction().expect("reconstruction"); + for k in 0..3 { + if got[k] != rec[k] { + let first = got[k].iter().zip(&rec[k]).position(|(a, b)| a != b).unwrap_or(0); + let pw = if k == 0 { w } else { w.div_ceil(2) }; + panic!("{w}x{h} frame {t} ({kind_s}, q{} lf{}): plane {k} differs first at ({}, {})", f.q_idx, f.filter_level, first % pw, first / pw); + } + } + db += psnr(&got[0], &y); + } + (bytes, db / n as f64) +} + +#[test] +fn inter_frames_decode_exactly() { + for (w, h) in [(64, 64), (96, 72), (33, 17), (130, 66)] { + for (q, kind) in [(30u8, 0u32), (70, 1), (92, 0), (100, 0)] { + let cfg = EncoderConfig { width: w as u32, height: h as u32, quality: q, keyframe_interval: 6, ..Default::default() }; + run_sequence(cfg, 8, kind); + } + } +} + +#[test] +fn inter_frames_with_tile_columns_decode_exactly() { + let cfg = EncoderConfig { width: 4160, height: 24, quality: 80, keyframe_interval: 0, ..Default::default() }; + run_sequence(cfg, 3, 0); +} + +#[test] +fn inter_psnr_rises_with_bitrate() { + let (w, h) = (160, 96); + let mut last = (0usize, 0.0f64); + let mut rows = vec![]; + for kbps in [60u32, 150, 400, 1200] { + let cfg = EncoderConfig { width: w, height: h, quality: 60, target_kbps: Some(kbps), frame_rate: 30.0, keyframe_interval: 30, ..Default::default() }; + let (bytes, db) = run_sequence(cfg, 20, 0); + rows.push(format!("{kbps} kbps target: {:.0} kbps actual, {db:.2} dB", bytes as f64 * 8.0 * 30.0 / 20.0 / 1000.0)); + assert!(bytes > last.0 && db > last.1, "{rows:#?}"); + last = (bytes, db); + } + eprintln!("{rows:#?}"); +} + +#[test] +fn inter_frames_are_much_smaller_than_intra() { + let (w, h) = (192u32, 108u32); + let mut report = vec![]; + for q in [50u8, 75, 90] { + let intra = EncoderConfig { width: w, height: h, quality: q, keyframe_interval: 1, ..Default::default() }; + let inter = EncoderConfig { keyframe_interval: 0, ..intra.clone() }; + let (bi, pi) = run_sequence(intra, 24, 0); + let (bp, pp) = run_sequence(inter, 24, 0); + report.push(format!("q{q}: intra {bi} B {pi:.2} dB, inter {bp} B {pp:.2} dB, ratio {:.2}", bi as f64 / bp as f64)); + assert!(pp > pi - 1.0, "{report:#?}"); + assert!(bi as f64 >= 3.0 * bp as f64, "{report:#?}"); + } + eprintln!("{report:#?}"); +} + +#[test] +fn ffmpeg_decodes_inter_frames_like_the_encoder() { + let (w, h) = (96usize, 64usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 75, keyframe_interval: 5, ..Default::default() }); + let n = 7; + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&(n as u32).to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + let mut recon = vec![]; + for t in 0..n { + let (y, u, v) = moving(w, h, t, 0); + let f = e.encode_frame(&y, &u, &v, false); + ivf.extend_from_slice(&(f.data.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&(t as u64).to_le_bytes()); + ivf.extend_from_slice(&f.data); + recon.push(e.reconstruction().expect("recon")); + } + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-inter-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let fsz = w * h + 2 * w.div_ceil(2) * h.div_ceil(2); + assert_eq!(out.stdout.len(), fsz * n); + for (t, r) in recon.iter().enumerate() { + let frame = &out.stdout[t * fsz..(t + 1) * fsz]; + assert_eq!(&frame[..w * h], &r[0][..], "ffmpeg luma of frame {t}"); + assert_eq!(&frame[w * h..w * h + r[1].len()], &r[1][..], "ffmpeg U of frame {t}"); + } +} + +/// Encoder speed on 720p (run with `--ignored --nocapture`). +#[test] +#[ignore] +fn bench_720p() { + let (w, h) = (1280usize, 720usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 80, keyframe_interval: 0, ..Default::default() }); + for t in 0..6 { + let (y, u, v) = moving(w, h, t, 0); + let t0 = std::time::Instant::now(); + let f = e.encode_frame(&y, &u, &v, false); + eprintln!("frame {t} ({}): {} B in {:.0} ms", if f.key { "key" } else { "inter" }, f.data.len(), t0.elapsed().as_secs_f64() * 1000.0); + } +} diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml index 62144b25..fe2e767c 100644 --- a/crates/vp9enc/Cargo.toml +++ b/crates/vp9enc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-vp9enc" -description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +description = "VP9 encoder (profile 0, 8-bit 4:2:0; intra + inter frames, loop filter, rate control) for WebM export, from the VP9 Bitstream & Decoding Process Specification" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md index c8f781b8..b9f94fae 100644 --- a/crates/vp9enc/README.md +++ b/crates/vp9enc/README.md @@ -1,15 +1,47 @@ # effectcraft-vp9enc -A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, -BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 -DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 -(mathematically lossless). +A small VP9 encoder for WebM export (profile 0, 8-bit 4:2:0, BT.709). + +* **Key frames**: 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 DCT/ADST + transforms, or the Walsh–Hadamard transform at quality 100 (mathematically lossless). +* **Inter frames**: one reference (the previous frame, LAST), 16×16 or 8×8 blocks chosen by a + cost comparison; motion search from the predicted vectors (NearestMv / NearMv / last frame's + co-located vector) with a whole-sample cross/diamond search refined to half and quarter + samples with the regular 8-tap interpolation filters; NEARESTMV / NEARMV / ZEROMV / NEWMV + chosen by bit cost; blocks with no coefficients worth coding are skipped; 4×4 DCT residual + with a rate–distortion check per transform block. Inter frames are coded error resilient + (`error_resilient_mode` = 1: no dependence on earlier probability contexts or motion vectors), + so `UsePrevFrameMvs` is never needed. No high-precision (1/8) vectors, no compound prediction, + no intra blocks inside inter frames. +* **Loop filter**: the in-loop deblocking filter (§8.8) on every lossy frame; the level is the + best of three trials against the source (sharpness 0, no mode/reference deltas). +* **Probabilities**: per-frame forward updates in the compressed header (`diff_update_prob` / + `update_mv_prob`) of the coefficient, skip, inter-mode, is-inter, reference, partition and + motion-vector probabilities. Tiles are recorded as symbols first, the frame's own counts pick + the updates that pay for their signalling, then the symbols are entropy coded with the updated + probabilities. There is no backward adaptation (`refresh_frame_context` = 0). +* **Rate control**: constant quality (quality → `base_q_idx`) or a target bitrate: a per-frame + q index from a bits ≈ complexity / step model per frame type, key frames getting a larger share + of each key-frame interval, with the running over/undershoot paid back over about a second. +* **Key frame interval** is configurable (1 = every frame a key frame, the old intra-only mode). + +The encoder reconstructs exactly what a decoder reconstructs (inter prediction with the +decoder's vector clamping and edge-clamped reference fetches, integer inverse transforms, the +loop filter) and exposes it (`Vp9Encoder::reconstruction`); the tests require FilmCraft's VP9 +decoder (and ffmpeg, when installed) to output exactly that reconstruction, frame by frame, for +moving synthetic sequences at several sizes and qualities, with tile columns, and lossless. + +On a moving-gradient sequence (192×108, 24 frames) inter coding is 3.9–7× smaller than all-intra +at the same quality setting with equal or better PSNR. Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March 2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode -info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) -and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The -boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec -tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR -Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an -external test oracle. No libvpx or other third-party encoder/decoder source was consulted. +info, motion vector and token syntax (§6.4), derives motion vector candidates with the §6.4 +`find_mv_refs` / `find_best_ref_mvs` processes, and reconstructs with the specification's intra +and inter prediction (§8.5), integer inverse transforms (§8.7) and loop filter (§8.8) so it stays +in step with any conformant decoder. The boolean encoder is the inverse of the §9.2 decoder in +the form given in RFC 6386 §7.3. Spec tables (§10) are the specification's; the key-frame tables +were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0), which +is also the decoder the tests round-trip through and whose behaviour (loop filter edge rules, +vector candidate order) the encoder was checked against; ffmpeg is used only as an external test +oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs index 45db92dd..0cd91b0e 100644 --- a/crates/vp9enc/src/bool.rs +++ b/crates/vp9enc/src/bool.rs @@ -64,31 +64,6 @@ impl BoolEncoder { } } - /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). - pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { - // Find the path to the leaf. - fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { - for b in 0..2 { - let t = tree[node + b]; - out.push((node >> 1, b == 1)); - if t <= 0 { - if (-t) as u8 == value { - return true; - } - } else if path(tree, t as usize, value, out) { - return true; - } - out.pop(); - } - false - } - let mut p = Vec::with_capacity(8); - path(tree, 0, value, &mut p); - for (i, bit) in p { - self.write(bit, probs[i]); - } - } - /// Flush (RFC 6386 §7.3) and return the bytes. pub fn finish(mut self) -> Vec { let mut c = self.bit_count; diff --git a/crates/vp9enc/src/lf.rs b/crates/vp9enc/src/lf.rs new file mode 100644 index 00000000..d781144f --- /dev/null +++ b/crates/vp9enc/src/lf.rs @@ -0,0 +1,124 @@ +//! The in-loop deblocking filter (spec 8.8), as the decoder applies it to a frame coded with 4×4 +//! transforms, 8×8 / 16×16 blocks, no segmentation and no reference/mode deltas: superblocks in +//! raster order, for each plane the vertical edges (pass 0) then the horizontal edges (pass 1). +//! Transform edges inside a skipped inter block are left alone; every 4×4 edge uses the 4-tap +//! filter except edges on a 32-sample boundary, which get the 8-tap (flat) filter. + +use crate::{BLOCK_16X16, Mi, Plane}; + +#[derive(Clone, Copy)] +struct Limits { + limit: i32, + blimit: i32, + thresh: i32, +} + +/// Filter the reconstructed planes (Y, U, V, or a prefix of them) in place at `level` (0 = off; +/// sharpness 0). +pub fn filter_frame(planes: &mut [Plane], mi: &[Mi], mi_rows: usize, mi_cols: usize, level: u8) { + if level == 0 { + return; + } + let l = level as i32; + let limit = l.max(1); + let lim = Limits { limit, blimit: 2 * (l + 2) + limit, thresh: l >> 4 }; + for sbr in 0..mi_rows.div_ceil(8) { + for sbc in 0..mi_cols.div_ceil(8) { + for (plane, p) in planes.iter_mut().enumerate() { + for pass in 0..2 { + sb_plane(p, mi, mi_rows, mi_cols, plane, pass, sbr * 8, sbc * 8, lim); + } + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn sb_plane(p: &mut Plane, mi: &[Mi], mi_rows: usize, mi_cols: usize, plane: usize, pass: usize, row: usize, col: usize, lim: Limits) { + let (sx, sy) = if plane > 0 { (1, 1) } else { (0, 0) }; + let (sub, edge_len) = if pass == 0 { (sx, 64 >> sy) } else { (sy, 64 >> sx) }; + let stride = p.w; + for edge in 0..(16usize >> sub) { + let mut i = 0; + while i < edge_len { + let (x, y) = if pass == 0 { (col * 8 + edge * (4 << sx), row * 8 + (i << sy)) } else { (col * 8 + (i << sx), row * 8 + edge * (4 << sy)) }; + if x >= 8 * mi_cols || y >= 8 * mi_rows || (pass == 0 && x == 0) || (pass == 1 && y == 0) { + i += 8; + continue; + } + let lc = ((x >> 3) >> sx) << sx; + let lr = ((y >> 3) >> sy) << sy; + let m = mi[lr * mi_cols + lc]; + let size = if sub == 0 { m.size } else { m.size.max(BLOCK_16X16) }; + let n8 = if size >= BLOCK_16X16 { 2 } else { 1 }; + let is_block_edge = if pass == 0 { x & (8 * n8 - 1) == 0 } else { y & (8 * n8 - 1) == 0 }; + let tx_ok = !m.inter || !m.skip; + if !is_block_edge && !tx_ok { + i += 8; + continue; + } + let wide = edge % 8 == 0; + let n = if pass == 0 { (8 * mi_rows - y).div_ceil(1 << sy) } else { (8 * mi_cols - x).div_ceil(1 << sx) }.min(8).min(edge_len - i); + // A transform edge is not filtered in the last odd chroma column. + let odd_last = pass == 1 && sx == 1 && mi_cols & 1 == 1 && edge & 1 == 1; + let (px, py) = (x >> sx, y >> sy); + for l in 0..n { + let is_tx_edge = tx_ok && !(odd_last && x + (l << sx) + 8 >= mi_cols * 8); + if !(is_block_edge || is_tx_edge) { + continue; + } + let (pos, step) = if pass == 0 { ((py + l) * stride + px, 1) } else { (py * stride + px + l, stride) }; + filter_lane(&mut p.rec, pos, step, wide, lim); + } + i += 8; + } + } +} + +/// Filter mask, narrow (4-tap) and 8-tap flat filters of spec 8.8.5 across one edge position; +/// `pos` is q0, samples across the edge are `step` apart. +fn filter_lane(d: &mut [u8], pos: usize, step: usize, wide: bool, lim: Limits) { + let at = |k: isize| d[(pos as isize + k * step as isize) as usize] as i32; + let (p3, p2, p1, p0) = (at(-4), at(-3), at(-2), at(-1)); + let (q0, q1, q2, q3) = (at(0), at(1), at(2), at(3)); + let Limits { limit, blimit, thresh } = lim; + let mask = (p3 - p2).abs() <= limit + && (p2 - p1).abs() <= limit + && (p1 - p0).abs() <= limit + && (q1 - q0).abs() <= limit + && (q2 - q1).abs() <= limit + && (q3 - q2).abs() <= limit + && (p0 - q0).abs() * 2 + (p1 - q1).abs() / 2 <= blimit; + if !mask { + return; + } + let flat = + wide && (p1 - p0).abs() <= 1 && (q1 - q0).abs() <= 1 && (p2 - p0).abs() <= 1 && (q2 - q0).abs() <= 1 && (p3 - p0).abs() <= 1 && (q3 - q0).abs() <= 1; + let mut set = |k: isize, v: i32| d[(pos as isize + k * step as isize) as usize] = v as u8; + if flat { + // 8-tap: positions p2..q2, each the rounded mean of a 7-wide clamped window plus itself. + let s = [p3, p2, p1, p0, q0, q1, q2, q3]; + let px = |k: isize| s[(k.clamp(-4, 3) + 4) as usize]; + for k in -3isize..3 { + let mut sum = px(k); + for j in -3..=3 { + sum += px(k + j); + } + set(k, (sum + 4) >> 3); + } + return; + } + let c = |v: i32| v.clamp(-128, 127); + let hev = (p1 - p0).abs() > thresh || (q1 - q0).abs() > thresh; + let (ps1, ps0, qs0, qs1) = (p1 - 128, p0 - 128, q0 - 128, q1 - 128); + let filter = c(if hev { c(ps1 - qs1) } else { 0 } + 3 * (qs0 - ps0)); + let f1 = c(filter + 4) >> 3; + let f2 = c(filter + 3) >> 3; + set(0, c(qs0 - f1) + 128); + set(-1, c(ps0 + f2) + 128); + if !hev { + let f = (f1 + 1) >> 1; + set(1, c(qs1 - f) + 128); + set(-2, c(ps1 + f) + 128); + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs index 02cbb0e2..8f72cb7b 100644 --- a/crates/vp9enc/src/lib.rs +++ b/crates/vp9enc/src/lib.rs @@ -1,290 +1,89 @@ -//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & -//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see -//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width -//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard -//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by -//! prediction error, default probabilities (no probability updates). The encoder reconstructs -//! exactly what a decoder reconstructs (the specification's integer inverse transforms and -//! intra edge rules), so prediction never drifts. +//! A VP9 encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & Decoding Process +//! Specification* v0.6 by mirroring its decoding process in reverse (see README.md). +//! +//! * **Key frames**: 64×64 superblocks split to 8×8 intra blocks (DC / V / H / TM chosen per +//! block), 4×4 DCT/ADST transforms (the Walsh–Hadamard transform at quality 100: lossless). +//! * **Inter frames** (error resilient, single reference: the previous frame as LAST): 16×16 or +//! 8×8 blocks with motion vectors found by a candidate + diamond search refined to quarter +//! samples with the regular 8-tap filters; NEARESTMV / NEARMV / ZEROMV / NEWMV chosen by cost, +//! motion vector prediction exactly as the decoder derives it; skipped blocks; 4×4 DCT residual. +//! * **In-loop deblocking** (8.8) on every lossy frame; the level is picked per frame by trial. +//! * **Probabilities**: per-frame forward updates (compressed header) of the coefficient, skip, +//! inter mode, reference, partition and motion vector probabilities, chosen from the frame's +//! own symbol counts (no backward adaptation: `refresh_frame_context` = 0). +//! * **Rate control**: constant quality (quality → base_q_idx), or a target bitrate with a +//! per-frame q_idx model. +//! +//! The encoder reconstructs exactly what a decoder reconstructs (prediction, integer inverse +//! transforms, loop filter), so prediction never drifts; [`Vp9Encoder::reconstruction`] returns +//! it. mod bool; +mod lf; +mod mc; +mod probs; mod tables; +mod tile; +mod transform; -use bool::BoolEncoder; -use tables::*; +pub use mc::Mv; +use probs::Probs; +use tables::{AC_QLOOKUP, DC_QLOOKUP}; +use transform::Quant; -const DC_PRED: u8 = 0; -const V_PRED: u8 = 1; -const H_PRED: u8 = 2; -const TM_PRED: u8 = 9; -const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; - -const DCT_DCT: u8 = 0; -const ADST_DCT: u8 = 1; -const DCT_ADST: u8 = 2; -const ADST_ADST: u8 = 3; - -/// mode2txfm_map for the modes used here. -fn mode_tx_type(mode: u8) -> u8 { - match mode { - V_PRED => ADST_DCT, - H_PRED => DCT_ADST, - TM_PRED => ADST_ADST, - _ => DCT_DCT, - } -} - -const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; -const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; - -/// extra_bits[token] = (cat, numExtra, base). -const EXTRA_BITS: [(u8, u8, u16); 11] = - [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; -const CAT_PROBS: [&[u8]; 7] = [ - &[], - &[159], - &[165, 145], - &[173, 148, 140], - &[176, 155, 140, 135], - &[180, 157, 141, 134, 130], - &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], -]; +pub(crate) const BLOCK_8X8: u8 = 3; +pub(crate) const BLOCK_16X16: u8 = 6; /// Encoder settings. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct EncoderConfig { pub width: u32, pub height: u32, - /// 1..=100; 100 is lossless. + /// 1..=100; 100 is lossless. With a target bitrate: the quality of the first frame. pub quality: u8, /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. pub full_range: bool, + /// A key frame every this many frames (1 = every frame is a key frame; 0 = only the first). + pub keyframe_interval: u32, + /// Target bitrate in kbit/s (`None`: constant quality). + pub target_kbps: Option, + /// Frames per second (rate control). + pub frame_rate: f64, + /// The in-loop deblocking filter. + pub loop_filter: bool, } -/// Probabilities of the full token tree (9.3.2) for model probability `p`. -fn pareto(p: u8) -> [u8; 8] { - let p = p.max(1) as usize; - let x = (p - 1) / 2; - let mut t = [0u8; 8]; - for n in 0..8 { - t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; - } - t -} - -fn scan(tx_type: u8) -> &'static [u16; 16] { - match tx_type { - ADST_DCT => &ROW_SCAN_4X4, - DCT_ADST => &COL_SCAN_4X4, - _ => &DEFAULT_SCAN_4X4, +impl Default for EncoderConfig { + fn default() -> Self { + EncoderConfig { width: 0, height: 0, quality: 80, full_range: false, keyframe_interval: 60, target_kbps: None, frame_rate: 30.0, loop_filter: true } } } -/// Coefficient context neighbours (9.3.2) of each scan position. -fn neighbors(tx_type: u8) -> [(usize, usize); 16] { - let sc = scan(tx_type); - let mut nb = [(0, 0); 16]; - for (c, &pos) in sc.iter().enumerate().skip(1) { - let pos = pos as usize; - let (i, j) = (pos / 4, pos % 4); - nb[c] = if i > 0 && j > 0 { - let a = (i - 1) * 4 + j; - let b = i * 4 + j - 1; - match tx_type { - DCT_ADST => (a, a), - ADST_DCT => (b, b), - _ => (a, b), - } - } else if i > 0 { - ((i - 1) * 4 + j, (i - 1) * 4 + j) - } else { - (j - 1, j - 1) - }; - } - nb +/// One coded frame. +#[derive(Clone, Debug)] +pub struct EncodedFrame { + pub data: Vec, + pub key: bool, + pub q_idx: u8, + pub filter_level: u8, } -// ---------------------------------------------------------------- transforms (8.7) - -fn r14(v: i32) -> i32 { - (v + (1 << 13)) >> 14 -} - -fn idct4(x: [i32; 4]) -> [i32; 4] { - let v0 = r14(x[0] * 11585 - x[2] * 11585); - let v1 = r14(x[0] * 11585 + x[2] * 11585); - let v2 = r14(x[1] * 6270 - x[3] * 15137); - let v3 = r14(x[1] * 15137 + x[3] * 6270); - [v1 + v3, v0 + v2, v0 - v2, v1 - v3] -} - -fn iadst4(x: [i32; 4]) -> [i32; 4] { - let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); - let x: [i64; 4] = x.map(|v| v as i64); - let s0 = s19 * x[0]; - let s1 = s29 * x[0]; - let s2 = s39 * x[1]; - let s3 = s49 * x[2]; - let s4 = s19 * x[2]; - let s5 = s29 * x[3]; - let s6 = s49 * x[3]; - let s7 = s39 * (x[0] - x[2] + x[3]); - let x0 = s0 + s3 + s5; - let x1 = s1 - s4 - s6; - let x2 = s7; - let x3 = s2; - let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; - [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] -} - -/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. -fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { - let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); - let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; - t[i] = if row_adst { iadst4(r) } else { idct4(r) }; - } - let mut out = [0i32; 16]; - for j in 0..4 { - let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; - let c = if col_adst { iadst4(c) } else { idct4(c) }; - for i in 0..4 { - out[i * 4 + j] = (c[i] + 8) >> 4; - } - } - out -} - -/// Inverse Walsh–Hadamard (8.7.1.10) on one line. -fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { - let mut a = x[0] >> shift; - let mut c = x[1] >> shift; - let mut d = x[2] >> shift; - let mut b = x[3] >> shift; - a += c; - d -= b; - let e = (a - d) >> 1; - b = e - b; - c = e - c; - a -= b; - d += c; - [a, b, c, d] -} - -/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. -fn fwht(y: [i32; 4]) -> [i32; 4] { - let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); - let d1 = dd - cc; - let a1 = aa + bb; - let e = (a1 - d1) >> 1; - let b0 = e - bb; - let c0 = e - cc; - let a0 = a1 - c0; - let d0 = d1 + b0; - [a0, c0, d0, b0] -} - -fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); - } - let mut out = [0i32; 16]; - for j in 0..4 { - let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); - for i in 0..4 { - out[i * 4 + j] = r[i]; - } - } - out -} - -/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. -fn forward_wht(res: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for j in 0..4 { - let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); - for i in 0..4 { - t[i][j] = col[i]; - } - } - let mut lv = [0i32; 16]; - for i in 0..4 { - let r = fwht(t[i]); - lv[i * 4..i * 4 + 4].copy_from_slice(&r); - } - lv -} - -/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse -/// transforms, so `inverse4(forward(res)) ≈ res`. -fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { - use std::sync::OnceLock; - static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); - M.get_or_init(|| { - let mut out = [[[0.0; 16]; 16]; 4]; - for (ty, o) in out.iter_mut().enumerate() { - // Columns of the inverse: response to a scaled unit coefficient. - let k = 1 << 16; - let mut inv = [[0.0f64; 16]; 16]; - for c in 0..16 { - let mut e = [0i32; 16]; - e[c] = k; - let r = inverse4(&e, ty as u8); - for p in 0..16 { - inv[p][c] = r[p] as f64 / k as f64; - } - } - *o = invert16(inv); - } - out - }) -} - -fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { - let mut a = m; - let mut inv = [[0.0; 16]; 16]; - for (i, r) in inv.iter_mut().enumerate() { - r[i] = 1.0; - } - for col in 0..16 { - let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); - a.swap(col, piv); - inv.swap(col, piv); - let d = a[col][col]; - for j in 0..16 { - a[col][j] /= d; - inv[col][j] /= d; - } - for r in 0..16 { - if r != col { - let f = a[r][col]; - if f != 0.0 { - for j in 0..16 { - a[r][j] -= f * a[col][j]; - inv[r][j] -= f * inv[col][j]; - } - } - } - } - } - inv -} - -// ---------------------------------------------------------------- planes and prediction - -struct Plane { - w: usize, - /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). - max_x: usize, - max_y: usize, - src: Vec, - rec: Vec, +/// A plane being coded: source and reconstruction at superblock-aligned size. +pub(crate) struct Plane { + /// Stride (superblock-aligned width). + pub w: usize, + /// Last valid column / row for intra prediction edges (the 8-aligned mode-info grid). + pub max_x: usize, + pub max_y: usize, + /// Visible size. + pub vis_w: usize, + pub vis_h: usize, + pub src: Vec, + pub rec: Vec, } impl Plane { + #[allow(clippy::too_many_arguments)] fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { let mut src = vec![0u8; w * h]; for y in 0..h { @@ -294,103 +93,141 @@ impl Plane { src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); } } - Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + Plane { w, max_x, max_y, vis_w: vw, vis_h: vh, src, rec: vec![0; w * h] } } } -/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. -fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { - let size = 1usize << log2; - let base = 128i32; - let buf = &p.rec; - let w = p.w; - let mut above = [0i32; 8]; - let mut left = [0i32; 8]; - if have_above { - for i in 0..size { - above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; - } - } else { - above[..size].fill(base - 1); - } - let top_left = if have_above && have_left { - buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 - } else if have_above { - base + 1 - } else { - base - 1 - }; - if have_left { - for i in 0..size { - left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; - } - } else { - left[..size].fill(base + 1); - } - let mut out = [0i32; 64]; - for i in 0..size { - for j in 0..size { - out[i * size + j] = match mode { - V_PRED => above[j], - H_PRED => left[i], - TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), - _ => 0, - }; - } - } - if mode == DC_PRED { - let v = match (have_left, have_above) { - (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), - (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, false) => base, - }; - out[..size * size].fill(v); - } - out +/// Mode info of one 8×8 position. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Mi { + pub inter: bool, + /// Intra mode (0..=9) or inter mode (NEARESTMV..=NEWMV). + pub mode: u8, + pub mv: Mv, + /// Skip as the loop filter sees it (inter blocks without coefficients). + pub skip: bool, + pub size: u8, } -// ---------------------------------------------------------------- the encoder +/// Rate control state (target bitrate mode). +#[derive(Clone, Debug)] +struct RateControl { + /// bits ≈ complexity / step: per frame type, from the last frame of that type. + key_complexity: Option, + inter_complexity: Option, + /// Bits spent minus bits budgeted so far. + debt: f64, + last_inter_q: Option, +} pub struct Vp9Encoder { cfg: EncoderConfig, - qidx: u8, + base_qidx: u8, lossless: bool, + frames: u64, + since_key: u32, + /// The reconstruction of the last frame (the LAST reference): planes at their strides. + reference: Option<[Plane; 3]>, + mvs: Vec, + rc: RateControl, } -/// Per-tile coding state. -struct Tile<'a> { - enc: &'a Vp9Encoder, - bc: BoolEncoder, - planes: &'a mut [Plane; 3], - mi_cols: usize, - mi_rows: usize, - col_start: usize, - above_nz: [Vec; 3], - left_nz: [Vec; 3], - above_part: Vec, - left_part: Vec, - /// y mode of every 8×8 block (for mode contexts). - modes: &'a mut [u8], +fn quality_to_qidx(q: u8) -> u8 { + // 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + ((100 - q.clamp(1, 99) as i32) * 255 / 85).clamp(1, 255) as u8 } impl Vp9Encoder { pub fn new(cfg: EncoderConfig) -> Self { let q = cfg.quality.clamp(1, 100); - let lossless = q >= 100; - // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. - let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; - Vp9Encoder { cfg, qidx, lossless } + let lossless = q >= 100 && cfg.target_kbps.is_none(); + let base_qidx = if lossless { 0 } else { quality_to_qidx(q) }; + Vp9Encoder { + cfg, + base_qidx, + lossless, + frames: 0, + since_key: 0, + reference: None, + mvs: vec![], + rc: RateControl { key_complexity: None, inter_complexity: None, debt: 0.0, last_inter_q: None }, + } } pub fn config(&self) -> &EncoderConfig { &self.cfg } - /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// Encode one frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: /// ((w+1)/2)×((h+1)/2), tightly packed). pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + self.encode_frame(y, u, v, false).data + } + + /// The decoder's reconstruction of the last coded frame (visible area, tightly packed Y, U, + /// V). + pub fn reconstruction(&self) -> Option<[Vec; 3]> { + let r = self.reference.as_ref()?; + Some(std::array::from_fn(|i| { + let p = &r[i]; + (0..p.vis_h).flat_map(|y| p.rec[y * p.w..y * p.w + p.vis_w].iter().copied()).collect() + })) + } + + fn is_key_due(&self, force: bool) -> bool { + force || self.reference.is_none() || self.cfg.keyframe_interval == 1 || (self.cfg.keyframe_interval > 1 && self.since_key >= self.cfg.keyframe_interval) + } + + /// The q index of the next frame. + fn pick_qidx(&mut self, key: bool) -> u8 { + if self.lossless { + return 0; + } + let Some(kbps) = self.cfg.target_kbps else { return self.base_qidx }; + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + let interval = if self.cfg.keyframe_interval == 0 { 300.0 } else { self.cfg.keyframe_interval as f64 }; + // Key frames get a larger share of the group's budget. + let boost = (interval / 6.0).clamp(1.0, 5.0); + let group = per_frame * interval; + let key_share = group * boost / (boost + interval - 1.0); + let inter_share = (group - key_share) / (interval - 1.0).max(1.0); + let mut target = if key { key_share } else { inter_share }; + // Pay back (or spend) the running difference over about a second. + let horizon = self.cfg.frame_rate.max(1.0); + target = (target - self.rc.debt / horizon).max(target * 0.25); + let complexity = if key { self.rc.key_complexity } else { self.rc.inter_complexity.or(self.rc.key_complexity.map(|c| c / (boost * 2.0))) }; + let Some(cx) = complexity else { return self.base_qidx }; + let step = (cx / target).max(4.0); + let mut q = (0..=255u8).min_by_key(|&q| ((AC_QLOOKUP[q as usize] as f64 - step).abs() * 16.0) as i64).unwrap_or(self.base_qidx); + if !key && let Some(last) = self.rc.last_inter_q { + q = q.clamp(last.saturating_sub(24), last.saturating_add(24)); + } + q.max(1) + } + + fn rc_update(&mut self, key: bool, qidx: u8, bytes: usize) { + let Some(kbps) = self.cfg.target_kbps else { return }; + let bits = bytes as f64 * 8.0; + let cx = bits * AC_QLOOKUP[qidx as usize] as f64; + if key { + self.rc.key_complexity = Some(cx); + } else { + self.rc.inter_complexity = Some(match self.rc.inter_complexity { + Some(c) => c * 0.5 + cx * 0.5, + None => cx, + }); + self.rc.last_inter_q = Some(qidx); + } + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + self.rc.debt += bits - per_frame; + } + + /// Encode one frame; `force_key` starts a new key frame. + pub fn encode_frame(&mut self, y: &[u8], u: &[u8], v: &[u8], force_key: bool) -> EncodedFrame { let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let key = self.is_key_due(force_key); + let qidx = self.pick_qidx(key); + let lossless = qidx == 0; let mi_cols = w.div_ceil(8); let mi_rows = h.div_ceil(8); let sb_cols = mi_cols.div_ceil(8); @@ -413,71 +250,77 @@ impl Vp9Encoder { } max_log2 -= 1; let tile_cols = 1usize << min_log2; - let mut modes = vec![0u8; mi_cols * mi_rows]; - let mut tiles = vec![]; + let quant = Quant { dcq: DC_QLOOKUP[qidx as usize], acq: AC_QLOOKUP[qidx as usize], lossless, round_dc: 0.5, round_ac: 0.38 }; + let step = AC_QLOOKUP[qidx as usize] as f64 / 8.0; + let lambda_sse = 0.35 * step * step; + let refs = if key { + None + } else { + self.reference.as_ref().map(|r| std::array::from_fn::<_, 3, _>(|i| mc::RefPlane { data: &r[i].rec, stride: r[i].w, w: r[i].vis_w, h: r[i].vis_h })) + }; + let prev = (!key && self.mvs.len() == mi_cols * mi_rows).then_some(&self.mvs[..]); + let phases = refs.as_ref().map(|r| mc::PhasePlanes::new(&r[0], 48)); + let fp = tile::FrameParams { key, quant, mi_cols, mi_rows, refs, phases, prev_mvs: prev, lambda_sad: lambda_sse.sqrt(), lambda_sse }; + let mut mi = vec![Mi::default(); mi_cols * mi_rows]; + let mut writers = vec![]; for t in 0..tile_cols { let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); - let mut tile = Tile { - enc: self, - bc: BoolEncoder::new(), - planes: &mut planes, - mi_cols, - mi_rows, - col_start: start, - above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], - left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], - above_part: vec![0; mi_cols + 8], - left_part: vec![0; 8], - modes: &mut modes, - }; - tile.bc.write(false, 128); - for r in (0..mi_rows).step_by(8) { - for p in 0..3 { - tile.left_nz[p].fill(0); - } - tile.left_part.fill(0); - for c in (start..end).step_by(8) { - tile.partition(r, c, 12); - } - } - tiles.push(tile.bc.finish()); + let mut tile = tile::Tile::new(&fp, &mut planes, &mut mi, start, end); + tile.code(); + writers.push(tile.w); } - // Compressed header. - let mut ch_enc = BoolEncoder::new(); - ch_enc.write(false, 128); - if !self.lossless { - ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + // Forward probability updates from this frame's counts. + let defaults = Probs::defaults(); + let mut counts = vec![[0u32; 2]; probs::NPROBS]; + for wr in &writers { + wr.count(&mut counts); } - ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) - for _ in 0..3 { - ch_enc.write(false, 252); // skip probabilities unchanged - } - let compressed = ch_enc.finish(); + let wanted = probs::plan_updates(&defaults, &counts, !key); + let (compressed, used) = probs::compressed_header(&defaults, &wanted, lossless, !key); + let tiles: Vec> = writers.iter().map(|wr| wr.emit(&used)).collect(); + drop(fp); + // Loop filter: the level with the least error against the source. + let level = if lossless || !self.cfg.loop_filter { 0 } else { choose_level(&mut planes, &mi, mi_rows, mi_cols, qidx, key) }; + lf::filter_frame(&mut planes, &mi, mi_rows, mi_cols, level); // Uncompressed header. let mut bw = BitWriter::default(); bw.put(2, 2); // frame_marker bw.put(0, 1); // profile low bit bw.put(0, 1); // profile high bit bw.put(0, 1); // show_existing_frame - bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(!key as u32, 1); // frame_type bw.put(1, 1); // show_frame - bw.put(0, 1); // error_resilient_mode - bw.put(0x49, 8); - bw.put(0x83, 8); - bw.put(0x42, 8); - bw.put(2, 3); // color_space CS_BT_709 - bw.put(self.cfg.full_range as u32, 1); - bw.put(w as u32 - 1, 16); - bw.put(h as u32 - 1, 16); - bw.put(0, 1); // render_and_frame_size_different - bw.put(0, 1); // refresh_frame_context - bw.put(1, 1); // frame_parallel_decoding_mode + if key { + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + } else { + bw.put(1, 1); // error_resilient_mode (no reliance on earlier contexts or vectors) + bw.put(0x01, 8); // refresh_frame_flags: slot 0 becomes the new LAST + for _ in 0..3 { + bw.put(0, 3); // ref_frame_idx: slot 0 + bw.put(0, 1); // sign bias + } + bw.put(1, 1); // found_ref: the size of LAST + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // allow_high_precision_mv + bw.put(0, 1); // is_filter_switchable + bw.put(1, 2); // raw_interpolation_filter → EIGHTTAP (regular) + } bw.put(0, 2); // frame_context_idx - bw.put(0, 6); // loop_filter_level + bw.put(level as u32, 6); // loop_filter_level bw.put(0, 3); // sharpness bw.put(0, 1); // mode_ref_delta_enabled - bw.put(self.qidx as u32, 8); + bw.put(qidx as u32, 8); bw.put(0, 1); // delta_q_y_dc bw.put(0, 1); // delta_q_uv_dc bw.put(0, 1); // delta_q_uv_ac @@ -496,10 +339,46 @@ impl Vp9Encoder { } out.extend_from_slice(&t); } - out + // State for the next frame. + self.mvs = mi.iter().map(|m| if m.inter { m.mv } else { Mv::ZERO }).collect(); + self.reference = Some(planes); + self.frames += 1; + self.since_key = if key { 1 } else { self.since_key + 1 }; + self.rc_update(key, qidx, out.len()); + EncodedFrame { data: out, key, q_idx: qidx, filter_level: level } } } +/// Try a few loop filter levels on the luma reconstruction; the one closest to the source +/// (squared error over the visible area) wins (0 when filtering doesn't help). +fn choose_level(planes: &mut [Plane; 3], mi: &[Mi], mi_rows: usize, mi_cols: usize, qidx: u8, key: bool) -> u8 { + let guess = ((qidx as f64 * if key { 0.25 } else { 0.2 }).round() as u8).clamp(1, 63); + let sse = |p: &Plane| -> u64 { + let mut s = 0u64; + for y in 0..p.vis_h { + for x in 0..p.vis_w { + let d = p.src[y * p.w + x] as i64 - p.rec[y * p.w + x] as i64; + s += (d * d) as u64; + } + } + s + }; + let mut best = (sse(&planes[0]), 0u8); + let saved = planes[0].rec.clone(); + for level in [guess / 2, guess, (guess as u16 * 3 / 2).min(63) as u8] { + if level == 0 { + continue; + } + lf::filter_frame(&mut planes[..1], mi, mi_rows, mi_cols, level); + let s = sse(&planes[0]); + planes[0].rec.copy_from_slice(&saved); + if s < best.0 { + best = (s, level); + } + } + best.1 +} + #[derive(Default)] struct BitWriter { out: Vec, @@ -527,254 +406,6 @@ impl BitWriter { } } -const BLOCK_8X8: u8 = 3; -const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; -const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; - -impl Tile<'_> { - fn partition(&mut self, r: usize, c: usize, bsize: u8) { - if r >= self.mi_rows || c >= self.mi_cols { - return; - } - let num8 = NUM_8X8[bsize as usize]; - let half = num8 >> 1; - let has_rows = (r + half) < self.mi_rows; - let has_cols = (c + half) < self.mi_cols; - let bsl = MI_WIDTH_LOG2[bsize as usize]; - let boffset = 3 - bsl; - let mut above = 0u8; - let mut left = 0u8; - for i in 0..num8 { - above |= self.above_part[c + i]; - left |= self.left_part[(r & 7) + i]; - } - let above = (above >> boffset) & 1; - let left = (left >> boffset) & 1; - let ctx = bsl * 4 + left as usize * 2 + above as usize; - let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; - if bsize == BLOCK_8X8 { - // PARTITION_NONE. - self.bc.write(false, probs[0]); - self.block(r, c); - self.above_part[c] = 15 >> 1; - self.left_part[r & 7] = 15 >> 1; - return; - } - // PARTITION_SPLIT. - if has_rows && has_cols { - self.bc.tree(&PARTITION_TREE, probs, 3); - } else if has_cols { - self.bc.write(true, probs[1]); - } else if has_rows { - self.bc.write(true, probs[2]); - } - let sub = bsize - 3; - self.partition(r, c, sub); - self.partition(r, c + half, sub); - self.partition(r + half, c, sub); - self.partition(r + half, c + half, sub); - } - - fn block(&mut self, r: usize, c: usize) { - let avail_u = r > 0; - let avail_l = c > self.col_start; - // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. - let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); - let uv_mode = { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - }; - // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). - self.bc.write(false, DEFAULT_SKIP_PROB[0]); - let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; - let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; - let o = (above_mode as usize * 10 + left_mode as usize) * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); - let o = y_mode as usize * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); - self.modes[r * self.mi_cols + c] = y_mode; - // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. - for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { - let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; - self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); - } - for p in 1..3 { - self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); - } - } - - fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { - let p = &self.planes[plane]; - let pred = predict(p, x, y, log2, mode, have_left, have_above); - let size = 1 << log2; - let mut s = 0i64; - for i in 0..size { - for j in 0..size { - s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; - } - } - s - } - - fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost = self.cost(plane, x, y, log2, m, have_left, have_above); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - } - - #[allow(clippy::too_many_arguments)] - fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { - let ss = (plane > 0) as usize; - let max_x = (self.mi_cols * 8) >> ss; - let max_y = (self.mi_rows * 8) >> ss; - let (x4, y4) = (x >> 2, y >> 2); - let ly = y4 & (15 >> ss); - if x >= max_x || y >= max_y { - self.above_nz[plane][x4] = 0; - self.left_nz[plane][ly] = 0; - return; - } - let p = &self.planes[plane]; - let pred = predict(p, x, y, 2, mode, have_left, have_above); - let mut res = [0i32; 16]; - for i in 0..4 { - for j in 0..4 { - res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; - } - } - let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); - // Quantised levels (raster order). - let levels: [i32; 16] = if self.enc.lossless { - forward_wht(&res) - } else { - let f = &forward_matrices()[tx_type as usize]; - let mut lv = [0i32; 16]; - for k in 0..16 { - let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); - let qs = if k == 0 { dcq } else { acq } as f64; - let a = coef.abs() / qs; - // Rounding with a small dead zone for AC coefficients. - let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; - lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; - } - lv - }; - let sc = scan(tx_type); - let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); - // Tokens. - let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; - self.tokens(plane, tx_type, &levels, eob, ctx0); - // Reconstruct exactly as the decoder does. - let mut deq = [0i32; 16]; - for k in 0..16 { - deq[k] = levels[k] * if k == 0 { dcq } else { acq }; - } - let resid = if eob == 0 { - [0i32; 16] - } else if self.enc.lossless { - inverse_wht(&deq) - } else { - inverse4(&deq, tx_type) - }; - let p = &mut self.planes[plane]; - for i in 0..4 { - for j in 0..4 { - p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; - } - } - let nz = (eob > 0) as u8; - self.above_nz[plane][x4] = nz; - self.left_nz[plane][ly] = nz; - } - - fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { - let sc = scan(tx_type); - let nb = neighbors(tx_type); - let ptype = (plane > 0) as usize; - // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. - let base = (ptype * 2) * 6 * 6 * 3; - let mut tc = [0u8; 16]; - let mut check_eob = true; - let mut c = 0; - while c < 16 { - let pos = sc[c] as usize; - let band = COEFBAND_4X4[c] as usize; - let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; - let o = base + (band * 6 + ctx) * 3; - let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; - if check_eob { - let more = c < eob; - self.bc.write(more, p[0]); - if !more { - break; - } - } - let v = levels[pos].unsigned_abs(); - if v == 0 { - self.bc.write(false, p[1]); - tc[pos] = 0; - check_eob = false; - c += 1; - continue; - } - self.bc.write(true, p[1]); - check_eob = true; - let token = match v { - 1 => 1, - 2 => 2, - 3 => 3, - 4 => 4, - 5..=6 => 5, - 7..=10 => 6, - 11..=18 => 7, - 19..=34 => 8, - 35..=66 => 9, - _ => 10, - }; - if token == 1 { - self.bc.write(false, p[2]); - } else { - self.bc.write(true, p[2]); - let pp = pareto(p[2]); - let path: &[(usize, bool)] = match token { - 2 => &[(0, false), (1, false)], - 3 => &[(0, false), (1, true), (2, false)], - 4 => &[(0, false), (1, true), (2, true)], - 5 => &[(0, true), (3, false), (4, false)], - 6 => &[(0, true), (3, false), (4, true)], - 7 => &[(0, true), (3, true), (5, false), (6, false)], - 8 => &[(0, true), (3, true), (5, false), (6, true)], - 9 => &[(0, true), (3, true), (5, true), (7, false)], - _ => &[(0, true), (3, true), (5, true), (7, true)], - }; - for &(n, b) in path { - self.bc.write(b, pp[n]); - } - let (cat, num_extra, base_v) = EXTRA_BITS[token]; - let extra = (v - base_v as u32).min((1 << num_extra) - 1); - let cp = CAT_PROBS[cat as usize]; - for e in 0..num_extra as usize { - self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); - } - } - tc[pos] = ENERGY_CLASS[token]; - self.bc.write(levels[pos] < 0, 128); - c += 1; - } - } -} - // ---------------------------------------------------------------- colour conversion /// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). @@ -822,36 +453,3 @@ pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); (y, vec![128; cw * ch], vec![128; cw * ch]) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn wht_is_exactly_invertible() { - let mut s = 7u32; - for _ in 0..2000 { - let mut r = [0i32; 16]; - for v in &mut r { - s = s.wrapping_mul(1664525).wrapping_add(1013904223); - *v = (s >> 16) as i32 % 511 - 255; - } - let lv = forward_wht(&r); - let deq = lv.map(|l| l * 4); - assert_eq!(inverse_wht(&deq), r); - } - } - - #[test] - fn forward_transforms_invert_the_decoder() { - for ty in 0..4u8 { - let f = &forward_matrices()[ty as usize]; - let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); - let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); - let back = inverse4(&c, ty); - for i in 0..16 { - assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); - } - } - } -} diff --git a/crates/vp9enc/src/mc.rs b/crates/vp9enc/src/mc.rs new file mode 100644 index 00000000..779a1ea5 --- /dev/null +++ b/crates/vp9enc/src/mc.rs @@ -0,0 +1,207 @@ +//! Inter prediction exactly as the decoder forms it: motion vector clamping (spec 8.5.2.2) and +//! the unscaled block inter prediction process (8.5.2.4) with the regular 8-tap filters and +//! reference samples clamped to the reference frame's visible area. + +use crate::tables::SUBPEL_REGULAR; + +/// A motion vector in 1/8 luma samples. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mv { + pub row: i32, + pub col: i32, +} + +impl Mv { + pub const ZERO: Mv = Mv { row: 0, col: 0 }; + pub fn new(row: i32, col: i32) -> Mv { + Mv { row, col } + } + pub fn minus(self, o: Mv) -> Mv { + Mv::new(self.row - o.row, self.col - o.col) + } +} + +/// A reference plane: `w`×`h` visible samples at `stride`. +pub struct RefPlane<'a> { + pub data: &'a [u8], + pub stride: usize, + pub w: usize, + pub h: usize, +} + +/// The block's motion vector for `plane` in 1/16 samples of that plane, clamped as the +/// decoder clamps it. `n8`: block size in 8×8 units. +pub fn clamp_for_plane(mv: Mv, r: usize, c: usize, n8: usize, mi_rows: usize, mi_cols: usize, chroma: bool) -> (i32, i32) { + let s = chroma as i32; + let (r, c, b) = (r as i32, c as i32, n8 as i32); + let to_top = (-(r * 8 * 16)) >> s; + let to_bottom = (((mi_rows as i32 - b - r) * 8) * 16) >> s; + let to_left = (-(c * 8 * 16)) >> s; + let to_right = (((mi_cols as i32 - b - c) * 8) * 16) >> s; + let spel = (4 + ((b * 8) >> s)) << 4; + let row = ((2 * mv.row) >> s).clamp(to_top - spel, (to_bottom + spel - 16).max(to_top - spel)); + let col = ((2 * mv.col) >> s).clamp(to_left - spel, (to_right + spel - 16).max(to_left - spel)); + (row, col) +} + +/// Predict a `w`×`h` block whose top-left sample is at (`x16`, `y16`) (1/16 samples) into `out` +/// (stride `w`). +pub fn predict(r: &RefPlane, x16: i32, y16: i32, w: usize, h: usize, out: &mut [u8]) { + let (x0, y0) = (x16 >> 4, y16 >> 4); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + let (lx, ly) = (r.w as i32 - 1, r.h as i32 - 1); + let inside = x0 - 3 >= 0 && y0 - 3 >= 0 && x0 + w as i32 + 4 <= lx && y0 + h as i32 + 4 <= ly; + let at = |x: i32, y: i32| -> i32 { + if inside { r.data[y as usize * r.stride + x as usize] as i32 } else { r.data[y.clamp(0, ly) as usize * r.stride + x.clamp(0, lx) as usize] as i32 } + }; + let fh = &SUBPEL_REGULAR[fx]; + let fv = &SUBPEL_REGULAR[fy]; + if fx == 0 && fy == 0 { + for i in 0..h { + for j in 0..w { + out[i * w + j] = at(x0 + j as i32, y0 + i as i32) as u8; + } + } + return; + } + // Horizontal pass over h + 7 rows (from y0 - 3), then vertical. + let mut tmp = [0u8; 23 * 16]; + for i in 0..h + 7 { + let y = y0 - 3 + i as i32; + for j in 0..w { + let x = x0 + j as i32 - 3; + let mut acc = 64; + for t in 0..8 { + acc += fh[t] * at(x + t as i32, y); + } + tmp[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } + for i in 0..h { + for j in 0..w { + let mut acc = 64; + for t in 0..8 { + acc += fv[t] * tmp[(i + t) * w + j] as i32; + } + out[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } +} + +/// The reference luma pre-interpolated at the 16 quarter-sample phases the encoder uses (motion +/// vectors are even in 1/8 units: fractions 0, 4, 8, 12 in 1/16), over the visible area plus a +/// border. Each phase plane holds exactly what [`predict`] returns at that phase (the decoder's +/// separable filter with edge clamping), so motion search reads predictions directly. +pub struct PhasePlanes { + border: usize, + stride: usize, + rows: usize, + planes: Vec>, +} + +impl PhasePlanes { + pub fn new(r: &RefPlane, border: usize) -> PhasePlanes { + const M: usize = 4; + let (pw, ph) = (r.w + 2 * border, r.h + 2 * border); + // Edge-replicated source with M more samples on every side for the filter taps. + let rs = pw + 2 * M; + let rh = ph + 2 * M; + let off = (border + M) as i32; + let mut raw = vec![0u8; rs * rh]; + for y in 0..rh { + let sy = (y as i32 - off).clamp(0, r.h as i32 - 1) as usize; + let src = &r.data[sy * r.stride..sy * r.stride + r.w]; + for x in 0..rs { + raw[y * rs + x] = src[(x as i32 - off).clamp(0, r.w as i32 - 1) as usize]; + } + } + // Horizontal phases over every raw row (inner columns). + let hpass = |fx: usize| -> Vec { + let f = &SUBPEL_REGULAR[fx]; + let mut out = vec![0u8; pw * rh]; + for y in 0..rh { + let row = &raw[y * rs..y * rs + rs]; + for x in 0..pw { + let s = &row[x + M - 3..x + M + 5]; + let acc: i32 = 64 + (0..8).map(|t| f[t] * s[t] as i32).sum::(); + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + out + }; + let unfiltered: Vec = (0..rh).flat_map(|y| raw[y * rs + M..y * rs + M + pw].iter().copied()).collect(); + let h: Vec> = (0..4).map(|k| if k == 0 { unfiltered.clone() } else { hpass(k * 4) }).collect(); + let mut planes = Vec::with_capacity(16); + for fy in 0..4 { + for hp in &h { + if fy == 0 { + planes.push(hp[M * pw..(M + ph) * pw].to_vec()); + continue; + } + let f = &SUBPEL_REGULAR[fy * 4]; + let mut out = vec![0u8; pw * ph]; + for y in 0..ph { + for x in 0..pw { + let mut acc = 64; + for t in 0..8 { + acc += f[t] * hp[(y + M + t - 3) * pw + x] as i32; + } + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + planes.push(out); + } + } + PhasePlanes { border, stride: pw, rows: ph, planes } + } + + /// The prediction of the `n`×`n` block at (`x16`, `y16`) (fractions multiples of 4) as rows of + /// a phase plane: (plane, offset of the first sample, stride). `None` beyond the border. + pub fn block(&self, x16: i32, y16: i32, n: usize) -> Option<(&[u8], usize, usize)> { + let (x0, y0) = ((x16 >> 4) + self.border as i32, (y16 >> 4) + self.border as i32); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + if fx % 4 != 0 || fy % 4 != 0 || x0 < 0 || y0 < 0 || x0 as usize + n > self.stride || y0 as usize + n > self.rows { + return None; + } + Some((&self.planes[(fy / 4) * 4 + fx / 4], y0 as usize * self.stride + x0 as usize, self.stride)) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn phase_planes_match_predict() { + let (w, h) = (37usize, 29usize); + let data: Vec = (0..w * h).map(|i| ((i * 7919) % 251) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let pp = PhasePlanes::new(&r, 24); + let mut out = [0u8; 64]; + for y16 in (-20 * 16..(h as i32 + 10) * 16).step_by(4 * 7) { + for x16 in (-20 * 16..(w as i32 + 10) * 16).step_by(4 * 5) { + let Some((p, o, s)) = pp.block(x16, y16, 8) else { continue }; + predict(&r, x16, y16, 8, 8, &mut out); + for i in 0..8 { + assert_eq!(&p[o + i * s..o + i * s + 8], &out[i * 8..i * 8 + 8], "({x16}, {y16}) row {i}"); + } + } + } + } + + use super::*; + + #[test] + fn whole_sample_and_linear_ramps() { + let (w, h) = (40usize, 30usize); + let data: Vec = (0..w * h).map(|i| ((i % w) * 4 + (i / w)) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let mut out = [0u8; 64]; + predict(&r, 10 << 4, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10]); + // Half a sample right on a linear ramp: exact midpoint. + predict(&r, (10 << 4) + 8, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10] + 2); + // Off the left edge: clamped to column 0. + predict(&r, -(20 << 4), 0, 4, 4, &mut out); + assert_eq!(out[0], data[0]); + } +} diff --git a/crates/vp9enc/src/probs.rs b/crates/vp9enc/src/probs.rs new file mode 100644 index 00000000..8f55bbba --- /dev/null +++ b/crates/vp9enc/src/probs.rs @@ -0,0 +1,426 @@ +//! Probabilities: the frame's adaptable probability table (one flat array laid out in the order +//! the compressed header updates them), a symbol recorder that codes every boolean decision +//! against either a fixed probability or an entry of that table, bit-cost estimates, and the +//! per-frame forward updates (spec 6.3: `diff_update_prob` with `decode_term_subexp` / +//! `inv_remap_prob`, and `update_mv_prob`). +//! +//! Tiles are first recorded, then the encoder counts how often each table entry coded a 0 or a +//! 1, picks updates that pay for themselves, writes them in the compressed header and finally +//! entropy codes the recorded symbols with the updated probabilities. + +use crate::bool::BoolEncoder; +use crate::tables::*; + +/// Coefficient probabilities `[ptype][ref][band][ctx][3]` for 4×4 transforms (band 0 uses +/// contexts 0..3 only). +pub const COEF: usize = 0; +pub const SKIP: usize = 432; +pub const INTER_MODE: usize = SKIP + 3; +pub const IS_INTER: usize = INTER_MODE + 21; +pub const SINGLE_REF: usize = IS_INTER + 4; +pub const PARTITION: usize = SINGLE_REF + 10; +pub const MV_JOINT: usize = PARTITION + 48; +pub const MV_SIGN: usize = MV_JOINT + 3; +pub const MV_CLASS: usize = MV_SIGN + 2; +pub const MV_CLASS0_BIT: usize = MV_CLASS + 20; +pub const MV_BITS: usize = MV_CLASS0_BIT + 2; +pub const MV_CLASS0_FR: usize = MV_BITS + 20; +pub const MV_FR: usize = MV_CLASS0_FR + 12; +pub const NPROBS: usize = MV_FR + 6; + +/// Index of the first of the three coefficient probabilities of a context. +pub fn coef_index(ptype: usize, inter: bool, band: usize, ctx: usize) -> usize { + COEF + (((ptype * 2 + inter as usize) * 6 + band) * 6 + ctx) * 3 +} + +#[derive(Clone)] +pub struct Probs(pub [u8; NPROBS]); + +impl Probs { + /// The default probabilities (setup_past_independence / key frames). + pub fn defaults() -> Probs { + let mut p = [128u8; NPROBS]; + p[COEF..COEF + 432].copy_from_slice(&DEFAULT_COEF_PROBS); + p[SKIP..SKIP + 3].copy_from_slice(&DEFAULT_SKIP_PROB); + p[INTER_MODE..INTER_MODE + 21].copy_from_slice(&DEFAULT_INTER_MODE_PROBS); + p[IS_INTER..IS_INTER + 4].copy_from_slice(&DEFAULT_IS_INTER_PROB); + p[SINGLE_REF..SINGLE_REF + 10].copy_from_slice(&DEFAULT_SINGLE_REF_PROB); + p[PARTITION..PARTITION + 48].copy_from_slice(&DEFAULT_PARTITION_PROBS); + p[MV_JOINT..MV_JOINT + 3].copy_from_slice(&DEFAULT_MV_JOINT_PROBS); + p[MV_SIGN..MV_SIGN + 2].copy_from_slice(&DEFAULT_MV_SIGN_PROB); + p[MV_CLASS..MV_CLASS + 20].copy_from_slice(&DEFAULT_MV_CLASS_PROBS); + p[MV_CLASS0_BIT..MV_CLASS0_BIT + 2].copy_from_slice(&DEFAULT_MV_CLASS0_BIT_PROB); + p[MV_BITS..MV_BITS + 20].copy_from_slice(&DEFAULT_MV_BITS_PROB); + p[MV_CLASS0_FR..MV_CLASS0_FR + 12].copy_from_slice(&DEFAULT_MV_CLASS0_FR_PROBS); + p[MV_FR..MV_FR + 6].copy_from_slice(&DEFAULT_MV_FR_PROBS); + Probs(p) + } +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +pub fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +/// The probability a recorded symbol is coded with. +#[derive(Clone, Copy, Debug)] +pub enum P { + Fixed(u8), + /// An entry of the frame's [`Probs`]. + Ctx(u16), + /// Node `n` of the Pareto token tree derived from the coefficient probability at the index. + Pareto(u16, u8), +} + +#[derive(Clone, Copy)] +struct Sym { + bit: bool, + p: P, +} + +/// Records the boolean decisions of a tile. +#[derive(Default)] +pub struct Writer { + syms: Vec, +} + +impl Writer { + #[inline] + pub fn put(&mut self, bit: bool, p: P) { + self.syms.push(Sym { bit, p }); + } + #[inline] + pub fn fixed(&mut self, bit: bool, prob: u8) { + self.put(bit, P::Fixed(prob)); + } + #[inline] + pub fn ctx(&mut self, bit: bool, idx: usize) { + self.put(bit, P::Ctx(idx as u16)); + } + /// Write `value` with `tree` (leaves stored negated); node `i` uses `prob(i)`. + pub fn tree(&mut self, tree: &[i8], value: u8, prob: impl Fn(usize) -> P) { + for (node, bit) in tree_path(tree, value) { + self.put(bit, prob(node)); + } + } + + /// Add the 0/1 counts of every table entry this tile used. + pub fn count(&self, counts: &mut [[u32; 2]]) { + for s in &self.syms { + if let P::Ctx(i) = s.p { + counts[i as usize][s.bit as usize] += 1; + } + } + } + + /// Entropy code the recorded symbols. + pub fn emit(&self, probs: &Probs) -> Vec { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); // marker bit + for s in &self.syms { + let p = match s.p { + P::Fixed(p) => p, + P::Ctx(i) => probs.0[i as usize], + P::Pareto(i, n) => pareto(probs.0[i as usize])[n as usize], + }; + bc.write(s.bit, p); + } + bc.finish() + } +} + +/// The (node, bit) path to `value` in a tree (node = tree index / 2). +pub fn tree_path(tree: &[i8], value: u8) -> impl Iterator { + struct Path { + steps: [(u8, bool); 16], + len: usize, + } + fn walk(tree: &[i8], node: usize, value: u8, out: &mut Path) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.steps[out.len] = ((node >> 1) as u8, b == 1); + out.len += 1; + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if walk(tree, t as usize, value, out) { + return true; + } + out.len -= 1; + } + false + } + let mut p = Path { steps: [(0, false); 16], len: 0 }; + walk(tree, 0, value, &mut p); + let Path { steps, len } = p; + steps.into_iter().take(len).map(|(n, b)| (n as usize, b)) +} + +// ---------------------------------------------------------------- costs + +/// Cost in 1/256 bit of coding a 0 (`[p][0]`) or a 1 (`[p][1]`) with probability `p`. +pub fn cost_table() -> &'static [[u32; 2]; 256] { + use std::sync::OnceLock; + static T: OnceLock<[[u32; 2]; 256]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [[0u32; 2]; 256]; + for (p, e) in t.iter_mut().enumerate() { + let q = (p.max(1) as f64) / 256.0; + e[0] = (-q.log2() * 256.0).round() as u32; + e[1] = (-(1.0 - q).log2() * 256.0).round() as u32; + } + t + }) +} + +#[inline] +pub fn cost(bit: bool, p: u8) -> u32 { + cost_table()[p as usize][bit as usize] +} + +/// Cost of `value` in a tree with probabilities `probs` (node i → probs[i]). +pub fn tree_cost(tree: &[i8], probs: &[u8], value: u8) -> u32 { + tree_path(tree, value).map(|(n, b)| cost(b, probs[n])).sum() +} + +// ---------------------------------------------------------------- forward updates + +fn inv_recenter_nonneg(v: i32, m: i32) -> i32 { + if v > 2 * m { + v + } else if v & 1 != 0 { + m - ((v + 1) >> 1) + } else { + m + (v >> 1) + } +} + +/// `inv_remap_prob` (6.3.5). +fn inv_remap_prob(delta: usize, prob: u8) -> u8 { + let v = inv_map_table()[delta.min(254)] as i32; + let m = prob as i32 - 1; + let r = if (m << 1) <= 255 { 1 + inv_recenter_nonneg(v, m) } else { 255 - inv_recenter_nonneg(v, 255 - 1 - m) }; + r.clamp(1, 255) as u8 +} + +/// The `decode_term_subexp` value that moves `old` to `new`. +fn delta_for(old: u8, new: u8) -> Option { + (0..255).find(|&d| inv_remap_prob(d, old) == new) +} + +/// Bits of `decode_term_subexp` for `d`. +fn subexp_bits(d: usize) -> u32 { + match d { + 0..16 => 5, + 16..32 => 6, + 32..64 => 8, + 64..129 => 10, + _ => 11, + } +} + +fn write_subexp(bc: &mut BoolEncoder, d: usize) { + let d = d as u32; + if d < 16 { + bc.literal(0, 1); + bc.literal(d, 4); + } else if d < 32 { + bc.literal(2, 2); + bc.literal(d - 16, 4); + } else if d < 64 { + bc.literal(6, 3); + bc.literal(d - 32, 5); + } else { + bc.literal(7, 3); + if d < 129 { + bc.literal(d - 64, 7); + } else { + let v = (d + 1) >> 1; + bc.literal(v, 7); + bc.literal((d + 1) & 1, 1); + } + } +} + +fn counts_cost(c: [u32; 2], p: u8) -> u64 { + c[0] as u64 * cost(false, p) as u64 + c[1] as u64 * cost(true, p) as u64 +} + +/// Whether table entry `i` is an MV probability (7-bit `update_mv_prob` rather than +/// `diff_update_prob`). +fn is_mv(i: usize) -> bool { + i >= MV_JOINT +} + +/// Choose updated probabilities: an entry changes when the bits saved on this frame's symbols +/// exceed the cost of signalling the change. `inter`: the inter-frame tables are updatable. +pub fn plan_updates(old: &Probs, counts: &[[u32; 2]], inter: bool) -> Probs { + let mut new = old.clone(); + let flag = cost(true, 252) as i64 - cost(false, 252) as i64; + let limit = if inter { NPROBS } else { INTER_MODE }; + for i in 0..limit { + let c = counts[i]; + let n = c[0] + c[1]; + if n == 0 { + continue; + } + let o = old.0[i]; + let ideal = ((c[0] as f64 * 256.0 / n as f64).round() as i32).clamp(1, 255) as u8; + let cands: Vec = if is_mv(i) { vec![ideal | 1, ideal.saturating_sub(1).max(1) | 1] } else { vec![ideal] }; + let base = counts_cost(c, o) as i64; + let mut best = (0i64, o); + for p in cands { + if p == o { + continue; + } + let signal = if is_mv(i) { + 7 * 256 + } else { + match delta_for(o, p) { + Some(d) => subexp_bits(d) as i64 * 256, + None => continue, + } + }; + let gain = base - counts_cost(c, p) as i64 - signal - flag; + if gain > best.0 { + best = (gain, p); + } + } + new.0[i] = best.1; + } + new +} + +/// `diff_update_prob` for one entry. +fn diff_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let (o, n) = (old.0[i], new.0[i]); + match (o != n).then(|| delta_for(o, n)).flatten() { + Some(d) => { + bc.write(true, 252); + write_subexp(bc, d); + } + None => bc.write(false, 252), + } +} + +/// `update_mv_prob` for one entry. +fn mv_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let n = new.0[i]; + if n != old.0[i] && n & 1 == 1 { + bc.write(true, 252); + bc.literal(n as u32 >> 1, 7); + } else { + bc.write(false, 252); + } +} + +/// The compressed header (6.3) of a frame coded with ONLY_4X4 transforms, single LAST +/// references and a fixed interpolation filter, moving the probabilities from `old` to `new`. +/// Entries `new` cannot signal are left at `old` (and must be coded with `old`): the caller +/// codes the tiles with the probabilities this returns. +pub fn compressed_header(old: &Probs, new: &Probs, lossless: bool, inter: bool) -> (Vec, Probs) { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); + if !lossless { + bc.literal(0, 2); // tx_mode = ONLY_4X4 + } + // read_coef_probs: TX_4X4 only. + let coef_changed = (COEF..COEF + 432).any(|i| new.0[i] != old.0[i]); + bc.literal(coef_changed as u32, 1); + if coef_changed { + for i in 0..2 { + for j in 0..2 { + for k in 0..6 { + for l in 0..if k == 0 { 3 } else { 6 } { + for m in 0..3 { + diff_update(&mut bc, old, new, coef_index(i, j == 1, k, l) + m); + } + } + } + } + } + } + for i in 0..3 { + diff_update(&mut bc, old, new, SKIP + i); + } + if inter { + for i in 0..21 { + diff_update(&mut bc, old, new, INTER_MODE + i); + } + for i in 0..4 { + diff_update(&mut bc, old, new, IS_INTER + i); + } + // reference_mode is SINGLE_REFERENCE (no compound: equal sign biases). + for i in 0..10 { + diff_update(&mut bc, old, new, SINGLE_REF + i); + } + for _ in 0..36 { + bc.write(false, 252); // y mode probabilities (no intra blocks in inter frames) + } + for i in 0..48 { + diff_update(&mut bc, old, new, PARTITION + i); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_JOINT + j); + } + for i in 0..2 { + mv_update(&mut bc, old, new, MV_SIGN + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_CLASS + i * 10 + j); + } + mv_update(&mut bc, old, new, MV_CLASS0_BIT + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_BITS + i * 10 + j); + } + } + for i in 0..2 { + for j in 0..6 { + mv_update(&mut bc, old, new, MV_CLASS0_FR + i * 6 + j); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_FR + i * 3 + j); + } + } + } + // What the decoder ends up with. + let mut used = old.clone(); + for i in 0..NPROBS { + let ok = if is_mv(i) { new.0[i] & 1 == 1 } else { delta_for(old.0[i], new.0[i]).is_some() }; + if ok && (inter || i < INTER_MODE) { + used.0[i] = new.0[i]; + } + } + (bc.finish(), used) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remap_reaches_every_probability() { + for old in 1..=255u8 { + for new in 1..=255u8 { + if new != old { + assert!(delta_for(old, new).is_some(), "{old} -> {new}"); + } + } + } + } + + #[test] + fn inv_map_table_shape() { + let t = inv_map_table(); + assert_eq!(&t[..4], &[7, 20, 33, 46]); + assert_eq!(t[19], 254); + assert_eq!(&t[20..27], &[1, 2, 3, 4, 5, 6, 8]); + assert_eq!(&t[252..], &[252, 253, 253]); + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs index dc8027a6..ffb26641 100644 --- a/crates/vp9enc/src/tables.rs +++ b/crates/vp9enc/src/tables.rs @@ -128,3 +128,76 @@ pub const DEFAULT_COEF_PROBS: [u8; 432] = [ 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, ]; + +// ---------------------------------------------------------------- inter frames (spec section 10.5) + +/// `inv_map_table` (used by `inv_remap_prob`): the 20 values 7 + 13k first, then every other value +/// 1..=253 in increasing order, then 253 again. +pub fn inv_map_table() -> &'static [u8; 255] { + use std::sync::OnceLock; + static T: OnceLock<[u8; 255]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [253u8; 255]; + let mut n = 0; + for k in 0..20 { + t[n] = 7 + 13 * k; + n += 1; + } + for v in 1..=253u8 { + if v < 7 || (v - 7) % 13 != 0 { + t[n] = v; + n += 1; + } + } + t + }) +} + +/// The regular 8-tap sub-sample interpolation filters (`subpel_filters[EIGHTTAP]`), 16 positions. +pub const SUBPEL_REGULAR: [[i32; 8]; 16] = [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 1, -5, 126, 8, -3, 1, 0], + [-1, 3, -10, 122, 18, -6, 2, 0], + [-1, 4, -13, 118, 27, -9, 3, -1], + [-1, 4, -16, 112, 37, -11, 4, -1], + [-1, 5, -18, 105, 48, -14, 4, -1], + [-1, 5, -19, 97, 58, -16, 5, -1], + [-1, 6, -19, 88, 68, -18, 5, -1], + [-1, 6, -19, 78, 78, -19, 6, -1], + [-1, 5, -18, 68, 88, -19, 6, -1], + [-1, 5, -16, 58, 97, -19, 5, -1], + [-1, 4, -14, 48, 105, -18, 5, -1], + [-1, 4, -11, 37, 112, -16, 4, -1], + [-1, 3, -9, 27, 118, -13, 4, -1], + [0, 2, -6, 18, 122, -10, 3, -1], + [0, 1, -3, 8, 126, -5, 1, 0], +]; + +/// `mv_ref_blocks` (row, column offsets) for BLOCK_8X8 and BLOCK_16X16. +pub const MV_REF_8X8: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, -1), (-2, 0), (0, -2), (-2, -1), (-1, -2), (-2, -2)]; +pub const MV_REF_16X16: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, 1), (1, -1), (-1, -1), (-3, 0), (0, -3), (-3, -3)]; + +/// `mode_2_counter` indexed by y mode (intra modes 0..=9, then NEARESTMV, NEARMV, ZEROMV, NEWMV). +pub const MODE_2_COUNTER: [u8; 14] = [9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 3, 1]; + +/// `counter_to_context`; INVALID_CASE entries are 9. +pub const COUNTER_TO_CONTEXT: [u8; 19] = [2, 3, 4, 1, 3, 9, 0, 9, 9, 5, 5, 9, 5, 9, 9, 9, 9, 9, 6]; + +pub const DEFAULT_PARTITION_PROBS: [u8; 48] = [ + 199, 122, 141, 147, 63, 159, 148, 133, 118, 121, 104, 114, 174, 73, 87, 92, 41, 83, 82, 99, 50, 53, 39, 39, 177, 58, 59, 68, 26, 63, 52, 79, 25, 17, 14, + 12, 222, 34, 30, 72, 16, 44, 58, 32, 12, 10, 7, 6, +]; + +pub const DEFAULT_IS_INTER_PROB: [u8; 4] = [9, 102, 187, 225]; + +pub const DEFAULT_SINGLE_REF_PROB: [u8; 10] = [33, 16, 77, 74, 142, 142, 172, 170, 238, 247]; + +pub const DEFAULT_INTER_MODE_PROBS: [u8; 21] = [2, 173, 34, 7, 145, 85, 7, 166, 63, 7, 94, 66, 8, 64, 46, 17, 81, 31, 25, 29, 30]; + +pub const DEFAULT_MV_JOINT_PROBS: [u8; 3] = [32, 64, 96]; +pub const DEFAULT_MV_SIGN_PROB: [u8; 2] = [128, 128]; +pub const DEFAULT_MV_CLASS_PROBS: [u8; 20] = [224, 144, 192, 168, 192, 176, 192, 198, 198, 245, 216, 128, 176, 160, 176, 176, 192, 198, 198, 208]; +pub const DEFAULT_MV_CLASS0_BIT_PROB: [u8; 2] = [216, 208]; +pub const DEFAULT_MV_BITS_PROB: [u8; 20] = [136, 140, 148, 160, 176, 192, 224, 234, 234, 240, 136, 140, 148, 160, 176, 192, 224, 234, 234, 240]; +pub const DEFAULT_MV_CLASS0_FR_PROBS: [u8; 12] = [128, 128, 64, 96, 112, 64, 128, 128, 64, 96, 112, 64]; +pub const DEFAULT_MV_FR_PROBS: [u8; 6] = [64, 96, 64, 64, 96, 64]; diff --git a/crates/vp9enc/src/tile.rs b/crates/vp9enc/src/tile.rs new file mode 100644 index 00000000..fe3a1a49 --- /dev/null +++ b/crates/vp9enc/src/tile.rs @@ -0,0 +1,894 @@ +//! Tile coding: partitions, intra blocks (key frames) and inter blocks (motion search, mode +//! decision, motion vector prediction, residual), recorded as symbols (see [`crate::probs`]). +//! +//! Mode info, motion vector candidate lists and contexts follow the decoding process of the +//! VP9 specification (6.4, 6.4.x `find_mv_refs` / `find_best_ref_mvs`) for the subset of the +//! syntax this encoder emits: 8×8 and 16×16 blocks, single LAST references, no +//! `UsePrevFrameMvs` (inter frames are error resilient), no high-precision vectors. + +use crate::mc::{Mv, PhasePlanes, RefPlane, clamp_for_plane, predict}; +use crate::probs::{self, P, Writer, cost, tree_cost}; +use crate::tables::*; +use crate::transform::{ADST_ADST, ADST_DCT, DCT_ADST, DCT_DCT, Quant}; +use crate::{BLOCK_8X8, BLOCK_16X16, Mi, Plane}; + +pub const DC_PRED: u8 = 0; +pub const V_PRED: u8 = 1; +pub const H_PRED: u8 = 2; +pub const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; +pub const NEARESTMV: u8 = 10; +pub const NEARMV: u8 = 11; +pub const ZEROMV: u8 = 12; +pub const NEWMV: u8 = 13; + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; +/// Leaves are `mode - NEARESTMV`. +const INTER_MODE_TREE: [i8; 6] = [-2, 2, 0, 4, -1, -3]; +const MV_JOINT_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const MV_CLASS_TREE: [i8; 20] = [0, 2, -1, 4, 6, 8, -2, -3, 10, 12, -4, -5, -6, 14, 16, 18, -7, -8, -9, -10]; +const MV_FR_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// mode2txfm_map for the intra modes used here (inter blocks use DCT_DCT). +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +fn eob_of(levels: &[i32; 16], tx_type: u8) -> usize { + let sc = scan(tx_type); + (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1) +} + +/// Rough coefficient bit cost (1/256 bit) for rate-distortion decisions. +fn levels_cost(levels: &[i32; 16], eob: usize) -> u32 { + if eob == 0 { + return 0; + } + let mut bits = 256; + for &l in levels { + let a = l.unsigned_abs(); + bits += if a == 0 { 160 } else { 640 + 384 * (32 - a.leading_zeros()) }; + } + bits +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict_intra(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- motion vector coding + +/// `mv_class` of an offset (|component| - 1). +fn mv_class(offset: u32) -> u32 { + if offset < 16 { 0 } else { (31 - offset.leading_zeros()) - 3 } +} + +fn mv_component_cost(i: usize, v: i32) -> u32 { + let d = &probs_defaults().0; + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + let mut bits = cost(v < 0, d[probs::MV_SIGN + i]); + bits += tree_cost(&MV_CLASS_TREE, &d[probs::MV_CLASS + i * 10..], c as u8); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = offset >> 3; + bits += cost(b == 1, d[probs::MV_CLASS0_BIT + i]); + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_CLASS0_FR + i * 6 + b as usize * 3..], fr); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + bits += cost((dd >> b) & 1 == 1, d[probs::MV_BITS + i * 10 + b as usize]); + } + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_FR + i * 3..], fr); + } + bits +} + +const MV_TABLE: i32 = 8192; + +/// Component costs for |v| ≤ MV_TABLE (index v + MV_TABLE) and the four joint costs. +fn mv_tables() -> &'static ([Vec; 2], [u32; 4]) { + use std::sync::OnceLock; + static T: OnceLock<([Vec; 2], [u32; 4])> = OnceLock::new(); + T.get_or_init(|| { + let comp = std::array::from_fn(|i| (-MV_TABLE..=MV_TABLE).map(|v| if v == 0 { 0 } else { mv_component_cost(i, v) }).collect()); + let d = &probs_defaults().0; + let joints = std::array::from_fn(|j| tree_cost(&MV_JOINT_TREE, &d[probs::MV_JOINT..], j as u8)); + (comp, joints) + }) +} + +/// Bits (1/256) of a NEWMV difference with the default probabilities. +pub fn mv_cost(diff: Mv) -> u32 { + let (comp, joints) = mv_tables(); + let joint = (diff.row != 0) as usize * 2 + (diff.col != 0) as usize; + let c = |i: usize, v: i32| if v.abs() <= MV_TABLE { comp[i][(v + MV_TABLE) as usize] } else { mv_component_cost(i, v) }; + joints[joint] + if diff.row != 0 { c(0, diff.row) } else { 0 } + if diff.col != 0 { c(1, diff.col) } else { 0 } +} + +fn probs_defaults() -> &'static probs::Probs { + use std::sync::OnceLock; + static D: OnceLock = OnceLock::new(); + D.get_or_init(probs::Probs::defaults) +} + +// ---------------------------------------------------------------- the tile + +/// Per-frame coding parameters shared by the tiles. +pub struct FrameParams<'a> { + pub key: bool, + pub quant: Quant, + pub mi_cols: usize, + pub mi_rows: usize, + /// Reference planes (inter frames). + pub refs: Option<[RefPlane<'a>; 3]>, + /// The reference luma at every quarter-sample phase (motion search). + pub phases: Option, + /// Encoder-side motion hints from the previous frame (by MI position). + pub prev_mvs: Option<&'a [Mv]>, + /// Lagrangian multipliers: per bit, in SAD and in squared-error units. + pub lambda_sad: f64, + pub lambda_sse: f64, +} + +pub struct Tile<'a> { + pub f: &'a FrameParams<'a>, + pub w: Writer, + pub planes: &'a mut [Plane; 3], + pub mi: &'a mut [Mi], + pub col_start: usize, + pub col_end: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// Motion vectors of the four 8×8 blocks evaluated for the current 16×16 area. + pending: [(usize, usize, Mv); 4], +} + +impl<'a> Tile<'a> { + pub fn new(f: &'a FrameParams<'a>, planes: &'a mut [Plane; 3], mi: &'a mut [Mi], col_start: usize, col_end: usize) -> Tile<'a> { + let n = f.mi_cols * 2 + 16; + Tile { + f, + w: Writer::default(), + planes, + mi, + col_start, + col_end, + above_nz: [vec![0; n], vec![0; n], vec![0; n]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; f.mi_cols + 8], + left_part: vec![0; 8], + pending: [(usize::MAX, 0, Mv::ZERO); 4], + } + } + + /// Code the tile's superblocks. + pub fn code(&mut self) { + for r in (0..self.f.mi_rows).step_by(8) { + for p in 0..3 { + self.left_nz[p].fill(0); + } + self.left_part.fill(0); + for c in (self.col_start..self.col_end).step_by(8) { + self.partition(r, c, 12); + } + } + } + + fn mi_at(&self, r: usize, c: usize) -> Mi { + self.mi[r * self.f.mi_cols + c] + } + + fn set_mi(&mut self, r: usize, c: usize, n8: usize, m: Mi) { + for dr in 0..n8 { + for dc in 0..n8 { + if r + dr < self.f.mi_rows && c + dc < self.f.mi_cols { + self.mi[(r + dr) * self.f.mi_cols + c + dc] = m; + } + } + } + } + + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + if r >= mi_rows || c >= mi_cols { + return; + } + let num8 = [1usize, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8][bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < mi_rows; + let has_cols = (c + half) < mi_cols; + let bsl = [0usize, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3][bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let ctx = bsl * 4 + (((left >> boffset) & 1) as usize) * 2 + ((above >> boffset) & 1) as usize; + let key = self.f.key; + let prob = |i: usize| if key { P::Fixed(KF_PARTITION_PROBS[ctx * 3 + i]) } else { P::Ctx((probs::PARTITION + ctx * 3 + i) as u16) }; + if bsize == BLOCK_8X8 { + self.w.put(false, prob(0)); // PARTITION_NONE + if key { + self.intra_block(r, c); + } else { + let mv = self.pending.iter().find(|p| p.0 == r && p.1 == c).map(|p| p.2); + let mv = mv.unwrap_or_else(|| self.search(r, c, BLOCK_8X8).0); + self.inter_block(r, c, BLOCK_8X8, mv); + } + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + if !key && bsize == BLOCK_16X16 && has_rows && has_cols { + if let Some(mv) = self.decide16(r, c) { + self.w.put(false, prob(0)); // PARTITION_NONE + self.inter_block(r, c, BLOCK_16X16, mv); + for i in 0..2 { + self.above_part[c + i] = 15 >> 2; + self.left_part[(r & 7) + i] = 15 >> 2; + } + return; + } + } else if !key && bsize == BLOCK_16X16 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.w.tree(&PARTITION_TREE, 3, prob); + } else if has_cols { + self.w.put(true, prob(1)); + } else if has_rows { + self.w.put(true, prob(2)); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + // ------------------------------------------------------------ intra (key frames) + + fn intra_block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + let y_mode = self.choose_intra(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.intra_cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (neighbours never skip in key frames: context 0); tx_size implied (4×4). + self.w.ctx(false, probs::SKIP); + let above_mode = if avail_u { self.mi_at(r - 1, c).mode } else { DC_PRED }; + let left_mode = if avail_l { self.mi_at(r, c - 1).mode } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.w.tree(&INTRA_MODE_TREE, y_mode, |i| P::Fixed(KF_Y_MODE_PROBS[o + i])); + let o = y_mode as usize * 9; + self.w.tree(&INTRA_MODE_TREE, uv_mode, |i| P::Fixed(KF_UV_MODE_PROBS[o + i])); + self.set_mi(r, c, 1, Mi { inter: false, mode: y_mode, mv: Mv::ZERO, skip: false, size: BLOCK_8X8 }); + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.f.quant.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.intra_tx(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.intra_tx(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + #[allow(clippy::too_many_arguments)] + fn intra_cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose_intra(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.intra_cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn intra_tx(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.f.mi_cols * 8) >> ss; + let max_y = (self.f.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let q = Quant { round_dc: 0.5, round_ac: 0.38, ..self.f.quant }; + let levels = q.levels(&res, tx_type); + let eob = eob_of(&levels, tx_type); + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, false, tx_type, &levels, eob, ctx0); + let resid = q.residual(&levels, tx_type, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, inter: bool, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = probs::coef_index(ptype, inter, band, ctx); + if check_eob { + let more = c < eob; + self.w.ctx(more, o); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.w.ctx(false, o + 1); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.w.ctx(true, o + 1); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.w.ctx(false, o + 2); + } else { + self.w.ctx(true, o + 2); + let path: &[(u8, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.w.put(b, P::Pareto((o + 2) as u16, n)); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.w.fixed((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.w.fixed(levels[pos] < 0, 128); + c += 1; + } + } + + // ------------------------------------------------------------ inter + + fn is_inside(&self, r: i32, c: i32) -> bool { + r >= 0 && (r as usize) < self.f.mi_rows && c >= self.col_start as i32 && (c as usize) < self.col_end + } + + /// `find_mv_refs` + `find_best_ref_mvs` for LAST_FRAME: (NearestMv, NearMv, mode context). + fn mv_refs(&self, r: usize, c: usize, bsize: u8) -> (Mv, Mv, usize) { + let search = if bsize == BLOCK_16X16 { &MV_REF_16X16 } else { &MV_REF_8X8 }; + let mut list = [Mv::ZERO; 2]; + let mut count = 0; + let add = |mv: Mv, list: &mut [Mv; 2], count: &mut usize| { + if *count >= 2 || (*count > 0 && mv == list[0]) { + return; + } + list[*count] = mv; + *count += 1; + }; + let mut counter = 0usize; + for (i, &(dr, dc)) in search.iter().enumerate() { + let (cr, cc) = (r as i32 + dr, c as i32 + dc); + if !self.is_inside(cr, cc) { + continue; + } + let m = self.mi_at(cr as usize, cc as usize); + if i < 2 { + counter += MODE_2_COUNTER[m.mode as usize] as usize; + } + if m.inter { + add(m.mv, &mut list, &mut count); + } + } + let ctx = COUNTER_TO_CONTEXT[counter.min(18)] as usize; + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let clamp = |mv: Mv, border: i32| -> Mv { + let (r, c, b) = (r as i32, c as i32, n8); + let to_top = -(r * 64); + let to_bottom = (self.f.mi_rows as i32 - b - r) * 64; + let to_left = -(c * 64); + let to_right = (self.f.mi_cols as i32 - b - c) * 64; + Mv::new( + mv.row.clamp(to_top - border, (to_bottom + border).max(to_top - border)), + mv.col.clamp(to_left - border, (to_right + border).max(to_left - border)), + ) + }; + for mv in list.iter_mut() { + *mv = clamp(*mv, 128); + // No high precision: odd components move toward zero. + let even = |v: i32| if v & 1 != 0 { v + if v > 0 { -1 } else { 1 } } else { v }; + *mv = clamp(Mv::new(even(mv.row), even(mv.col)), (160 - 4) << 3); + } + (list[0], list[1], ctx) + } + + /// The allowed motion vector range of a block (no decoder clamping inside it). + fn mv_range(&self, r: usize, c: usize, n: i32) -> (i32, i32, i32, i32) { + let n8 = n / 8; + let (r, c) = (r as i32, c as i32); + let lo_c = -(c * 64) - (4 + n) * 8; + let hi_c = (self.f.mi_cols as i32 - n8 - c) * 64 + (4 + n) * 8 - 8; + let lo_r = -(r * 64) - (4 + n) * 8; + let hi_r = (self.f.mi_rows as i32 - n8 - r) * 64 + (4 + n) * 8 - 8; + let lim = 8 * 1000; + (lo_r.max(-lim), hi_r.min(lim), lo_c.max(-lim), hi_c.min(lim)) + } + + /// SAD of the luma prediction at `mv` against the source. + fn sad(&self, r: usize, c: usize, n: usize, mv: Mv) -> u32 { + let p = &self.planes[0]; + let (x16, y16) = (((c * 8) << 4) as i32 + 2 * mv.col, ((r * 8) << 4) as i32 + 2 * mv.row); + let mut s = 0u32; + if let Some((pl, o, st)) = self.f.phases.as_ref().and_then(|ph| ph.block(x16, y16, n)) { + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + let b = &pl[o + i * st..o + i * st + n]; + s += a.iter().zip(b).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + return s; + } + let refs = self.f.refs.as_ref().expect("reference"); + let mut pred = [0u8; 256]; + predict(&refs[0], x16, y16, n, n, &mut pred); + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + s += a.iter().zip(&pred[i * n..i * n + n]).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + s + } + + /// Motion search for the block at (`r`, `c`): the best vector and its cost + /// (SAD + λ·bits). + fn search(&self, r: usize, c: usize, bsize: u8) -> (Mv, f64) { + let n = if bsize == BLOCK_16X16 { 16 } else { 8 }; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let (lo_r, hi_r, lo_c, hi_c) = self.mv_range(r, c, n as i32); + let inside = |mv: Mv| mv.row >= lo_r && mv.row <= hi_r && mv.col >= lo_c && mv.col <= hi_c; + let lambda = self.f.lambda_sad / 256.0; + let d = &probs_defaults().0; + let mb: [u32; 4] = std::array::from_fn(|m| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m as u8)); + let mode_bits = |m: u8| mb[(m - NEARESTMV) as usize]; + let cost = |mv: Mv| -> f64 { + let bits = if mv == nearest { + mode_bits(NEARESTMV) + } else if mv == Mv::ZERO { + mode_bits(ZEROMV) + } else if mv == near { + mode_bits(NEARMV) + } else { + mode_bits(NEWMV) + mv_cost(mv.minus(nearest)) + }; + self.sad(r, c, n, mv) as f64 + lambda * bits as f64 + }; + let mut best = (Mv::ZERO, cost(Mv::ZERO)); + let prev = self.f.prev_mvs.map(|p| p[r * self.f.mi_cols + c]); + for cand in [Some(nearest), Some(near), prev].into_iter().flatten() { + if cand != Mv::ZERO && inside(cand) { + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + // A perfect match needs no search. + let good_enough = lambda * 256.0 * 0.5; + if best.1 > good_enough { + // Whole-sample search from the best candidate: a coarse cross, then a small diamond. + let round8 = |v: i32| (v + 4).div_euclid(8) * 8; + let mut cur = Mv::new(round8(best.0.row), round8(best.0.col)); + let mut cur_j = if inside(cur) { cost(cur) } else { f64::MAX }; + if cur_j < best.1 { + best = (cur, cur_j); + } else { + cur = best.0; + cur_j = best.1; + } + for (step, max_iters) in [(16, 1), (8, 2), (4, 3), (2, 4), (1, 8)] { + let s = step * 8; + for _ in 0..max_iters { + let mut moved = false; + let center = cur; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < cur_j { + cur = cand; + cur_j = j; + moved = true; + } + } + if !moved { + break; + } + } + } + if cur_j < best.1 { + best = (cur, cur_j); + } + } + // Sub-sample refinement: half then quarter samples (1/8 units, even: no high precision). + for s in [4, 2] { + let center = best.0; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s), (-s, -s), (-s, s), (s, -s), (s, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + best + } + + /// Choose between one 16×16 block and four 8×8 blocks at (`r`, `c`): `Some(mv)` for 16×16. + fn decide16(&mut self, r: usize, c: usize) -> Option { + let (mv16, j16) = self.search(r, c, BLOCK_16X16); + let lambda = self.f.lambda_sad; + // A near-perfect 16×16 prediction needs no split. + if j16 < 2.0 * 256.0 / 256.0 * 16.0 { + return Some(mv16); + } + let mut j8 = lambda * 4.0; // partition split + three more blocks of mode info (≈ 4 bits) + for (k, (dr, dc)) in [(0, 0), (0, 1), (1, 0), (1, 1)].into_iter().enumerate() { + let (rr, cc) = (r + dr, c + dc); + let (mv, j) = self.search(rr, cc, BLOCK_8X8); + j8 += j; + self.pending[k] = (rr, cc, mv); + // Tentative mode info so the next 8×8 blocks see this one as a candidate. + self.set_mi(rr, cc, 1, Mi { inter: true, mode: NEWMV, mv, skip: false, size: BLOCK_8X8 }); + } + if j16 <= j8 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + Some(mv16) + } else { + None + } + } + + /// Code an inter block with motion vector `mv`. + fn inter_block(&mut self, r: usize, c: usize, bsize: u8, mv: Mv) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let n = 8 * n8; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let d = &probs_defaults().0; + let mode_bits = |m: u8| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m - NEARESTMV); + let mut mode = (NEWMV, mode_bits(NEWMV) + mv_cost(mv.minus(nearest))); + for (m, ok) in [(NEARESTMV, mv == nearest), (NEARMV, mv == near), (ZEROMV, mv == Mv::ZERO)] { + if ok && mode_bits(m) < mode.1 { + mode = (m, mode_bits(m)); + } + } + let mode = mode.0; + // Prediction of every plane (written straight into the reconstruction). + let refs = self.f.refs.as_ref().expect("reference"); + for plane in 0..3 { + let ch = plane > 0; + let (row16, col16) = clamp_for_plane(mv, r, c, n8, mi_rows, mi_cols, ch); + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let mut pred = [0u8; 256]; + predict(&refs[plane], ((bx as i32) << 4) + col16, ((by as i32) << 4) + row16, bn, bn, &mut pred); + let p = &mut self.planes[plane]; + for i in 0..bn { + p.rec[(by + i) * p.w + bx..][..bn].copy_from_slice(&pred[i * bn..i * bn + bn]); + } + } + // Residual: quantise every 4×4, dropping blocks whose coefficients don't pay for themselves. + let q = Quant { round_dc: 0.5, round_ac: 0.42, ..self.f.quant }; + // Weight of the rate term when dropping a transform block's coefficients (low: keep quality on par + // with key frames at the same q index). + let zl = 0.07; + let tx = DCT_DCT; + let mut blocks: Vec<(usize, usize, usize, [i32; 16], usize)> = Vec::with_capacity(24); + let mut any = false; + for plane in 0..3 { + let ch = plane > 0; + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let p = &self.planes[plane]; + for ty in (0..bn).step_by(4) { + for tx4 in (0..bn).step_by(4) { + let (x, y) = (bx + tx4, by + ty); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + res[i * 4 + j] = p.src[k] as i32 - p.rec[k] as i32; + } + } + let mut levels = q.levels(&res, tx); + let mut eob = eob_of(&levels, tx); + if eob > 0 && !q.lossless { + let back = q.residual(&levels, tx, eob); + let d_skip: i64 = res.iter().map(|v| (*v as i64) * (*v as i64)).sum(); + let d_code: i64 = res.iter().zip(&back).map(|(a, b)| ((a - b) as i64).pow(2)).sum(); + if d_skip as f64 <= d_code as f64 + zl * self.f.lambda_sse * levels_cost(&levels, eob) as f64 / 256.0 { + levels = [0; 16]; + eob = 0; + } + } + any |= eob > 0; + blocks.push((plane, x, y, levels, eob)); + } + } + } + let skip = !any; + // Mode info. + let avail_u = r > 0; + let avail_l = c > self.col_start; + let above = avail_u.then(|| self.mi_at(r - 1, c)); + let left = avail_l.then(|| self.mi_at(r, c - 1)); + let skip_ctx = above.is_some_and(|m| m.skip) as usize + left.is_some_and(|m| m.skip) as usize; + self.w.ctx(skip, probs::SKIP + skip_ctx); + // is_inter context: neighbours are inter (no intra blocks in inter frames). + let intra = |m: Option| m.is_some_and(|m| !m.inter); + let inter_ctx = match (above, left) { + (Some(_), Some(_)) => { + if intra(above) && intra(left) { + 3 + } else { + (intra(above) || intra(left)) as usize + } + } + (Some(_), None) => 2 * intra(above) as usize, + (None, Some(_)) => 2 * intra(left) as usize, + (None, None) => 0, + }; + self.w.ctx(true, probs::IS_INTER + inter_ctx); + // single_ref_p1 = 0 (LAST_FRAME); context for single-reference LAST neighbours. + let ref_ctx = match (above, left) { + (Some(a), Some(l)) => match (a.inter, l.inter) { + (false, false) => 2, + (false, true) => 4, + (true, false) => 4, + (true, true) => 4, + }, + (Some(m), None) | (None, Some(m)) => { + if m.inter { + 4 + } else { + 2 + } + } + (None, None) => 2, + }; + self.w.ctx(false, probs::SINGLE_REF + ref_ctx * 2); + self.w.tree(&INTER_MODE_TREE, mode - NEARESTMV, |i| P::Ctx((probs::INTER_MODE + mctx * 3 + i) as u16)); + if mode == NEWMV { + self.write_mv(mv.minus(nearest)); + } + self.set_mi(r, c, n8, Mi { inter: true, mode, mv, skip, size: bsize }); + // Tokens and reconstruction. + for (plane, x, y, levels, eob) in blocks { + let ss = (plane > 0) as usize; + let (x4, ly) = (x >> 2, (y >> 2) & (15 >> ss)); + if !skip { + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, true, tx, &levels, eob, ctx0); + if eob > 0 { + let resid = q.residual(&levels, tx, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + p.rec[k] = (p.rec[k] as i32 + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + } + } + let nz = (!skip && eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + } + + fn write_mv(&mut self, diff: Mv) { + let joint = (diff.row != 0) as u8 * 2 + (diff.col != 0) as u8; + self.w.tree(&MV_JOINT_TREE, joint, |i| P::Ctx((probs::MV_JOINT + i) as u16)); + if diff.row != 0 { + self.write_mv_component(0, diff.row); + } + if diff.col != 0 { + self.write_mv_component(1, diff.col); + } + } + + fn write_mv_component(&mut self, i: usize, v: i32) { + debug_assert!(v != 0 && v & 1 == 0, "vectors are even without high precision"); + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + self.w.ctx(v < 0, probs::MV_SIGN + i); + self.w.tree(&MV_CLASS_TREE, c as u8, |k| P::Ctx((probs::MV_CLASS + i * 10 + k) as u16)); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = (offset >> 3) as usize; + self.w.ctx(b == 1, probs::MV_CLASS0_BIT + i); + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_CLASS0_FR + i * 6 + b * 3 + k) as u16)); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + self.w.ctx((dd >> b) & 1 == 1, probs::MV_BITS + i * 10 + b as usize); + } + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_FR + i * 3 + k) as u16)); + } + } +} diff --git a/crates/vp9enc/src/transform.rs b/crates/vp9enc/src/transform.rs new file mode 100644 index 00000000..e6a685d5 --- /dev/null +++ b/crates/vp9enc/src/transform.rs @@ -0,0 +1,249 @@ +//! 4×4 transforms: the decoder's integer inverse DCT / ADST (spec 8.7.1) and Walsh–Hadamard +//! transform (8.7.1.10), and their forward counterparts (the exact inverse matrices of the +//! decoder's linear transforms, so `inverse4(forward(res)) ≈ res`; the WHT inverse is exact). + +pub const DCT_DCT: u8 = 0; +pub const ADST_DCT: u8 = 1; +pub const DCT_ADST: u8 = 2; +pub const ADST_ADST: u8 = 3; + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +pub fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +pub fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +pub fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms. +pub fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +/// Quantisation settings of one block. +#[derive(Clone, Copy)] +pub struct Quant { + pub dcq: i32, + pub acq: i32, + pub lossless: bool, + /// Rounding offsets (fractions of a step) for DC and AC; smaller = wider dead zone. + pub round_dc: f64, + pub round_ac: f64, +} + +impl Quant { + /// Quantised levels (raster order) of a residual. + pub fn levels(&self, res: &[i32; 16], tx_type: u8) -> [i32; 16] { + if self.lossless { + return forward_wht(res); + } + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let (qs, rnd) = if k == 0 { (self.dcq, self.round_dc) } else { (self.acq, self.round_ac) }; + let l = (coef.abs() / qs as f64 + rnd).floor() as i32; + lv[k] = if coef < 0.0 { -l } else { l }; + } + lv + } + + /// The decoder's residual for `levels` (all zero when `eob` is 0). + pub fn residual(&self, levels: &[i32; 16], tx_type: u8, eob: usize) -> [i32; 16] { + if eob == 0 { + return [0; 16]; + } + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { self.dcq } else { self.acq }; + } + if self.lossless { inverse_wht(&deq) } else { inverse4(&deq, tx_type) } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/docs/parity.md b/docs/parity.md index 38df7882..8fc67b9a 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -110,7 +110,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 key + inter frames with motion search, loop filter and rate control; Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | From 30a92ba82f5e2d064e3d4265957fe7274cef95b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:53:05 -0400 Subject: [PATCH 20/36] M13.2: OpenType features (stylistic sets, figures, fractions, ordinals, dlig, true small caps / superscripts with faux fallback), Character panel OpenType popup, text.fontFeatures Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/layer.rs | 2 +- crates/engine/src/commands/text_edit.rs | 41 ++++ crates/engine/src/lib.rs | 5 + crates/engine/src/tests_text_edit.rs | 25 +++ crates/keyframe/src/lib.rs | 2 +- crates/keyframe/src/text_doc.rs | 243 +++++++++++++++++++++++ crates/text/src/fonts.rs | 15 ++ crates/text/src/layout.rs | 207 +++++++++++++++++-- crates/text/src/lib.rs | 1 + crates/ui-egui/src/panels/text_panels.rs | 108 +++++++++- 10 files changed, 628 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index 29b4c595..bf87a96d 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -1157,7 +1157,7 @@ pub fn specs() -> Vec { "Edit Text", [], None, - "{layer?, range?: [start, end] (characters; default all), text?, font?, style?, size?, fill?, stroke?, applyFill?, applyStroke?, strokeWidth?, tracking?, leading?: px|\"auto\", baselineShift? px, hScale? %, vScale? %, tsume? %, fauxBold?, fauxItalic?, allCaps?, smallCaps?, baseline?: normal|superscript|subscript, superscript?, subscript?, kerning?: metrics|optical|number, ligatures?, justify?: left|center|right|justifyLeft|justifyCenter|justifyRight|justifyAll, indentLeft?, indentRight?, indentFirst?, spaceBefore?, spaceAfter?, direction?: ltr|rtl, composer?: everyLine|singleLine, hangingPunctuation?, strokeOverFill?, box?: [x,y,w,h]|null (layer space), vertical?}", + "{layer?, range?: [start, end] (characters; default all), text?, font?, style?, size?, fill?, stroke?, applyFill?, applyStroke?, strokeWidth?, tracking?, leading?: px|\"auto\", baselineShift? px, hScale? %, vScale? %, tsume? %, fauxBold?, fauxItalic?, allCaps?, smallCaps?, baseline?: normal|superscript|subscript, superscript?, subscript?, kerning?: metrics|optical|number, ligatures?, OpenType: discretionaryLigatures?, contextualAlternates?, stylisticAlternates?, stylisticSets?: [1–20], ss01…ss20?, swash?, titling?, ordinals?, fractions?, allSmallCaps?, figureStyle?: default|lining|oldStyle, figureWidth?: default|proportional|tabular, figures?, justify?: left|center|right|justifyLeft|justifyCenter|justifyRight|justifyAll, indentLeft?, indentRight?, indentFirst?, spaceBefore?, spaceAfter?, direction?: ltr|rtl, composer?: everyLine|singleLine, hangingPunctuation?, strokeOverFill?, box?: [x,y,w,h]|null (layer space), vertical?}", has_layers, set_text ), diff --git a/crates/engine/src/commands/text_edit.rs b/crates/engine/src/commands/text_edit.rs index 28048389..37ed9da0 100644 --- a/crates/engine/src/commands/text_edit.rs +++ b/crates/engine/src/commands/text_edit.rs @@ -465,6 +465,38 @@ pub(crate) fn sanitize(s: &mut Session) { } } +/// The OpenType features of a font (`font` / `style`, else the target text layer's style at the +/// caret or its first character), and which Character panel options it sets with its own glyphs +/// (the others are synthesized or have no effect). +pub fn font_features(s: &mut Session, p: &Value) -> Result { + let (family, style) = match p.get("font").and_then(Value::as_str) { + Some(f) => (f.to_string(), p.get("style").and_then(Value::as_str).unwrap_or("Regular").to_string()), + None => { + let lid = target(s, p, "text.fontFeatures")?; + let doc = layer_doc(s, lid).ok_or_else(|| bad("text.fontFeatures", "not a text layer"))?; + let at = s.state.text_edit.as_ref().filter(|e| e.layer == lid).map_or(0, |e| e.range().start); + let st = doc.style_at(at); + (st.font, st.style) + } + }; + let r = effectcraft_text::resolve(&family, &style); + let face = effectcraft_text::fonts::face(r.face); + let feats = face.features(); + let has = |t: &str| feats.iter().any(|f| f == t); + let sets: Vec = (1..=20u32).filter(|n| has(&format!("ss{n:02}"))).collect(); + Ok(json!({ + "family": face.info.family, "style": face.info.style, "missing": r.missing, + "features": feats, + "options": { + "kerning": has("kern"), "ligatures": has("liga") || has("clig"), "discretionaryLigatures": has("dlig"), + "contextualAlternates": has("calt"), "stylisticAlternates": has("salt"), "stylisticSets": sets, + "swash": has("swsh"), "titling": has("titl"), "ordinals": has("ordn"), "fractions": has("frac"), + "smallCaps": has("smcp"), "allSmallCaps": has("c2sc"), "superscript": has("sups"), "subscript": has("subs"), + "oldStyleFigures": has("onum"), "liningFigures": has("lnum"), "tabularFigures": has("tnum"), "proportionalFigures": has("pnum"), + }, + })) +} + fn always_ok(_: &Session) -> std::result::Result<(), String> { Ok(()) } @@ -485,6 +517,15 @@ pub fn specs() -> Vec { edit ), cmd!("text.endEdit", "Exit Text Editing", [], None, "{}", always_ok, end_edit), + cmd!( + "text.fontFeatures", + "OpenType Features", + [], + None, + "{layer?, font?, style?} → {family, style, features: [tags], options: {smallCaps, superscript, stylisticSets: [n], fractions, …}} (what the font sets with its own glyphs)", + always_ok, + font_features + ), cmd!( "text.setSelection", "Set Text Selection", diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9333329a..f28c07a7 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -747,6 +747,11 @@ pub fn text_presets() -> Vec<(String, String)> { pub fn text_families() -> Vec { effectcraft_text::families().into_iter().map(|(f, _)| f).collect() } + +/// The OpenType feature tags (GSUB / GPOS) of the face a family + style resolves to. +pub fn font_features(family: &str, style: &str) -> Vec { + effectcraft_text::fonts::face(effectcraft_text::resolve(family, style).face).features() +} #[cfg(test)] mod tests_paint; #[cfg(test)] diff --git a/crates/engine/src/tests_text_edit.rs b/crates/engine/src/tests_text_edit.rs index 2476727f..46610786 100644 --- a/crates/engine/src/tests_text_edit.rs +++ b/crates/engine/src/tests_text_edit.rs @@ -202,3 +202,28 @@ fn edit_state_round_trips_through_serde() { let back: crate::EditorState = serde_json::from_value(j).unwrap(); assert_eq!(back.text_edit, s.state.text_edit); } + +#[test] +fn opentype_features_per_range_render_and_query() { + let (mut s, t) = session("Office 1/2"); + s.execute("layer.setText", json!({"layer": t, "range": [7, 10], "fractions": true, "ss01": true})).unwrap(); + let d = doc(&s, t); + assert!(!d.style_at(0).opentype.fractions); + assert!(d.style_at(8).opentype.fractions && d.style_at(8).opentype.stylistic_set(1)); + // The laid-out glyphs change (frac turns 1/2 into a fraction). + let plain = effectcraft_text::layout_doc(&TextDoc { text: "1/2".into(), ..Default::default() }); + let frac = effectcraft_text::layout_doc(&d.slice(7..10)); + let ids = |l: &effectcraft_text::TextLayout| l.glyphs.iter().map(|g| g.gid).collect::>(); + assert_ne!(ids(&plain), ids(&frac)); + // Undo restores the uniform style. + s.execute("edit.undo", json!({})).unwrap(); + assert!(doc(&s, t).is_uniform()); + // Font feature query (font by name, or the layer's). + let f = s.execute("text.fontFeatures", json!({"font": "Noto Serif"})).unwrap(); + assert_eq!(f["options"]["smallCaps"], true); + assert!(f["features"].as_array().unwrap().iter().any(|t| t == "onum")); + let f = s.execute("text.fontFeatures", json!({"layer": t})).unwrap(); + assert_eq!(f["family"], "Inter"); + assert!(f["options"]["stylisticSets"].as_array().unwrap().contains(&json!(1))); + assert_eq!(f["options"]["smallCaps"], false); +} diff --git a/crates/keyframe/src/lib.rs b/crates/keyframe/src/lib.rs index ec2b4754..8932530d 100644 --- a/crates/keyframe/src/lib.rs +++ b/crates/keyframe/src/lib.rs @@ -13,7 +13,7 @@ pub mod value; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use serde::{Deserialize, Serialize}; -pub use text_doc::{BaselineOption, CharStyle, Composer, Direction, Kerning, ParaStyle, StyleRun}; +pub use text_doc::{BaselineOption, CharStyle, Composer, Direction, FigureStyle, FigureWidth, Kerning, OpenType, ParaStyle, StyleRun}; pub use value::{FeatherPoint, Gradient, Justify, ShapePath, TextDoc, Value}; /// Temporal interpolation on one side of a keyframe. diff --git a/crates/keyframe/src/text_doc.rs b/crates/keyframe/src/text_doc.rs index 2e9d5c6e..ae0b2e56 100644 --- a/crates/keyframe/src/text_doc.rs +++ b/crates/keyframe/src/text_doc.rs @@ -143,6 +143,125 @@ fn yes() -> bool { true } +/// Figure style (OpenType `lnum` / `onum`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum FigureStyle { + /// The font's default figures. + #[default] + Default, + Lining, + OldStyle, +} + +/// Figure spacing (OpenType `pnum` / `tnum`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum FigureWidth { + #[default] + Default, + Proportional, + Tabular, +} + +/// OpenType layout features of a run of characters (Character panel ▸ OpenType). Kerning, +/// standard ligatures, small caps and superscript / subscript have their own [`CharStyle`] +/// fields (they use the font's `smcp` / `sups` / `subs` glyphs when it has them and synthesize +/// them otherwise). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(default)] +pub struct OpenType { + /// `dlig`. + pub discretionary_ligatures: bool, + /// `calt` (on by default, as in the font). + pub contextual_alternates: bool, + /// `salt`. + pub stylistic_alternates: bool, + /// Stylistic sets: bit `n - 1` turns on `ssNN` (n = 1…20). + pub stylistic_sets: u32, + /// `swsh`. + pub swash: bool, + /// `titl`. + pub titling: bool, + /// `ordn`. + pub ordinals: bool, + /// `frac`. + pub fractions: bool, + /// All Small Caps: capitals become small capitals too (`c2sc` with `smcp`). + pub all_small_caps: bool, + pub figure_style: FigureStyle, + pub figure_width: FigureWidth, +} + +impl Default for OpenType { + fn default() -> Self { + OpenType { + discretionary_ligatures: false, + contextual_alternates: true, + stylistic_alternates: false, + stylistic_sets: 0, + swash: false, + titling: false, + ordinals: false, + fractions: false, + all_small_caps: false, + figure_style: FigureStyle::Default, + figure_width: FigureWidth::Default, + } + } +} + +impl OpenType { + pub fn is_default(&self) -> bool { + *self == OpenType::default() + } + /// Whether stylistic set `n` (1…20) is on. + pub fn stylistic_set(&self, n: u32) -> bool { + (1..=20).contains(&n) && self.stylistic_sets & (1 << (n - 1)) != 0 + } + pub fn set_stylistic_set(&mut self, n: u32, on: bool) { + if (1..=20).contains(&n) { + if on { + self.stylistic_sets |= 1 << (n - 1); + } else { + self.stylistic_sets &= !(1 << (n - 1)); + } + } + } + /// The OpenType feature settings these options ask for (tag, value), beyond the font's + /// defaults: e.g. `[("dlig", 1), ("ss02", 1), ("calt", 0)]`. + pub fn feature_settings(&self) -> Vec<([u8; 4], u32)> { + let mut v = Vec::new(); + for (tag, f) in [ + (b"dlig", self.discretionary_ligatures), + (b"salt", self.stylistic_alternates), + (b"swsh", self.swash), + (b"titl", self.titling), + (b"ordn", self.ordinals), + (b"frac", self.fractions), + (b"c2sc", self.all_small_caps), + (b"smcp", self.all_small_caps), + (b"lnum", self.figure_style == FigureStyle::Lining), + (b"onum", self.figure_style == FigureStyle::OldStyle), + (b"pnum", self.figure_width == FigureWidth::Proportional), + (b"tnum", self.figure_width == FigureWidth::Tabular), + ] { + if f { + v.push((*tag, 1)); + } + } + if !self.contextual_alternates { + v.push((*b"calt", 0)); + } + for n in 1..=20u32 { + if self.stylistic_set(n) { + let d = format!("ss{n:02}"); + let b = d.as_bytes(); + v.push(([b[0], b[1], b[2], b[3]], 1)); + } + } + v + } +} + /// The formatting of a run of characters (Character panel). #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] @@ -177,6 +296,9 @@ pub struct CharStyle { /// Vertical type: keep Roman (half-width) characters upright instead of turning them on /// their side (Character panel menu ▸ Standard Vertical Roman Alignment). pub vertical_roman_upright: bool, + /// OpenType features (stylistic sets, figures, fractions…). + #[serde(skip_serializing_if = "OpenType::is_default")] + pub opentype: OpenType, } impl Default for CharStyle { @@ -258,6 +380,9 @@ pub struct TextDoc { /// Base Tate-Chu-Yoko and Standard Vertical Roman Alignment (see [`CharStyle`]). pub tate_chu_yoko: bool, pub vertical_roman_upright: bool, + /// Base OpenType features (see [`CharStyle::opentype`]). + #[serde(skip_serializing_if = "OpenType::is_default")] + pub opentype: OpenType, pub indent_left: f64, pub indent_right: f64, pub indent_first: f64, @@ -309,6 +434,7 @@ impl Default for TextDoc { ligatures: true, tate_chu_yoko: false, vertical_roman_upright: false, + opentype: OpenType::default(), indent_left: 0.0, indent_right: 0.0, indent_first: 0.0, @@ -395,6 +521,7 @@ impl TextDoc { ligatures: self.ligatures, tate_chu_yoko: self.tate_chu_yoko, vertical_roman_upright: self.vertical_roman_upright, + opentype: self.opentype, } } @@ -422,6 +549,7 @@ impl TextDoc { self.ligatures = s.ligatures; self.tate_chu_yoko = s.tate_chu_yoko; self.vertical_roman_upright = s.vertical_roman_upright; + self.opentype = s.opentype; } /// The base (first paragraph's) settings. @@ -777,6 +905,9 @@ fn color_json(v: &J) -> Option<[f32; 4]> { fn num(key: &str, v: &J) -> Result { v.as_f64().filter(|x| x.is_finite()).ok_or_else(|| format!("{key}: expected a number")) } +fn norm(v: &J) -> Option { + v.as_str().map(|s| s.to_ascii_lowercase().replace(['-', ' ', '_'], "")) +} fn flag(key: &str, v: &J) -> Result { v.as_bool().or_else(|| v.as_f64().map(|x| x != 0.0)).ok_or_else(|| format!("{key}: expected a boolean")) } @@ -808,6 +939,18 @@ pub const CHAR_ATTRS: &[&str] = &[ "ligatures", "tateChuYoko", "verticalRomanUpright", + "discretionaryLigatures", + "contextualAlternates", + "stylisticAlternates", + "stylisticSets", + "swash", + "titling", + "ordinals", + "fractions", + "allSmallCaps", + "figureStyle", + "figureWidth", + "figures", ]; /// Paragraph attribute keys of `layer.setText`. @@ -850,6 +993,7 @@ pub fn apply_char_attr(s: &mut CharStyle, key: &str, v: &J) -> Result { @@ -869,6 +1013,65 @@ pub fn apply_char_attr(s: &mut CharStyle, key: &str, v: &J) -> Result s.ligatures = flag(key, v)?, "tateChuYoko" => s.tate_chu_yoko = flag(key, v)?, "verticalRomanUpright" => s.vertical_roman_upright = flag(key, v)?, + "discretionaryLigatures" => s.opentype.discretionary_ligatures = flag(key, v)?, + "contextualAlternates" => s.opentype.contextual_alternates = flag(key, v)?, + "stylisticAlternates" => s.opentype.stylistic_alternates = flag(key, v)?, + "swash" => s.opentype.swash = flag(key, v)?, + "titling" => s.opentype.titling = flag(key, v)?, + "ordinals" => s.opentype.ordinals = flag(key, v)?, + "fractions" => s.opentype.fractions = flag(key, v)?, + "allSmallCaps" => { + s.opentype.all_small_caps = flag(key, v)?; + if s.opentype.all_small_caps { + s.all_caps = false; + } + } + "stylisticSets" => { + s.opentype.stylistic_sets = match v { + J::Array(a) => { + let mut m = 0u32; + for n in a { + let n = n.as_u64().filter(|n| (1..=20).contains(n)).ok_or("stylisticSets: expected set numbers 1–20")?; + m |= 1 << (n - 1); + } + m + } + J::Number(_) => (num(key, v)? as u64 & 0xF_FFFF) as u32, + _ => return Err("stylisticSets: expected [1, 2, …] or a bit mask".into()), + } + } + "figureStyle" => { + s.opentype.figure_style = match norm(v).as_deref() { + Some("default") => FigureStyle::Default, + Some("lining") => FigureStyle::Lining, + Some("oldstyle") => FigureStyle::OldStyle, + _ => return Err("figureStyle: expected default|lining|oldStyle".into()), + } + } + "figureWidth" => { + s.opentype.figure_width = match norm(v).as_deref() { + Some("default") => FigureWidth::Default, + Some("proportional") => FigureWidth::Proportional, + Some("tabular") => FigureWidth::Tabular, + _ => return Err("figureWidth: expected default|proportional|tabular".into()), + } + } + "figures" => { + let (st, w) = match norm(v).as_deref() { + Some("default") => (FigureStyle::Default, FigureWidth::Default), + Some("tabularlining") => (FigureStyle::Lining, FigureWidth::Tabular), + Some("proportionallining") => (FigureStyle::Lining, FigureWidth::Proportional), + Some("tabularoldstyle") => (FigureStyle::OldStyle, FigureWidth::Tabular), + Some("proportionaloldstyle") => (FigureStyle::OldStyle, FigureWidth::Proportional), + _ => return Err("figures: expected default|tabularLining|proportionalLining|tabularOldstyle|proportionalOldstyle".into()), + }; + s.opentype.figure_style = st; + s.opentype.figure_width = w; + } + k if k.len() == 4 && k.starts_with("ss") && k[2..].parse::().is_ok_and(|n| (1..=20).contains(&n)) => { + let n = k[2..].parse::().unwrap_or(0); + s.opentype.set_stylistic_set(n, flag(key, v)?); + } _ => return Ok(false), } Ok(true) @@ -915,6 +1118,13 @@ pub fn char_style_json(s: &CharStyle) -> J { "hScale": s.h_scale, "vScale": s.v_scale, "tsume": s.tsume, "fauxBold": s.faux_bold, "fauxItalic": s.faux_italic, "allCaps": s.all_caps, "smallCaps": s.small_caps, "baseline": s.baseline.key(), "kerning": s.kerning.to_json(), "ligatures": s.ligatures, "tateChuYoko": s.tate_chu_yoko, "verticalRomanUpright": s.vertical_roman_upright, + "discretionaryLigatures": s.opentype.discretionary_ligatures, "contextualAlternates": s.opentype.contextual_alternates, + "stylisticAlternates": s.opentype.stylistic_alternates, + "stylisticSets": (1..=20u32).filter(|n| s.opentype.stylistic_set(*n)).collect::>(), + "swash": s.opentype.swash, "titling": s.opentype.titling, "ordinals": s.opentype.ordinals, "fractions": s.opentype.fractions, + "allSmallCaps": s.opentype.all_small_caps, + "figureStyle": match s.opentype.figure_style { FigureStyle::Default => "default", FigureStyle::Lining => "lining", FigureStyle::OldStyle => "oldStyle" }, + "figureWidth": match s.opentype.figure_width { FigureWidth::Default => "default", FigureWidth::Proportional => "proportional", FigureWidth::Tabular => "tabular" }, }) } @@ -1127,4 +1337,37 @@ mod tests { assert_eq!(word_at(t, 8), 7..10); assert_eq!(word_at(t, 6), 6..7); } + + #[test] + fn opentype_attributes_per_range_and_serde() { + let mut d = TextDoc::plain("Office 1/2 1st"); + // Untouched documents serialize without an OpenType block. + assert!(!serde_json::to_string(&d).unwrap().contains("opentype")); + d.set_attr("ss02", &J::Bool(true), Some(0..6)).unwrap(); + d.set_attr("fractions", &J::Bool(true), Some(7..10)).unwrap(); + d.set_attr("figures", &J::from("tabularOldstyle"), None).unwrap(); + assert!(d.style_at(0).opentype.stylistic_set(2)); + assert!(!d.style_at(8).opentype.stylistic_set(2)); + assert!(d.style_at(8).opentype.fractions); + assert_eq!(d.style_at(12).opentype.figure_width, FigureWidth::Tabular); + assert!(d.set_attr("stylisticSets", &serde_json::json!([1, 20]), Some(0..1)).unwrap()); + assert_eq!(d.style_at(0).opentype.stylistic_sets, 1 | (1 << 19)); + assert!(d.set_attr("stylisticSets", &serde_json::json!([21]), None).is_err()); + let f = d.style_at(8).opentype.feature_settings(); + assert!(f.contains(&(*b"frac", 1)) && f.contains(&(*b"onum", 1)) && f.contains(&(*b"tnum", 1))); + let mut s = d.style_at(0); + s.opentype.contextual_alternates = false; + assert!(s.opentype.feature_settings().contains(&(*b"calt", 0))); + // Round trip, and the JSON form names the attributes as layer.setText does. + let back: TextDoc = serde_json::from_str(&serde_json::to_string(&d).unwrap()).unwrap(); + assert_eq!(back, d); + let j = char_style_json(&d.style_at(0)); + assert_eq!(j["stylisticSets"], serde_json::json!([1, 20])); + assert_eq!(j["figureStyle"], "oldStyle"); + // All Small Caps and All Caps exclude each other. + let mut s = CharStyle::default(); + apply_char_attr(&mut s, "allCaps", &J::Bool(true)).unwrap(); + apply_char_attr(&mut s, "allSmallCaps", &J::Bool(true)).unwrap(); + assert!(!s.all_caps && s.opentype.all_small_caps); + } } diff --git a/crates/text/src/fonts.rs b/crates/text/src/fonts.rs index c3a60f5d..297a816c 100644 --- a/crates/text/src/fonts.rs +++ b/crates/text/src/fonts.rs @@ -212,6 +212,21 @@ impl Face { underline_thickness: m.underline.map_or(px * 0.06, |u| u.thickness.max(px * 0.03)), } } + /// The OpenType layout features the face offers (GSUB and GPOS tags, sorted, deduplicated). + pub fn features(&self) -> Vec { + use skrifa::raw::TableProvider; + let Some(f) = self.font() else { return vec![] }; + let mut out: Vec = vec![]; + if let Ok(list) = f.gsub().and_then(|g| g.feature_list()) { + out.extend(list.feature_records().iter().map(|r| String::from_utf8_lossy(&r.feature_tag().to_be_bytes()).into_owned())); + } + if let Ok(list) = f.gpos().and_then(|g| g.feature_list()) { + out.extend(list.feature_records().iter().map(|r| String::from_utf8_lossy(&r.feature_tag().to_be_bytes()).into_owned())); + } + out.sort(); + out.dedup(); + out + } /// Whether the face's GSUB offers an OpenType feature (e.g. `smcp`). pub fn has_feature(&self, tag: &[u8; 4]) -> bool { use skrifa::raw::TableProvider; diff --git a/crates/text/src/layout.rs b/crates/text/src/layout.rs index 824b7373..3333cdd1 100644 --- a/crates/text/src/layout.rs +++ b/crates/text/src/layout.rs @@ -18,6 +18,8 @@ use std::sync::{Arc, Mutex, OnceLock}; use harfrust::{Direction, Feature, Tag, UnicodeBuffer}; use unicode_bidi::{BidiInfo, Level}; +use effectcraft_keyframe::OpenType; + use crate::fonts::{self, FaceId, Resolved}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] @@ -81,6 +83,8 @@ pub struct TextStyle { pub script: Script, /// Baseline-to-baseline distance in pixels; None = auto (120% of the size). pub leading: Option, + /// OpenType features (stylistic sets, figures, fractions, all small caps…). + pub opentype: OpenType, } impl Default for TextStyle { @@ -104,6 +108,7 @@ impl Default for TextStyle { tsume: 0.0, script: Script::Normal, leading: None, + opentype: OpenType::default(), } } } @@ -188,6 +193,7 @@ impl Hash for TextStyle { self.manual_kern.map(f32::to_bits).hash(h); self.leading.map(f32::to_bits).hash(h); (self.kerning, self.optical, self.ligatures, self.faux_bold, self.faux_italic, self.caps, self.underline, self.script).hash(h); + self.opentype.hash(h); } } @@ -342,6 +348,8 @@ struct ShapedGlyph { dy: f32, size: f32, run: usize, + /// Upward baseline offset (baseline shift, plus a synthesized superscript / subscript's). + rise: f32, } struct Item { @@ -360,6 +368,50 @@ fn upper_single(c: char) -> char { const SMALL_CAPS_SCALE: f32 = 0.78; +/// How a character is set: as itself, as a synthesized small capital (a scaled-down capital), +/// with the font's small-capital glyph (`smcp` / `c2sc`), or with the font's superior / inferior +/// glyph (`sups` / `subs`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum Form { + Normal, + FauxSmall, + TrueSmall, + TrueScript, +} + +/// Whether OpenType feature `tag` gives character `c` its own glyph in `face` (cached): the +/// true small caps / superscripts / subscripts test, with synthesized forms as the fallback. +pub fn feature_substitutes(face: FaceId, tag: &[u8; 4], c: char) -> bool { + static C: OnceLock>> = OnceLock::new(); + let cache = C.get_or_init(Default::default); + if let Some(v) = cache.lock().unwrap_or_else(|e| e.into_inner()).get(&(face, *tag, c)) { + return *v; + } + let f = fonts::face(face); + let v = f.has_feature(tag) + && match (f.font(), f.shaper_data()) { + (Some(font), Some(data)) => { + let shaper = data.shaper(&font).build(); + let shape = |feats: &[Feature]| -> Vec { + let mut buf = UnicodeBuffer::new(); + buf.add(c, 0); + buf.guess_segment_properties(); + shaper.shape(buf, harfrust::ShapeOptions::new().features(feats)).glyph_infos().iter().map(|i| i.glyph_id).collect() + }; + let plain = shape(&[]); + let with = shape(&[Feature::new(Tag::new(tag), 1, ..)]); + plain != with && !with.contains(&0) + } + _ => false, + }; + let mut m = cache.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 100_000 { + m.clear(); + } + m.insert((face, *tag, c), v); + v +} + /// Horizontal ink extent `(x0, x1)` of a glyph in font units. fn ink_x(face: FaceId, gid: u32) -> Option<(f32, f32)> { let p = crate::outline_units(face, gid)?; @@ -370,7 +422,7 @@ fn ink_x(face: FaceId, gid: u32) -> Option<(f32, f32)> { Some((r.x0 as f32, r.x1 as f32)) } -fn shape_item(chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle, run: usize) -> Vec { +fn shape_item(chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, style: &TextStyle, run: usize, form: Form) -> Vec { let f = fonts::face(face); let (Some(font), Some(data)) = (f.font(), f.shaper_data()) else { return Vec::new() }; let shaper = data.shaper(&font).build(); @@ -388,6 +440,30 @@ fn shape_item(chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, feats.push(Feature::new(Tag::new(b"liga"), 0, ..)); feats.push(Feature::new(Tag::new(b"clig"), 0, ..)); } + // OpenType options; small caps go per character (true where the font has them). + for (tag, v) in style.opentype.feature_settings() { + if &tag != b"smcp" && &tag != b"c2sc" { + feats.push(Feature::new(Tag::new(&tag), v, ..)); + } + } + if style.caps == Caps::All { + // Case-sensitive forms (punctuation raised to sit with capitals). + feats.push(Feature::new(Tag::new(b"case"), 1, ..)); + } + let rise = match form { + Form::TrueScript => style.baseline_shift, + _ => style.rise(), + }; + match form { + Form::TrueSmall => { + feats.push(Feature::new(Tag::new(b"smcp"), 1, ..)); + if style.opentype.all_small_caps { + feats.push(Feature::new(Tag::new(b"c2sc"), 1, ..)); + } + } + Form::TrueScript => feats.push(Feature::new(Tag::new(if style.script == Script::Sub { b"subs" } else { b"sups" }), 1, ..)), + _ => {} + } let out = shaper.shape(buf, harfrust::ShapeOptions::new().features(&feats)); let k = size / f.units_per_em(); let hs = style.h_scale; @@ -404,6 +480,7 @@ fn shape_item(chars: &[(usize, char, char)], rtl: bool, face: FaceId, size: f32, dy: p.y_offset as f32 * k, size, run, + rise, }) .collect() } @@ -539,38 +616,54 @@ fn paragraph(text: &str, base: usize, runs: &[(Range, TextStyle)], primar let bidi = BidiInfo::new(text, default_level); let pinfo = &bidi.paragraphs[0]; let base_rtl = pinfo.level.is_rtl(); - // per char: (byte, char, shaped char, face, rtl, small caps, run) - let chars: Vec<(usize, char, char, FaceId, bool, bool, usize)> = text + // per char: (byte, char, shaped char, face, rtl, form, run) + let chars: Vec<(usize, char, char, FaceId, bool, Form, usize)> = text .char_indices() .map(|(b, c)| { let si = run_at(runs, base + b); let style = &runs[si].1; - let (sc, small) = match style.caps { - Caps::Normal => (c, false), - Caps::All => (upper_single(c), false), - Caps::Small if c.is_lowercase() => (upper_single(c), true), - Caps::Small => (c, false), - }; let primary = primaries[si].face; - let face = if sc.is_whitespace() { primary } else { fonts::fallback_for(sc, primary) }; - (b, c, sc, face, bidi.levels[b].is_rtl(), small, si) + let face_of = |ch: char| if ch.is_whitespace() { primary } else { fonts::fallback_for(ch, primary) }; + let all_small = style.opentype.all_small_caps && style.caps != Caps::All; + let small = (style.caps == Caps::Small || all_small) && c.is_lowercase(); + let small_cap = all_small && c.is_uppercase(); + let (sc, mut form) = match style.caps { + Caps::All => (upper_single(c), Form::Normal), + _ if small && feature_substitutes(face_of(c), b"smcp", c) => (c, Form::TrueSmall), + _ if small_cap && feature_substitutes(face_of(c), b"c2sc", c) => (c, Form::TrueSmall), + _ if small => (upper_single(c), Form::FauxSmall), + _ if small_cap => (c, Form::FauxSmall), + _ => (c, Form::Normal), + }; + let face = face_of(sc); + if form == Form::Normal && style.script != Script::Normal && !sc.is_whitespace() { + let tag = if style.script == Script::Sub { b"subs" } else { b"sups" }; + if feature_substitutes(face, tag, sc) { + form = Form::TrueScript; + } + } + (b, c, sc, face, bidi.levels[b].is_rtl(), form, si) }) .collect(); // items: runs of equal (face, rtl, small, style) let mut items: Vec = Vec::new(); let mut i = 0; while i < chars.len() { - let key = |c: &(usize, char, char, FaceId, bool, bool, usize)| (c.3, c.4, c.5, c.6); + let key = |c: &(usize, char, char, FaceId, bool, Form, usize)| (c.3, c.4, c.5, c.6); let k0 = key(&chars[i]); let mut j = i + 1; while j < chars.len() && key(&chars[j]) == k0 { j += 1; } - let (face, rtl, small, si) = k0; + let (face, rtl, form, si) = k0; let style = &runs[si].1; let sub: Vec<(usize, char, char)> = chars[i..j].iter().map(|c| (c.0, c.1, c.2)).collect(); - let size = style.glyph_size() * if small { SMALL_CAPS_SCALE } else { 1.0 }; - let mut glyphs = shape_item(&sub, rtl, face, size, style, si); + let size = match form { + Form::FauxSmall => style.glyph_size() * SMALL_CAPS_SCALE, + Form::TrueScript => style.size, + _ => style.glyph_size(), + }; + let mut glyphs = shape_item(&sub, rtl, face, size, style, si, form); space_item(&mut glyphs, style, text, true); let end = if j < chars.len() { chars[j].0 } else { text.len() }; items.push(Item { range: chars[i].0..end, rtl, glyphs }); @@ -718,7 +811,7 @@ fn paragraph(text: &str, base: usize, runs: &[(Range, TextStyle)], primar face: g.face, id: g.id, x: pen + g.dx, - y: -g.dy - st.rise(), + y: -g.dy - g.rise, size: g.size, cluster: base + g.cluster, synth_bold: primaries[g.run].synth_bold || st.faux_bold, @@ -960,6 +1053,82 @@ mod tests { TextStyle { size, ..Default::default() } } + fn ids(text: &str, s: &TextStyle) -> Vec { + layout_uncached(text, s, &ParagraphStyle::default()).glyphs.iter().map(|g| g.id).collect() + } + + fn ot(f: impl FnOnce(&mut OpenType)) -> TextStyle { + let mut s = st(100.0); + f(&mut s.opentype); + s + } + + fn serif() -> TextStyle { + TextStyle { family: "Noto Serif".into(), ..st(100.0) } + } + + #[test] + fn bundled_faces_list_their_features() { + let inter = fonts::face(fonts::resolve("Inter", "Regular").face).features(); + for t in ["ss01", "dlig", "frac", "tnum", "sups", "ordn"] { + assert!(inter.iter().any(|f| f == t), "{t} in {inter:?}"); + } + let serif = fonts::face(fonts::resolve("Noto Serif", "Regular").face).features(); + for t in ["smcp", "c2sc", "onum", "lnum"] { + assert!(serif.iter().any(|f| f == t), "{t} in {serif:?}"); + } + } + + #[test] + fn opentype_features_substitute_glyphs() { + let plain = st(100.0); + // Stylistic sets, fractions, ordinals, figure widths and discretionary ligatures. + assert_ne!(ids("1234567890", &plain), ids("1234567890", &ot(|o| o.set_stylistic_set(1, true))), "ss01 alternate digits"); + assert_eq!(ids("1234567890", &plain), ids("1234567890", &ot(|o| o.set_stylistic_set(20, true))), "Inter has no ss20"); + assert_ne!(ids("1/2", &plain), ids("1/2", &ot(|o| o.fractions = true)), "frac"); + assert_ne!(ids("1a 2o No", &plain), ids("1a 2o No", &ot(|o| o.ordinals = true)), "ordn"); + let tab = measure("1111", &ot(|o| o.figure_width = effectcraft_keyframe::FigureWidth::Tabular)); + let prop = measure("1111", &ot(|o| o.figure_width = effectcraft_keyframe::FigureWidth::Proportional)); + assert!((tab - prop).abs() > 1.0, "tabular {tab} vs proportional {prop}"); + // Old-style figures (Noto Serif). + let old = TextStyle { opentype: OpenType { figure_style: effectcraft_keyframe::FigureStyle::OldStyle, ..Default::default() }, ..serif() }; + assert_ne!(ids("123", &serif()), ids("123", &old), "onum"); + // Contextual alternates are on by default; turning them off can only keep or change glyphs. + let no_calt = ot(|o| o.contextual_alternates = false); + assert_eq!(ids("Hello", &plain).len(), ids("Hello", &no_calt).len()); + } + + #[test] + fn true_small_caps_and_superscripts_with_faux_fallback() { + // Noto Serif has smcp: lowercase letters take their own small-capital glyphs at full size. + let sc = TextStyle { caps: Caps::Small, ..serif() }; + let l = layout_uncached("Ab", &sc, &ParagraphStyle::default()); + let upper_b = ids("B", &serif())[0]; + assert_ne!(l.glyphs[1].id, upper_b, "true small cap glyph, not a scaled capital"); + assert_eq!(l.glyphs[1].size, 100.0); + // Inter has no smcp: a capital scaled down. + let faux = layout_uncached("Ab", &TextStyle { caps: Caps::Small, ..st(100.0) }, &ParagraphStyle::default()); + assert_eq!(faux.glyphs[1].id, ids("B", &st(100.0))[0]); + assert!((faux.glyphs[1].size - 78.0).abs() < 0.01); + // All Small Caps: capitals become small capitals too (c2sc). + let all = TextStyle { opentype: OpenType { all_small_caps: true, ..Default::default() }, ..serif() }; + let a = layout_uncached("AB", &all, &ParagraphStyle::default()); + assert_ne!(a.glyphs[0].id, ids("A", &serif())[0]); + // Superscript: Inter's sups figures at full size on the baseline; letters it lacks are + // synthesized (smaller and raised). + let sup = TextStyle { script: Script::Super, ..st(100.0) }; + let s2 = layout_uncached("2", &sup, &ParagraphStyle::default()); + assert_ne!(s2.glyphs[0].id, ids("2", &st(100.0))[0]); + assert_eq!(s2.glyphs[0].size, 100.0); + assert_eq!(s2.glyphs[0].y, 0.0); + let sq = layout_uncached("q", &sup, &ParagraphStyle::default()); + if !feature_substitutes(sq.glyphs[0].face, b"sups", 'q') { + assert!(sq.glyphs[0].size < 60.0 && sq.glyphs[0].y < -20.0, "{:?}", sq.glyphs[0]); + } + let sub = layout_uncached("2", &TextStyle { script: Script::Sub, ..st(100.0) }, &ParagraphStyle::default()); + assert_ne!(sub.glyphs[0].id, s2.glyphs[0].id); + } + #[test] fn kerning_and_ligatures_change_advances() { let kern = measure("AVAVAV", &st(100.0)); @@ -1110,7 +1279,11 @@ mod tests { let base = st(100.0); let sup = TextStyle { script: Script::Super, ..base.clone() }; let sub = TextStyle { script: Script::Sub, ..base.clone() }; - let l = rich("x2y2", &[(0..1, base.clone()), (1..2, sup), (2..3, base.clone()), (3..4, sub)], ParagraphStyle::default()); + // A letter the font has no superior / inferior glyph for: synthesized. + let inter = fonts::resolve("Inter", "Regular").face; + let c = "qwkzvjxyQWK".chars().find(|&c| !feature_substitutes(inter, b"sups", c) && !feature_substitutes(inter, b"subs", c)).unwrap(); + let text: String = ['x', c, 'y', c].iter().collect(); + let l = rich(&text, &[(0..1, base.clone()), (1..2, sup), (2..3, base.clone()), (3..4, sub)], ParagraphStyle::default()); let g = &l.glyphs; assert!((g[1].size - 58.3).abs() < 0.1); assert!((g[1].y - (g[0].y - 33.3)).abs() < 0.1, "superscript raised by a third of the size"); diff --git a/crates/text/src/lib.rs b/crates/text/src/lib.rs index 26a659bd..beef0382 100644 --- a/crates/text/src/lib.rs +++ b/crates/text/src/lib.rs @@ -161,6 +161,7 @@ pub fn text_style(s: &CharStyle) -> TextStyle { BaselineOption::Subscript => Script::Sub, }, leading: s.leading.map(|l| l as f32), + opentype: s.opentype, } } diff --git a/crates/ui-egui/src/panels/text_panels.rs b/crates/ui-egui/src/panels/text_panels.rs index fef958a7..61c0f215 100644 --- a/crates/ui-egui/src/panels/text_panels.rs +++ b/crates/ui-egui/src/panels/text_panels.rs @@ -2,7 +2,7 @@ //! Character and Paragraph panels show and change the selected text (creating style runs); //! otherwise they apply to the whole selected text layer. -use effectcraft_engine::keyframe::{BaselineOption, Composer, Direction, Justify, Kerning, TextDoc}; +use effectcraft_engine::keyframe::{BaselineOption, Composer, Direction, FigureStyle, FigureWidth, Justify, Kerning, TextDoc}; use effectcraft_engine::render::EvalCtx; use egui::{Align2, Rect, Sense, pos2, vec2}; use serde_json::json; @@ -103,7 +103,7 @@ pub fn character(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(json!({"font": fams[i]})); } y += 28.0; - let sr = Rect::from_min_size(pos2(x0, y), vec2(w, 22.0)); + let sr = Rect::from_min_size(pos2(x0, y), vec2(w - 84.0, 22.0)); let spop = egui::Id::new("char-style-pop"); if widgets::dropdown(ui, sr, &doc.style, &t, egui::Id::new("char-style")).clicked() && enabled { widgets::open_popup(ui, spop); @@ -232,6 +232,16 @@ pub fn character(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } app.auto.add("character.ligatures", lr, "Ligatures"); p.text(pos2(x0 + 20.0, y + 7.0), Align2::LEFT_CENTER, "Ligatures", Tokens::ui(11.5), t.text_dim); + // OpenType: a popup with the font's layout features (like the Character panel's OpenType menu). + let or = Rect::from_min_size(pos2(sr.max.x + 6.0, sr.min.y), vec2(78.0, 22.0)); + let opop = egui::Id::new("char-opentype-pop"); + let n_on = doc.opentype.feature_settings().len(); + let olabel = if n_on > 0 { format!("OT ({n_on})") } else { "OpenType".to_string() }; + if widgets::dropdown(ui, or, &olabel, &t, egui::Id::new("char-opentype")).clicked() && enabled { + widgets::open_popup(ui, opop); + } + app.auto.add("character.opentype", or, "OpenType"); + opentype_popup(app, ui, opop, pos2((or.max.x - 300.0).max(rect.min.x), or.max.y), &doc, &mut actions); // Vertical type: Tate-Chu-Yoko and Standard Vertical Roman Alignment (Character panel menu). if doc.vertical { y += 22.0; @@ -267,6 +277,100 @@ pub fn character(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// The Character panel's OpenType popup: feature toggles (dimmed when the font lacks them), +/// figure styles and the twenty stylistic sets. Every row is an automation target +/// (`character.opentype.`), and every change is a `layer.setText` attribute. +fn opentype_popup(app: &mut EffectcraftApp, ui: &mut egui::Ui, id: egui::Id, pos: egui::Pos2, doc: &TextDoc, actions: &mut Vec) { + let open_id = id.with("open"); + if !ui.data(|d| d.get_temp::(open_id).unwrap_or(false)) { + return; + } + let t = app.tokens; + let feats = effectcraft_engine::font_features(&doc.font, &doc.style); + let has = |tag: &str| feats.iter().any(|f| f == tag); + let o = doc.opentype; + let mut rects: Vec<(String, Rect, String)> = vec![]; + let area = egui::Area::new(id.with("area")).order(egui::Order::Foreground).fixed_pos(pos).show(ui.ctx(), |ui| { + egui::Frame::popup(ui.style()).show(ui, |ui| { + ui.set_min_width(230.0); + ui.label(egui::RichText::new(format!("{} {}", doc.font, doc.style)).small().color(t.text_dim)); + let toggles: [(&str, &str, bool, &[&str]); 12] = [ + ("ligatures", "Standard Ligatures", doc.ligatures, &["liga", "clig"]), + ("discretionaryLigatures", "Discretionary Ligatures", o.discretionary_ligatures, &["dlig"]), + ("contextualAlternates", "Contextual Alternates", o.contextual_alternates, &["calt"]), + ("stylisticAlternates", "Stylistic Alternates", o.stylistic_alternates, &["salt"]), + ("swash", "Swash", o.swash, &["swsh"]), + ("titling", "Titling Alternates", o.titling, &["titl"]), + ("ordinals", "Ordinals", o.ordinals, &["ordn"]), + ("fractions", "Fractions", o.fractions, &["frac"]), + ("smallCaps", "Small Caps", doc.small_caps, &["smcp"]), + ("allSmallCaps", "All Small Caps", o.all_small_caps, &["c2sc"]), + ("superscript", "Superscript / Superior", doc.baseline == BaselineOption::Superscript, &["sups"]), + ("subscript", "Subscript / Inferior", doc.baseline == BaselineOption::Subscript, &["subs"]), + ]; + for (key, label, on, tags) in toggles { + let avail = tags.iter().any(|g| has(g)); + // Unavailable features stay clickable: small caps and scripts are synthesized. + let synth = matches!(key, "smallCaps" | "allSmallCaps" | "superscript" | "subscript"); + let text = match (avail, synth) { + (true, _) => label.to_string(), + (false, true) => format!("{label} (synthesized)"), + (false, false) => format!("{label} (not in font)"), + }; + let r = ui.selectable_label(on, egui::RichText::new(text).color(if avail { t.text } else { t.text_faint })); + rects.push((format!("character.opentype.{key}"), r.rect, label.to_string())); + if r.clicked() { + actions.push(json!({key: !on})); + } + } + ui.separator(); + ui.label(egui::RichText::new("Figures").small().color(t.text_dim)); + let figs = [ + ("default", "Default Figure Style", FigureStyle::Default, FigureWidth::Default), + ("tabularLining", "Tabular Lining", FigureStyle::Lining, FigureWidth::Tabular), + ("proportionalOldstyle", "Proportional Oldstyle", FigureStyle::OldStyle, FigureWidth::Proportional), + ("proportionalLining", "Proportional Lining", FigureStyle::Lining, FigureWidth::Proportional), + ("tabularOldstyle", "Tabular Oldstyle", FigureStyle::OldStyle, FigureWidth::Tabular), + ]; + for (key, label, st, fw) in figs { + let on = o.figure_style == st && o.figure_width == fw; + let r = ui.selectable_label(on, label); + rects.push((format!("character.opentype.figures.{key}"), r.rect, label.to_string())); + if r.clicked() { + actions.push(json!({"figures": key})); + } + } + ui.separator(); + ui.label(egui::RichText::new("Stylistic Sets").small().color(t.text_dim)); + egui::Grid::new(id.with("sets")).spacing(vec2(3.0, 3.0)).show(ui, |ui| { + for n in 1..=20u32 { + let key = format!("ss{n:02}"); + let on = o.stylistic_set(n); + let avail = has(&key); + let r = ui.add_sized( + vec2(38.0, 20.0), + egui::Button::selectable(on, egui::RichText::new(format!("{n}")).color(if avail || on { t.text } else { t.text_faint })), + ); + rects.push((format!("character.opentype.{key}"), r.rect, format!("Stylistic Set {n}"))); + if r.clicked() { + actions.push(json!({key: !on})); + } + if n % 5 == 0 { + ui.end_row(); + } + } + }); + }); + }); + for (aid, r, label) in rects { + app.auto.add(&aid, r, &label); + } + let outside = ui.input(|i| i.pointer.any_pressed()) && !area.response.contains_pointer() && !area.response.hovered(); + if outside || ui.input(|i| i.key_pressed(egui::Key::Escape)) { + ui.data_mut(|d| d.insert_temp(open_id, false)); + } +} + /// The seven Paragraph panel alignment buttons: (justify, `layer.setText` key). const ALIGNS: [(Justify, &str); 7] = [ (Justify::Left, "left"), From 89c9b1b6eefb7a652a6ef1b2561d236b2520227b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:07:20 -0400 Subject: [PATCH 21/36] =?UTF-8?q?M13.1:=20branching=20history,=20puppet=20?= =?UTF-8?q?pin=20recording,=20WebAssembly=20plug-in=20API,=20ScriptUI=20mo?= =?UTF-8?q?del=20and=20hosts,=20File=20=E2=96=B8=20Scripts,=20deterministi?= =?UTF-8?q?c=20render=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine: History keeps undone lines as branches; edit.history.list/goto; History panel tree; MCP history tool - puppet.recordPin / puppet.recordOptions; viewer Cmd-drag records; Record Options form - effects::plugin (EffectPlugin trait, register_plugin, API v1); effectcraft-plugin (wasmi, sandboxed, fuel-limited); effect.plugins.load/list - ScriptUI: engine scriptui model + layout + scriptui.* commands; script crate scriptui.js and script-host threads (dialogs block show()) - File ▸ Scripts: install script / ScriptUI panel, sample scripts, window.scriptPanel - host test: bit-identical renders across runs and thread counts Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 151 ++++- Cargo.toml | 3 + apps/effectcraft-web/src/store.rs | 12 + crates/automation/src/tools.rs | 19 + crates/effects/Cargo.toml | 1 + crates/effects/src/lib.rs | 28 +- crates/effects/src/plugin.rs | 338 +++++++++++ .../Create Null at Selected Layers.jsx | 41 ++ crates/engine/scripts/Rename Layers.jsx | 71 +++ crates/engine/scripts/Render Queue Batch.jsx | 52 ++ .../scripts/ScriptUI Panels/Layer Tools.jsx | 99 +++ .../scripts/Sort Layers by In Point.jsx | 37 ++ crates/engine/src/commands/edit.rs | 54 +- crates/engine/src/commands/effect.rs | 81 ++- crates/engine/src/commands/file_more.rs | 35 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/puppet.rs | 157 ++++- crates/engine/src/commands/scripts.rs | 222 +++++++ crates/engine/src/config.rs | 22 +- crates/engine/src/history.rs | 346 +++++++++++ crates/engine/src/lib.rs | 35 +- crates/engine/src/mask_track.rs | 5 +- crates/engine/src/menus.rs | 4 +- crates/engine/src/scriptui.rs | 572 ++++++++++++++++++ crates/engine/src/session_settings.rs | 6 +- crates/engine/src/tests_history.rs | 135 +++++ crates/engine/src/tests_paint.rs | 53 ++ crates/engine/src/tracking.rs | 5 +- crates/host/Cargo.toml | 6 + crates/host/src/lib.rs | 2 + crates/host/tests/determinism.rs | 74 +++ crates/plugin/Cargo.toml | 25 + crates/plugin/src/lib.rs | 66 ++ crates/plugin/src/tests.rs | 164 +++++ crates/plugin/src/wasm.rs | 151 +++++ crates/script/src/lib.rs | 15 +- crates/script/src/model.rs | 4 +- crates/script/src/runtime.rs | 170 ++++-- crates/script/src/scriptui.js | 544 +++++++++++++++++ crates/script/src/tests_ui.rs | 299 +++++++++ crates/script/src/ui.rs | 339 +++++++++++ crates/ui-egui/src/header.rs | 16 +- crates/ui-egui/src/panels/effects_presets.rs | 4 +- crates/ui-egui/src/panels/forms.rs | 13 + crates/ui-egui/src/panels/misc.rs | 49 +- crates/ui-egui/src/panels/viewer.rs | 39 +- docs/architecture.md | 7 +- xtask/src/main.rs | 1 + 48 files changed, 4440 insertions(+), 134 deletions(-) create mode 100644 crates/effects/src/plugin.rs create mode 100644 crates/engine/scripts/Create Null at Selected Layers.jsx create mode 100644 crates/engine/scripts/Rename Layers.jsx create mode 100644 crates/engine/scripts/Render Queue Batch.jsx create mode 100644 crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx create mode 100644 crates/engine/scripts/Sort Layers by In Point.jsx create mode 100644 crates/engine/src/commands/scripts.rs create mode 100644 crates/engine/src/history.rs create mode 100644 crates/engine/src/scriptui.rs create mode 100644 crates/engine/src/tests_history.rs create mode 100644 crates/host/tests/determinism.rs create mode 100644 crates/plugin/Cargo.toml create mode 100644 crates/plugin/src/lib.rs create mode 100644 crates/plugin/src/tests.rs create mode 100644 crates/plugin/src/wasm.rs create mode 100644 crates/script/src/scriptui.js create mode 100644 crates/script/src/tests_ui.rs create mode 100644 crates/script/src/ui.rs diff --git a/Cargo.lock b/Cargo.lock index c7934100..3aba6695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "effectcraft-project", "effectcraft-raster", "effectcraft-track", + "log", "rayon", "serde", "serde_json", @@ -1309,12 +1310,14 @@ dependencies = [ "effectcraft-export", "effectcraft-expr", "effectcraft-media", + "effectcraft-plugin", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", "effectcraft-script", "effectcraft-svg", "effectcraft-time", + "rayon", "serde_json", ] @@ -1398,6 +1401,18 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "effectcraft-plugin" +version = "0.1.0" +dependencies = [ + "effectcraft-effects", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-raster", + "serde_json", + "wasmi", +] + [[package]] name = "effectcraft-project" version = "0.1.0" @@ -2102,6 +2117,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -2276,7 +2297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" dependencies = [ "bytemuck", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "log", "peniko", @@ -2416,6 +2437,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -2424,7 +2454,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2435,7 +2465,7 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2753,6 +2783,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lebe" version = "0.5.3" @@ -4432,6 +4468,12 @@ dependencies = [ "serde", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -4459,6 +4501,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + [[package]] name = "strum" version = "0.28.0" @@ -5127,6 +5179,99 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2608e8bb6d67fd68f5a8d0eb1363d6e7bcbc1f8ded5a0bd3a1e382462b876b22" +dependencies = [ + "leb128fmt", + "wasmparser 0.261.0", +] + +[[package]] +name = "wasmi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78693fcdd618e0fc34af59c6b8efa9ac5d58c68df940beff4bedddb6acfe7c27" +dependencies = [ + "spin", + "wasmi_collections", + "wasmi_core", + "wasmi_ir", + "wasmparser 0.228.0", + "wat", +] + +[[package]] +name = "wasmi_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8be2aa467cf2d29e96ff759472c36eeb44a3c81c67fc9cb76c9a24c519c557" +dependencies = [ + "string-interner", +] + +[[package]] +name = "wasmi_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69372d5fda3ea3d1e0aa6603c7888110e0187e88ea17cd8fc2e2df0a0e1f37fa" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmi_ir" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17b774caa13c618c7244f1ee51fe23c5e7b8538a471fa46d9949779758aed6" +dependencies = [ + "wasmi_core", +] + +[[package]] +name = "wasmparser" +version = "0.228.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abf1132c1fdf747d56bbc1bb52152400c70f336870f968b85e89ea422198ae3" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wasmparser" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wast" +version = "261.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776443145731a4062e5b0d392892a2005909b6ab72d9fdc3cad53dd1a714e44a" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder", +] + +[[package]] +name = "wat" +version = "1.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b4d1a49ea73a8f3326e74e3a05db667001b16bd1035ed3356fc1a0ed05ca7f" +dependencies = [ + "wast", +] + [[package]] name = "wayland-backend" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 22709f60..51a2c8ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } effectcraft-opusenc = { path = "crates/opusenc" } effectcraft-script = { path = "crates/script" } +effectcraft-plugin = { path = "crates/plugin" } # foundation thiserror = "2" @@ -52,6 +53,8 @@ proptest = "1" # scripting (expressions) boa_engine = { version = "0.22", default-features = false } +# WebAssembly effect plug-ins (sandboxed interpreter; MIT/Apache-2.0, pure Rust) +wasmi = { version = "2", default-features = false, features = ["std", "validate", "wat", "deterministic", "portable-dispatch"] } # geometry / fonts kurbo = { version = "0.13", features = ["serde"] } diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs index 0e4ea029..471d96b5 100644 --- a/apps/effectcraft-web/src/store.rs +++ b/apps/effectcraft-web/src/store.rs @@ -216,6 +216,18 @@ impl ConfigStore for WebConfig { self.store.changed(); Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{CONFIG}{}/", dir.trim_end_matches('/')); + let mut v: Vec = self + .store + .lock() + .list(&pre) + .into_iter() + .filter_map(|(k, ..)| k.strip_prefix(&pre).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string)) + .collect(); + v.sort(); + v + } fn dir(&self) -> Option { Some(PathBuf::from("/")) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 739c5524..f9034a38 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -315,6 +315,18 @@ fn redo(b: &mut Backend, a: &Value) -> Result { history(b, a, "edit.redo") } +fn history_tool(b: &mut Backend, a: &Value) -> Result { + if let Some(g) = get(a, "goto") { + let p = match g { + Value::String(id) if id.parse::().is_err() => json!({"id": id}), + Value::String(n) => json!({"index": n.parse::().unwrap_or(0)}), + v => json!({"index": v}), + }; + b.exec("edit.history.goto", p)?; + } + json_reply(b.exec("edit.history.list", json!({}))?) +} + // ------------------------------------------------------------------ bridge-only tools fn screenshot(b: &mut Backend, a: &Value) -> Result { @@ -535,6 +547,13 @@ static TOOLS: &[ToolDef] = &[ schema: || schema(json!({"steps": {"type": "integer", "minimum": 1}}), &[]), run: redo, }, + ToolDef { + name: "history", + description: "The branching undo history (History panel): without arguments lists every state {index, id, label, parent, depth (0 = working line, >0 = branch), current, future}; undoing and then editing keeps the undone states as a branch. With `goto` (an index or id from the listing) jumps to that state, on any branch, without losing the others.", + bridge_only: false, + schema: || schema(json!({"goto": {"type": ["integer", "string"], "description": "State index or id to jump to (edit.history.goto)."}}), &[]), + run: history_tool, + }, ToolDef { name: "screenshot", description: "Screenshot of the live app window, or one panel (`Composition`, `Timeline`, `EffectControls`, `Project`, ...) or element id, as PNG (longest side `max_side`, default 1600). Use it to check what the UI looks like after an action.", diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index 965cdbd2..9b065360 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -19,4 +19,5 @@ effectcraft-track = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +log = { workspace = true } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..663b4bf7 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -41,6 +41,7 @@ pub mod ocio_config; pub mod paint; mod perspective; mod perspective2; +pub mod plugin; pub mod psim; pub mod puppet; pub mod roto; @@ -423,14 +424,37 @@ pub fn registry() -> &'static [EffectSpec] { }) } +/// A built-in effect or a registered plug-in by id. pub fn find(id: &str) -> Option<&'static EffectSpec> { - registry().iter().find(|s| s.id == id) + registry().iter().find(|s| s.id == id).or_else(|| plugin::plugins().into_iter().find(|s| s.id == id)) +} + +/// Every effect: the built-ins ([`registry`]) and the registered plug-ins +/// ([`plugin::register_plugin`]), sorted by category then name. +pub fn all() -> Vec<&'static EffectSpec> { + let plugins = plugin::plugins(); + let mut v: Vec<&'static EffectSpec> = registry().iter().collect(); + if !plugins.is_empty() { + v.extend(plugins); + v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); + } + v +} + +/// Effects & Presets categories: [`CATEGORIES`] then the plug-ins' own, alphabetically. +pub fn categories() -> Vec<&'static str> { + let mut v: Vec<&'static str> = CATEGORIES.to_vec(); + let mut extra: Vec<&'static str> = plugin::plugins().iter().map(|s| s.category).filter(|c| !CATEGORIES.contains(c)).collect(); + extra.sort_unstable(); + extra.dedup(); + v.extend(extra); + v } /// Find by id or (case-insensitive) display name. pub fn lookup(name_or_id: &str) -> Option<&'static EffectSpec> { find(name_or_id) - .or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) + .or_else(|| all().into_iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) .or_else(|| { // After Effects' third-party display names we register under a generic name. keylight::KEYLIGHT_ALIASES.iter().any(|a| a.eq_ignore_ascii_case(name_or_id)).then(|| find("ec.keying.keylight")).flatten() diff --git a/crates/effects/src/plugin.rs b/crates/effects/src/plugin.rs new file mode 100644 index 00000000..242f52b9 --- /dev/null +++ b/crates/effects/src/plugin.rs @@ -0,0 +1,338 @@ +//! The effect plug-in API, version [`PLUGIN_API_VERSION`]. +//! +//! A plug-in is anything implementing [`EffectPlugin`]: a [`PluginManifest`] (stable id, +//! display name, Effects & Presets category, typed parameters) and a `render` that processes a +//! [`PluginFrame`] in place. [`register_plugin`] turns it into an ordinary [`EffectSpec`]: +//! instances are property groups like every built-in effect, so plug-in parameters animate, +//! take expressions, show in Effect Controls and are addressable by agents, and the effect is +//! applied by id (`effect.apply {"effect": "org.example.posterize"}`) like a built-in. +//! +//! Plug-ins come from Rust code (implement the trait and register it at start-up) or from +//! sandboxed WebAssembly modules (`effectcraft-plugin`, see `docs/plugins.md`), which implement +//! the same contract through a small C-like ABI. +//! +//! The contract (stable within an API version): +//! * pixels are premultiplied RGBA `f32` (0–1, may exceed 1 in 32 bpc projects), row-major, +//! in the layer's buffer, which may be scaled (preview resolution) and padded; +//! * parameters are flattened to `f64`s in declaration order: slider / angle / checkbox (0 or 1) +//! / popup (0-based index) take one value, point two (**frame pixels**, already mapped to the +//! buffer), colour four (RGBA 0–1); +//! * `time` is layer time in seconds; `scale` is the buffer's pixels per layer pixel (scale +//! pixel distances like blur radii by it); +//! * rendering must be deterministic (same inputs, same output) and may run on several threads +//! at once. + +use std::sync::{Arc, OnceLock, RwLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use serde::{Deserialize, Serialize}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, RenderFn}; + +/// The plug-in API version this build implements. +pub const PLUGIN_API_VERSION: u32 = 1; + +/// The most plug-in effects one process can register. +pub const MAX_PLUGINS: usize = 64; + +/// A plug-in parameter's kind, default and range. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum PluginParamKind { + Slider { + default: f64, + #[serde(default)] + min: f64, + #[serde(default = "hundred")] + max: f64, + /// Slider range (defaults to min..max). + #[serde(default, rename = "sliderMin")] + slider_min: Option, + #[serde(default, rename = "sliderMax")] + slider_max: Option, + #[serde(default = "two")] + decimals: u8, + }, + Angle { + #[serde(default)] + default: f64, + }, + Checkbox { + #[serde(default)] + default: bool, + }, + Popup { + options: Vec, + /// 0-based index. + #[serde(default)] + default: u32, + }, + /// Default as fractions of the layer size (`[0.5, 0.5]` = centre). + Point { + #[serde(default = "centre")] + default: [f64; 2], + }, + /// RGBA, 0–1. + Color { + #[serde(default = "white")] + default: [f64; 4], + }, +} + +fn hundred() -> f64 { + 100.0 +} +fn two() -> u8 { + 2 +} +fn centre() -> [f64; 2] { + [0.5, 0.5] +} +fn white() -> [f64; 4] { + [1.0; 4] +} + +impl PluginParamKind { + /// How many `f64`s the parameter flattens to. + pub fn width(&self) -> usize { + match self { + PluginParamKind::Point { .. } => 2, + PluginParamKind::Color { .. } => 4, + _ => 1, + } + } +} + +/// One parameter of a plug-in effect. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginParam { + /// Stable id (the property's match name; letters, digits, `_`). + pub id: String, + pub name: String, + #[serde(flatten)] + pub kind: PluginParamKind, +} + +/// What a plug-in declares about itself. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginManifest { + /// The plug-in API version it was written for (must be [`PLUGIN_API_VERSION`]). + pub api: u32, + /// Stable, reverse-domain effect id (`org.example.posterize`); `ec.` is reserved. + pub id: String, + pub name: String, + /// Effects & Presets category: one of [`crate::CATEGORIES`] or a new one. + pub category: String, + #[serde(default)] + pub version: String, + #[serde(default)] + pub author: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub params: Vec, +} + +impl PluginManifest { + /// Check the manifest against this API version. + pub fn validate(&self) -> Result<(), String> { + if self.api != PLUGIN_API_VERSION { + return Err(format!("plug-in `{}` targets plug-in API {} but this build implements API {PLUGIN_API_VERSION}", self.id, self.api)); + } + let ok_id = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); + if !ok_id(&self.id) || self.id.starts_with("ec.") { + return Err(format!("bad plug-in id `{}` (reverse-domain, letters, digits, `.`, `_`, `-`; `ec.` is reserved)", self.id)); + } + if self.name.trim().is_empty() || self.category.trim().is_empty() { + return Err(format!("plug-in `{}` needs a name and a category", self.id)); + } + let mut seen = std::collections::HashSet::new(); + for p in &self.params { + if p.id.is_empty() || !p.id.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') || !seen.insert(p.id.as_str()) { + return Err(format!("plug-in `{}`: bad or duplicate parameter id `{}`", self.id, p.id)); + } + if let PluginParamKind::Popup { options, default } = &p.kind + && (options.is_empty() || *default as usize >= options.len()) + { + return Err(format!("plug-in `{}`: popup `{}` needs options and a default index in range", self.id, p.id)); + } + } + Ok(()) + } + + /// Total flattened parameter count. + pub fn flat_len(&self) -> usize { + self.params.iter().map(|p| p.kind.width()).sum() + } +} + +/// The pixels a plug-in renders, in place. +pub struct PluginFrame<'a> { + pub width: u32, + pub height: u32, + /// Premultiplied RGBA, row-major, `width * height` pixels. + pub pixels: &'a mut [[f32; 4]], + /// Buffer pixels per layer pixel (preview resolution). + pub scale: f64, +} + +/// Parameter values for one render, flattened (see the module docs) with typed accessors. +pub struct PluginParams<'a> { + pub manifest: &'a PluginManifest, + pub flat: &'a [f64], +} + +impl PluginParams<'_> { + fn slot(&self, id: &str) -> Option<&[f64]> { + let mut at = 0; + for p in &self.manifest.params { + let w = p.kind.width(); + if p.id == id { + return self.flat.get(at..at + w); + } + at += w; + } + None + } + /// Slider / angle / popup index / checkbox (0 or 1). + pub fn f(&self, id: &str) -> f64 { + self.slot(id).and_then(|s| s.first().copied()).unwrap_or(0.0) + } + pub fn b(&self, id: &str) -> bool { + self.f(id) != 0.0 + } + /// Point in frame pixels. + pub fn point(&self, id: &str) -> [f64; 2] { + self.slot(id).map(|s| [s[0], s[1]]).unwrap_or([0.0; 2]) + } + pub fn color(&self, id: &str) -> [f64; 4] { + self.slot(id).map(|s| [s[0], s[1], s[2], s[3]]).unwrap_or([1.0; 4]) + } +} + +/// An effect plug-in. +pub trait EffectPlugin: Send + Sync { + fn manifest(&self) -> &PluginManifest; + /// Process `frame` in place. `time` is layer time in seconds. An error leaves the frame as + /// it was (the error is logged once). + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String>; + /// Where the plug-in came from (a file path, `built-in`…), for listings. + fn source(&self) -> String { + "built-in".into() + } +} + +static SLOTS: [OnceLock>; MAX_PLUGINS] = [const { OnceLock::new() }; MAX_PLUGINS]; + +/// Registered plug-in effects, in registration order. +fn plugin_specs() -> &'static RwLock> { + static P: OnceLock>> = OnceLock::new(); + P.get_or_init(|| RwLock::new(vec![])) +} + +/// The registered plug-in effects. +pub fn plugins() -> Vec<&'static EffectSpec> { + plugin_specs().read().map(|v| v.clone()).unwrap_or_default() +} + +/// The plug-in behind a registered effect id. +pub fn plugin(id: &str) -> Option> { + SLOTS.iter().filter_map(|s| s.get()).find(|p| p.manifest().id == id).cloned() +} + +fn leak(s: &str) -> &'static str { + Box::leak(s.to_string().into_boxed_str()) +} + +fn param_spec(p: &PluginParam) -> ParamSpec { + let (default, ui) = match &p.kind { + PluginParamKind::Slider { default, min, max, slider_min, slider_max, decimals } => ( + Value::Scalar(*default), + ParamUi::Slider { min: *min, max: *max, slider_min: slider_min.unwrap_or(*min), slider_max: slider_max.unwrap_or(*max), decimals: *decimals }, + ), + PluginParamKind::Angle { default } => (Value::Scalar(*default), ParamUi::Angle), + PluginParamKind::Checkbox { default } => (Value::Bool(*default), ParamUi::Checkbox), + PluginParamKind::Popup { options, default } => (Value::Enum(*default), ParamUi::Popup { options: options.clone() }), + PluginParamKind::Point { default } => (Value::Vec2(*default), ParamUi::Point), + PluginParamKind::Color { default } => (Value::Color(*default), ParamUi::Color), + }; + ParamSpec { id: leak(&p.id), name: leak(&p.name), default, ui } +} + +/// Register a plug-in effect: it appears in Effects & Presets under its category and is +/// applied by id like a built-in. Fails for a bad manifest, a taken id or when +/// [`MAX_PLUGINS`] are registered. +pub fn register_plugin(plugin: Arc) -> Result<&'static EffectSpec, String> { + let m = plugin.manifest().clone(); + m.validate()?; + let mut specs = plugin_specs().write().map_err(|_| "plug-in registry poisoned".to_string())?; + if crate::registry().iter().any(|s| s.id == m.id) || specs.iter().any(|s| s.id == m.id) { + return Err(format!("an effect with id `{}` is already registered", m.id)); + } + let slot = SLOTS.iter().position(|s| s.get().is_none()).ok_or_else(|| format!("too many plug-ins (at most {MAX_PLUGINS})"))?; + SLOTS[slot].set(plugin).map_err(|_| "plug-in slot taken".to_string())?; + let spec: &'static EffectSpec = Box::leak(Box::new(EffectSpec { + id: leak(&m.id), + name: leak(&m.name), + category: leak(&m.category), + params: m.params.iter().map(param_spec).collect(), + render: TRAMPOLINES[slot], + gpu: false, + float: true, + })); + specs.push(spec); + Ok(spec) +} + +/// Each slot needs its own `fn` (effect render functions are plain function pointers). +fn trampoline(ctx: &EffectCtx, buf: Buf) -> Buf { + run_slot(I, ctx, buf) +} + +macro_rules! trampolines { + ($($i:literal)*) => { [$(trampoline::<$i> as RenderFn),*] }; +} + +const TRAMPOLINES: [RenderFn; MAX_PLUGINS] = trampolines!( + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 +); + +/// Flatten an instance's evaluated parameters for the plug-in (points mapped to frame pixels). +pub fn flatten(m: &PluginManifest, ctx: &EffectCtx, buf: &Buf) -> Vec { + let mut out = Vec::with_capacity(m.flat_len()); + for p in &m.params { + let v = ctx.params.get(&p.id); + match &p.kind { + PluginParamKind::Point { .. } => { + let q = v.map(Value::as_vec2).unwrap_or([0.0; 2]); + let (x, y) = buf.to_px(q); + out.extend([x, y]); + } + PluginParamKind::Color { .. } => { + let c = v.map(Value::as_color).unwrap_or([1.0; 4]); + out.extend(c.iter().map(|x| *x as f64)); + } + PluginParamKind::Checkbox { .. } => out.push(if v.map(Value::as_bool).unwrap_or(false) { 1.0 } else { 0.0 }), + PluginParamKind::Popup { .. } => out.push(v.map(Value::as_enum).unwrap_or(0) as f64), + _ => out.push(v.map(Value::as_f64).unwrap_or(0.0)), + } + } + out +} + +fn run_slot(slot: usize, ctx: &EffectCtx, mut buf: Buf) -> Buf { + let Some(plugin) = SLOTS[slot].get() else { return buf }; + let m = plugin.manifest(); + let flat = flatten(m, ctx, &buf); + let params = PluginParams { manifest: m, flat: &flat }; + let (width, height) = (buf.img.width, buf.img.height); + let before = buf.img.data.clone(); + let mut frame = PluginFrame { width, height, pixels: &mut buf.img.data, scale: buf.scale }; + if let Err(e) = plugin.render(&mut frame, ¶ms, ctx.time) { + log::warn!("plug-in `{}`: {e}", m.id); + buf.img.data = before; + } + buf +} diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx b/crates/engine/scripts/Create Null at Selected Layers.jsx new file mode 100644 index 00000000..44442c05 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx @@ -0,0 +1,41 @@ +// Create Null at Selected Layers — an EffectCraft sample script (original work, MIT OR +// Apache-2.0). +// +// Adds a null at the centroid of the selected layers' positions (at the current time) and +// parents the layers that have no parent to it (set PARENT below to false to skip that). +// One undo step. + +(function createNullAtCentroid() { + var PARENT = true; + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Create Null: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + alert("Create Null: select one or more layers."); + return; + } + var t = comp.time; + var sum = [0, 0, 0]; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.valueAtTime(t, false); + sum[0] += p[0]; + sum[1] += p[1]; + sum[2] += p.length > 2 ? p[2] : 0; + } + var c = [sum[0] / layers.length, sum[1] / layers.length, sum[2] / layers.length]; + + app.beginUndoGroup("Create Null at Selected Layers"); + var nul = comp.layers.addNull(); + nul.name = "Centroid"; + nul.transform.position.setValue(c); + if (PARENT) { + for (var k = 0; k < layers.length; k++) { + if (!layers[k].parent) layers[k].setParentWithJump(nul); + } + } + app.endUndoGroup(); + writeLn("Null at [" + c[0].toFixed(1) + ", " + c[1].toFixed(1) + "]"); +})(); diff --git a/crates/engine/scripts/Rename Layers.jsx b/crates/engine/scripts/Rename Layers.jsx new file mode 100644 index 00000000..6de9d4d4 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx @@ -0,0 +1,71 @@ +// Rename Layers — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Renames the selected layers of the active composition (all layers when none are selected): +// find & replace, then an optional prefix, suffix and running number. One undo step. + +(function renameLayers() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Rename Layers: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + + var dlg = new Window("dialog", "Rename Layers"); + dlg.orientation = "column"; + dlg.alignChildren = ["fill", "top"]; + + var find = dlg.add("panel", undefined, "Find and Replace"); + find.alignChildren = ["fill", "top"]; + var row1 = find.add("group"); + row1.add("statictext", undefined, "Find:"); + var findText = row1.add("edittext", undefined, ""); + findText.characters = 20; + var row2 = find.add("group"); + row2.add("statictext", undefined, "Replace:"); + var replaceText = row2.add("edittext", undefined, ""); + replaceText.characters = 20; + + var extra = dlg.add("panel", undefined, "Add"); + extra.alignChildren = ["fill", "top"]; + var row3 = extra.add("group"); + row3.add("statictext", undefined, "Prefix:"); + var prefix = row3.add("edittext", undefined, ""); + prefix.characters = 12; + row3.add("statictext", undefined, "Suffix:"); + var suffix = row3.add("edittext", undefined, ""); + suffix.characters = 12; + var number = extra.add("checkbox", undefined, "Number the layers"); + var row4 = extra.add("group"); + row4.add("statictext", undefined, "Start at:"); + var start = row4.add("edittext", undefined, "1"); + start.characters = 5; + start.enabled = false; + number.onClick = function () { + start.enabled = number.value; + }; + + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "OK", { name: "ok" }); + + if (dlg.show() !== 1) return; + + app.beginUndoGroup("Rename Layers"); + var n = parseInt(start.text, 10); + if (isNaN(n)) n = 1; + for (var k = 0; k < layers.length; k++) { + var name = layers[k].name; + if (findText.text !== "") name = name.split(findText.text).join(replaceText.text); + name = prefix.text + name + suffix.text; + if (number.value) name += " " + (n + k); + layers[k].name = name; + } + app.endUndoGroup(); + writeLn("Renamed " + layers.length + " layer(s)."); +})(); diff --git a/crates/engine/scripts/Render Queue Batch.jsx b/crates/engine/scripts/Render Queue Batch.jsx new file mode 100644 index 00000000..1dc5ca4c --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx @@ -0,0 +1,52 @@ +// Render Queue Batch — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Adds the compositions selected in the Project panel (every composition when none are +// selected) to the Render Queue, each with an output file named after the comp in a folder you +// choose, and optionally starts rendering. + +(function renderQueueBatch() { + var comps = []; + var sel = app.project.selection; + for (var i = 0; i < sel.length; i++) if (sel[i] instanceof CompItem) comps.push(sel[i]); + if (comps.length === 0) { + for (var j = 1; j <= app.project.numItems; j++) { + var it = app.project.item(j); + if (it instanceof CompItem) comps.push(it); + } + } + if (comps.length === 0) { + alert("Render Queue Batch: the project has no compositions."); + return; + } + + var dlg = new Window("dialog", "Render Queue Batch"); + dlg.alignChildren = ["fill", "top"]; + dlg.add("statictext", undefined, comps.length + " composition(s) will be queued."); + var row = dlg.add("group"); + row.add("statictext", undefined, "Output folder:"); + var folder = row.add("edittext", undefined, Folder.desktop ? Folder.desktop.fsName : ""); + folder.characters = 30; + var fmtRow = dlg.add("group"); + fmtRow.add("statictext", undefined, "Format:"); + var format = fmtRow.add("dropdownlist", undefined, ["H.264 (.mp4)", "PNG Sequence"]); + format.selection = 0; + var go = dlg.add("checkbox", undefined, "Start rendering"); + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "Queue", { name: "ok" }); + if (dlg.show() !== 1) return; + + var png = format.selection && format.selection.index === 1; + app.beginUndoGroup("Render Queue Batch"); + for (var k = 0; k < comps.length; k++) { + var rq = app.project.renderQueue.items.add(comps[k]); + var name = comps[k].name.replace(/[\\\/:*?"<>|]/g, "_"); + var om = rq.outputModule(1); + if (png) om.format = "png"; + om.file = new File(folder.text + "/" + name + (png ? "_[#####].png" : ".mp4")); + } + app.endUndoGroup(); + writeLn("Queued " + comps.length + " composition(s)."); + if (go.value) app.project.renderQueue.render(); +})(); diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx new file mode 100644 index 00000000..28991849 --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx @@ -0,0 +1,99 @@ +// Layer Tools — an EffectCraft sample ScriptUI panel (original work, MIT OR Apache-2.0). +// +// Installed in the ScriptUI Panels folder it docks like any panel (Window ▸ Layer Tools.jsx); +// run from File ▸ Scripts it opens as a floating palette. Buttons act on the selected layers of +// the active composition. + +(function layerTools(thisObj) { + var ui = thisObj instanceof Panel ? thisObj : new Window("palette", "Layer Tools", undefined, { resizeable: true }); + ui.orientation = "column"; + ui.alignChildren = ["fill", "top"]; + ui.spacing = 6; + + function comp() { + var c = app.project.activeItem; + if (!(c instanceof CompItem)) { + status.text = "Open a composition first."; + return null; + } + return c; + } + function selected(c) { + var l = c.selectedLayers; + if (l.length === 0) status.text = "Select some layers."; + return l; + } + + var labelRow = ui.add("group"); + labelRow.add("statictext", undefined, "Opacity:"); + var opacity = labelRow.add("slider", undefined, 100, 0, 100); + opacity.preferredSize.width = 120; + var opacityValue = labelRow.add("statictext", undefined, "100%"); + opacityValue.characters = 5; + opacity.onChanging = function () { + opacityValue.text = Math.round(opacity.value) + "%"; + }; + opacity.onChange = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Set Opacity"); + for (var i = 0; i < layers.length; i++) layers[i].transform.opacity.setValue(Math.round(opacity.value)); + app.endUndoGroup(); + status.text = "Opacity " + Math.round(opacity.value) + "% on " + layers.length + " layer(s)."; + }; + + var shy = ui.add("checkbox", undefined, "Shy"); + shy.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Shy Layers"); + for (var i = 0; i < layers.length; i++) layers[i].shy = shy.value; + app.endUndoGroup(); + }; + + var nameRow = ui.add("group"); + nameRow.alignChildren = ["fill", "center"]; + var newName = nameRow.add("edittext", undefined, "Layer"); + newName.characters = 14; + var renameBtn = nameRow.add("button", undefined, "Rename"); + renameBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Rename Layers"); + for (var i = 0; i < layers.length; i++) layers[i].name = layers.length > 1 ? newName.text + " " + (i + 1) : newName.text; + app.endUndoGroup(); + status.text = "Renamed " + layers.length + " layer(s)."; + }; + + var nullBtn = ui.add("button", undefined, "Null at Centroid"); + nullBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + if (layers.length === 0) return; + var x = 0, y = 0; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.value; + x += p[0]; + y += p[1]; + } + app.beginUndoGroup("Null at Centroid"); + var n = c.layers.addNull(); + n.transform.position.setValue([x / layers.length, y / layers.length]); + app.endUndoGroup(); + status.text = "Added " + n.name + "."; + }; + + var status = ui.add("statictext", undefined, "Select layers, then pick a tool."); + status.characters = 30; + + if (ui instanceof Window) { + ui.center(); + ui.show(); + } else { + ui.layout.layout(true); + } +})(this); diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx b/crates/engine/scripts/Sort Layers by In Point.jsx new file mode 100644 index 00000000..c59473d7 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx @@ -0,0 +1,37 @@ +// Sort Layers by In Point — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Reorders the selected layers of the active composition (all layers when none are selected) +// so the layer that starts first is on top. Layers with the same in point keep their order. +// One undo step. + +(function sortLayersByInPoint() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Sort Layers by In Point: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + if (layers.length < 2) return; + + // The slots the layers occupy now (stack indices, top first). + var slots = layers.map(function (l) { return l.index; }).sort(function (a, b) { return a - b; }); + var order = layers + .map(function (l, k) { return { layer: l, inPoint: l.inPoint, k: k }; }) + .sort(function (a, b) { return a.inPoint - b.inPoint || a.k - b.k; }); + + app.beginUndoGroup("Sort Layers by In Point"); + // Place each layer, earliest first, in the next slot from the top. + for (var j = 0; j < order.length; j++) { + var target = slots[j]; + var layer = order[j].layer; + if (layer.index === target) continue; + var anchor = comp.layer(target); + layer.moveBefore(anchor); + } + app.endUndoGroup(); + writeLn("Sorted " + layers.length + " layer(s) by in point."); +})(); diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index 03cd8087..a76c2a27 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -584,8 +584,7 @@ fn edit_original(s: &mut Session, _: &Value) -> Result { } fn purge(s: &mut Session, _: &Value) -> Result { - s.history.undo.clear(); - s.history.redo.clear(); + s.history.clear(); s.toast("Purged undo history"); Ok(Value::Null) } @@ -630,9 +629,60 @@ pub fn specs() -> Vec { cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), + crate::query!( + "edit.history.list", + "History", + "{} → {states: [{index, id, label, parent, depth, current, line, future}], current, branches}", + history_list + ), + cmd!( + "edit.history.goto", + "Go to History State", + [], + None, + "{index? (from edit.history.list) | id? | steps? (negative = back along the line)}", + always_ok, + history_goto + ), ] } +// ---------------------------------------------------------------- History panel + +/// `edit.history.list`: every state of the (branching) undo history. +fn history_list(s: &mut Session, _: &Value) -> Result { + let nodes = s.history_tree(); + let current = nodes.iter().position(|n| n.current); + Ok(json!({"states": nodes, "current": current, "branches": s.history.branches.len()})) +} + +/// `edit.history.goto`: jump to a state by `index` (from `edit.history.list`), `id`, or +/// `steps` along the working line (negative = back, like Undo). +fn history_goto(s: &mut Session, p: &Value) -> Result { + let c = "edit.history.goto"; + if let Some(n) = p.get("steps").and_then(Value::as_i64) { + for _ in 0..n.unsigned_abs() { + let moved = if n < 0 { s.undo() } else { s.redo() }; + if !moved { + break; + } + } + } else { + let nodes = s.history_tree(); + let id = match (p.get("id").and_then(Value::as_str), p.get("index").and_then(Value::as_u64)) { + (Some(id), _) => id.to_string(), + (None, Some(i)) => nodes.get(i as usize).map(|n| n.id.clone()).ok_or_else(|| super::bad(c, format!("no state {i} (0..{})", nodes.len())))?, + _ => return Err(super::bad(c, "give `index`, `id` or `steps`")), + }; + if !s.goto_history(&id) { + return Err(super::bad(c, format!("no history state `{id}`"))); + } + } + let nodes = s.history_tree(); + let cur = nodes.iter().find(|n| n.current).cloned(); + Ok(json!({"current": cur.as_ref().map(|n| n.index), "label": cur.map(|n| n.label)})) +} + fn layers_or_keys(s: &Session) -> std::result::Result<(), String> { if !s.state.selected_keys.is_empty() || !s.state.selected_vertices.is_empty() { super::has_comp(s) } else { has_layers(s) } } diff --git a/crates/engine/src/commands/effect.rs b/crates/engine/src/commands/effect.rs index b0fb8c31..6ce73ac3 100644 --- a/crates/engine/src/commands/effect.rs +++ b/crates/engine/src/commands/effect.rs @@ -7,6 +7,68 @@ use serde_json::{Value, json}; use super::{CommandSpec, b_p, bad, has_layers, layer_mut, layers_p, str_p}; use crate::{EngineError, Result, Session, cmd}; +/// `effect.plugins.list`: the registered plug-in effects (WebAssembly or Rust). +fn plugins_list(s: &mut Session, _: &Value) -> Result { + use effectcraft_effects::plugin; + let list: Vec = plugin::plugins() + .into_iter() + .map(|spec| { + let p = plugin::plugin(spec.id); + let m = p.as_ref().map(|p| p.manifest().clone()); + json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()), + "author": m.as_ref().map(|m| m.author.clone()), + "description": m.as_ref().map(|m| m.description.clone()), + "source": p.map(|p| p.source()), + "params": spec.params.iter().map(|p| p.id).collect::>(), + }) + }) + .collect(); + Ok(json!({"api": plugin::PLUGIN_API_VERSION, "wasm": s.plugin_loader.is_some(), "plugins": list})) +} + +/// `effect.plugins.load`: load WebAssembly effect plug-ins (a file, or every `.wasm` in a +/// folder) through the host's loader ([`Session::plugin_loader`]). +fn plugins_load(s: &mut Session, p: &Value) -> Result { + let c = "effect.plugins.load"; + let loader = s.plugin_loader.ok_or_else(|| EngineError::Other("WebAssembly effect plug-ins are not available in this build".into()))?; + let files: Vec = match (str_p(p, "path"), str_p(p, "folder")) { + (Some(path), _) => vec![path.to_string()], + (None, Some(dir)) => { + let mut v: Vec = std::fs::read_dir(dir) + .map_err(|e| bad(c, format!("cannot list {dir}: {e}")))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("wasm"))) + .map(|p| p.to_string_lossy().to_string()) + .collect(); + v.sort(); + v + } + _ => return Err(bad(c, "give `path` or `folder`")), + }; + let (mut loaded, mut errors) = (vec![], vec![]); + for f in &files { + let r = s.services.read_file(f).map_err(|e| format!("cannot read {f}: {e}")).and_then(|bytes| loader(&bytes, f)); + match r { + Ok(v) => loaded.push(v), + Err(e) => errors.push(json!({"path": f, "error": e})), + } + } + if str_p(p, "path").is_some() + && let Some(e) = errors.first() + { + return Err(EngineError::Other(format!("{}: {}", files[0], e["error"].as_str().unwrap_or("failed")))); + } + if !loaded.is_empty() { + s.toast(format!("Loaded {} effect plug-in(s)", loaded.len())); + } + Ok(json!({"loaded": loaded, "errors": errors})) +} + fn apply(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "effect").ok_or_else(|| bad("effect.apply", "missing `effect` (id or name)"))?; let spec = effectcraft_effects::lookup(name).ok_or_else(|| bad("effect.apply", format!("unknown effect `{name}`")))?; @@ -284,8 +346,8 @@ fn last(s: &mut Session, p: &Value) -> Result { fn list(_: &mut Session, p: &Value) -> Result { let filter = str_p(p, "filter").map(str::to_ascii_lowercase); - let v: Vec = effectcraft_effects::registry() - .iter() + let v: Vec = effectcraft_effects::all() + .into_iter() .filter(|e| filter.as_ref().is_none_or(|f| e.name.to_ascii_lowercase().contains(f) || e.id.contains(f.as_str()))) .map(|e| json!({"id": e.id, "name": e.name, "category": e.category, "params": e.params.iter().map(|p| json!({"id": p.id, "name": p.name, "default": p.default.to_json()})).collect::>()})) .collect(); @@ -305,5 +367,20 @@ pub fn specs() -> Vec { cmd!("effect.paste", "Paste Effects", [], None, "{layers?} — adds the copied effects to the layers", has_layers, paste), cmd!("effect.reset", "Reset Effect", [], None, "{layer?, effect}", has_layers, reset), crate::query!("effect.list", "List Effects", "{filter?}", list), + crate::query!( + "effect.plugins.list", + "List Effect Plug-ins", + "{} → {api, wasm, plugins: [{id, name, category, version, author, source, params}]}", + plugins_list + ), + cmd!( + "effect.plugins.load", + "Load Effect Plug-in...", + [], + None, + "{path (.wasm / .wat plug-in, API v1) | folder (loads every .wasm in it)}", + super::always, + plugins_load + ), ] } diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 6908bcd5..9175af59 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -364,6 +364,14 @@ fn collect_files(s: &mut Session, p: &Value) -> Result { /// array or JSON lines of `{"command": id, "params": {…}}` (also accepts the control channel's /// `{"method": "engine.execute", "params": {"id", "params"}}`). fn run_script(s: &mut Session, p: &Value) -> Result { + // File ▸ Scripts ▸ . + if let Some(name) = str_p(p, "name") + && p.get("path").is_none() + && p.get("steps").is_none() + { + let name = name.to_string(); + return super::scripts::run_named(s, &name); + } if let Some(path) = str_p(p, "path") && p.get("steps").is_none() && is_js_script(path) @@ -375,13 +383,7 @@ fn run_script(s: &mut Session, p: &Value) -> Result { let code = String::from_utf8_lossy(&bytes).to_string(); let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let out = run_js(s, &code, path, false)?; - if let Some(e) = out.get("error").filter(|e| !e.is_null()) { - let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); - let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); - s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); - return Err(EngineError::Other(format!("{name}{line}: {msg}"))); - } - return Ok(out); + return report_js(s, &name, out); } let steps: Vec = match (p.get("steps"), str_p(p, "path")) { (Some(Value::Array(a)), _) => a.clone(), @@ -426,6 +428,23 @@ fn is_js_script(path: &str) -> bool { l.ends_with(".jsx") || l.ends_with(".js") || l.ends_with(".jsxbin") } +/// A script error becomes a toast and a command error. +fn report_js(s: &mut Session, name: &str, out: Value) -> Result { + if let Some(e) = out.get("error").filter(|e| !e.is_null()) { + let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); + let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); + s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); + return Err(EngineError::Other(format!("{name}{line}: {msg}"))); + } + Ok(out) +} + +/// Run a script's code under its file name (installed / sample scripts, ScriptUI panels). +pub(crate) fn run_js_named(s: &mut Session, code: &str, name: &str) -> Result { + let out = run_js(s, code, name, false)?; + report_js(s, name, out) +} + /// Run JavaScript through the session's scripting engine. fn run_js(s: &mut Session, code: &str, name: &str, console: bool) -> Result { let runner = s.script.ok_or_else(|| EngineError::Other("JavaScript scripting is not available in this build".into()))?; @@ -809,7 +828,7 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}]}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}] | name (an installed or sample script, see file.scripts.list)}", always, run_script ), diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4112cf6c..16afc125 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -40,6 +40,7 @@ mod query; mod render_queue; pub(crate) mod rig3d; pub mod roto_cmds; +pub mod scripts; mod settings; mod shape_stroke; mod stubs; @@ -158,6 +159,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(essential::specs()); v.extend(expr_tools::specs()); v.extend(proxy::specs()); + v.extend(scripts::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/puppet.rs b/crates/engine/src/commands/puppet.rs index 4ada1dcd..890e1091 100644 --- a/crates/engine/src/commands/puppet.rs +++ b/crates/engine/src/commands/puppet.rs @@ -7,7 +7,7 @@ use effectcraft_effects::puppet::{self, MeshOpts, PinKind}; use effectcraft_effects::{Buf, Params}; -use effectcraft_keyframe::Value as KV; +use effectcraft_keyframe::{Keyframe, Value as KV}; use effectcraft_project::build::Ids; use effectcraft_project::{ItemId, LayerId, Node, PropGroup, Uid}; use effectcraft_render::{EvalCtx, RenderOpts, Renderer}; @@ -309,6 +309,122 @@ fn mesh_opts(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// Parse recorded samples `[[t, x, y]…]` (t = seconds since the drag began, layer space). +fn samples(p: &Value, cmd: &str) -> Result> { + let arr = p + .get("samples") + .and_then(Value::as_array) + .ok_or_else(|| bad(cmd, "missing `samples`: [[t, x, y]…] (t = seconds since the drag began, layer space)"))?; + let mut rec = Vec::with_capacity(arr.len()); + for (i, v) in arr.iter().enumerate() { + let a: Vec = v.as_array().map(|a| a.iter().filter_map(Value::as_f64).collect()).unwrap_or_default(); + if a.len() < 3 || a.iter().any(|x| !x.is_finite()) { + return Err(bad(cmd, format!("sample {i}: expected [t, x, y]"))); + } + rec.push((a[0], [a[1], a[2]])); + } + if rec.is_empty() { + return Err(bad(cmd, "need at least one sample")); + } + rec.sort_by(|a, b| a.0.total_cmp(&b.0)); + Ok(rec) +} + +/// Puppet tool Record mode (⌘/Ctrl-drag a pin): the pin's drag, sampled in real time, becomes +/// Position keyframes at the comp frame rate from the current time on, replacing the keys in +/// the recorded span. Record Options: `speed` (%: 100 plays back at the speed of the drag, +/// 200 twice as slow) and `smoothing` (removes extraneous keys like the Smoother; 0 = a key +/// on every frame). The current time stays where recording began. +fn record_pin(s: &mut Session, p: &Value) -> Result { + let cmd = "puppet.recordPin"; + let (cid, lid) = layer_p(s, p, cmd)?; + let key = p.get("pin").cloned().ok_or_else(|| bad(cmd, "missing `pin` (uid or name)"))?; + let uid = pin_uid(s, cid, lid, &key, cmd)?; + let rec = samples(p, cmd)?; + let opts = &s.state.puppet; + let speed = f_p(p, "speed").unwrap_or(opts.record_speed).clamp(1.0, 10_000.0) / 100.0; + // Smoothing 10 (the default) keeps the path within 1 layer pixel of the drag. + let smoothing = f_p(p, "smoothing").unwrap_or(opts.record_smoothing).max(0.0) / 10.0; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let fr = comp.frame_rate; + let fd = fr.frame_duration().seconds(); + let start = match p.get("start").and_then(Value::as_f64) { + Some(t) => fr.snap_nearest(Tick::from_seconds_f64(t)), + None => s.time_of(cid), + }; + let f0 = fr.frame_at(start); + let end_frame = fr.frame_at(comp.duration - comp.frame_duration()); + let t_rec0 = rec[0].0; + let span = (rec[rec.len() - 1].0 - t_rec0) * speed; + let nframes = (span / fd).round().max(0.0) as i64; + let at = |secs: f64| { + let i = rec.partition_point(|r| r.0 <= secs).clamp(1, rec.len().max(2) - 1); + if rec.len() == 1 { + return rec[0].1; + } + let (a, b) = (rec[i - 1], rec[i]); + let f = if b.0 > a.0 { ((secs - a.0) / (b.0 - a.0)).clamp(0.0, 1.0) } else { 1.0 }; + [a.1[0] + (b.1[0] - a.1[0]) * f, a.1[1] + (b.1[1] - a.1[1]) * f] + }; + let n = s.edit("Record Puppet Pin", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + let lc = l.clone(); + let g = l.props.find_group_mut(uid).ok_or_else(|| bad(cmd, "pin gone"))?; + let kind = PinKind::from_index(g.get("kind").map(|k| k.value.as_enum()).unwrap_or(0)); + if !kind.moves() { + return Err(bad(cmd, "only Position and Advanced pins record motion")); + } + let pos = g.get_mut("position").ok_or_else(|| bad(cmd, "the pin has no Position"))?; + let mut new: Vec = vec![]; + for k in 0..=nframes { + let f = f0 + k; + if f > end_frame { + break; + } + let q = at(t_rec0 + k as f64 * fd / speed); + new.push(Keyframe::new(lc.layer_time(fr.tick_of(f)), KV::Vec2(q))); + } + let (Some(t0), Some(t1)) = (new.first().map(|k| k.time), new.last().map(|k| k.time)) else { return Ok(0) }; + let mut keys: Vec = pos.keys.iter().filter(|k| k.time < t0 || k.time > t1).cloned().collect(); + keys.extend(new); + keys.sort_by_key(|k| k.time); + let (a, b) = (keys.iter().position(|k| k.time == t0).unwrap_or(0), keys.iter().position(|k| k.time == t1).unwrap_or(0)); + if smoothing > 0.0 && b > a + 1 { + let samples: Vec<(Tick, Vec)> = keys[a..=b].iter().map(|k| (k.time, k.value.components())).collect(); + keys = super::anim_tools::smooth_keys(&keys, a, b, &samples, smoothing, true); + } + pos.keys = keys; + pos.value = KV::Vec2(rec[0].1); + let puid = pos.uid; + st.selected_props = vec![(lid, uid)]; + st.selected_keys = pos.keys.iter().filter(|k| k.time >= t0 && k.time <= t1).map(|k| crate::KeyRef { layer: lid, prop: puid, time: k.time }).collect(); + Ok(st.selected_keys.len()) + })?; + // The current time returns to where recording began. + s.state.times.insert(cid, start); + let end = fr.tick_of((f0 + nframes).min(end_frame)); + Ok(json!({"keys": n, "start": start.seconds(), "end": end.seconds(), "frames": nframes + 1})) +} + +/// Record Options (the Puppet tool's Record Options… dialog): `speed`, `smoothing`, +/// `useDraftDeformation`, `showMesh`. Returns the options. +fn record_options(s: &mut Session, p: &Value) -> Result { + let o = &mut s.state.puppet; + if let Some(v) = f_p(p, "speed") { + o.record_speed = v.clamp(1.0, 10_000.0); + } + if let Some(v) = f_p(p, "smoothing") { + o.record_smoothing = v.clamp(0.0, 100.0); + } + if let Some(v) = p.get("useDraftDeformation").and_then(Value::as_bool) { + o.record_draft = v; + } + if let Some(v) = p.get("showMesh").and_then(Value::as_bool) { + o.record_show_mesh = v; + } + Ok(json!({"speed": o.record_speed, "smoothing": o.record_smoothing, "useDraftDeformation": o.record_draft, "showMesh": o.record_show_mesh})) +} + /// Meshes, pins and the deformed mesh at the current time (layer space). fn info(s: &mut Session, p: &Value) -> Result { let (cid, lid) = layer_p(s, p, "puppet.info")?; @@ -364,6 +480,24 @@ pub fn specs() -> Vec { mesh_opts ), crate::query!("puppet.info", "Puppet Mesh Info", "{layer?, time? | frame?}", info), + cmd!( + "puppet.recordPin", + "Record Puppet Pin", + [], + None, + "{layer?, pin: uid | name, samples: [[t, x, y]…] (t = seconds since the drag began, layer space), start? (s, default current time), speed? (%, Record Options), smoothing? (Record Options)}", + has_comp, + record_pin + ), + cmd!( + "puppet.recordOptions", + "Record Options...", + [], + None, + "{speed? (%, 100), smoothing? (0-100), useDraftDeformation?, showMesh?}", + super::always, + record_options + ), ] } @@ -376,11 +510,30 @@ pub struct PuppetOptions { pub triangles: f64, /// Show the mesh in the viewer. pub show_mesh: bool, + /// Record Options ▸ Speed (%): 100 plays a recording back at the speed of the drag. + pub record_speed: f64, + /// Record Options ▸ Smoothing: removes extraneous keys from recorded motion (0 = a key per + /// frame). + pub record_smoothing: f64, + /// Record Options ▸ Use Draft Deformation: the viewer shows the draft (unexpanded) mesh + /// while recording. + pub record_draft: bool, + /// Record Options ▸ Show Mesh while recording. + pub record_show_mesh: bool, } impl Default for PuppetOptions { fn default() -> Self { let d = MeshOpts::default(); - PuppetOptions { density: d.density, expansion: d.expansion, triangles: d.triangles, show_mesh: false } + PuppetOptions { + density: d.density, + expansion: d.expansion, + triangles: d.triangles, + show_mesh: false, + record_speed: 100.0, + record_smoothing: 10.0, + record_draft: false, + record_show_mesh: false, + } } } diff --git a/crates/engine/src/commands/scripts.rs b/crates/engine/src/commands/scripts.rs new file mode 100644 index 00000000..628533af --- /dev/null +++ b/crates/engine/src/commands/scripts.rs @@ -0,0 +1,222 @@ +//! File ▸ Scripts and ScriptUI. +//! +//! * **Installed scripts** live in the settings store under `Scripts/` (File ▸ Scripts ▸ Install +//! Script File…) and `Scripts/ScriptUI Panels/` (Install ScriptUI Panel…), next to the +//! settings on the desktop, so dropping `.jsx` files in those folders works too. File ▸ Scripts +//! lists them with the bundled sample scripts ([`SAMPLES`]) and runs them by name; ScriptUI +//! panels appear at the bottom of the Window menu and open as dockable panels +//! (`window.scriptPanel`), running with `this` = the panel as in After Effects. +//! * **ScriptUI windows** (`scriptui.*`): list the open script windows, read their control +//! trees, click buttons and set values like a user, close them (see [`crate::scriptui`]). + +use serde::Serialize; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, str_p}; +use crate::{EngineError, Event, Result, Session, cmd, scriptui}; + +/// The settings-store folders of installed scripts. +pub const SCRIPTS_DIR: &str = "Scripts"; +pub const PANELS_DIR: &str = "Scripts/ScriptUI Panels"; + +/// Sample scripts that ship with EffectCraft (original work): (file name, ScriptUI panel, code). +pub const SAMPLES: &[(&str, bool, &str)] = &[ + ("Create Null at Selected Layers.jsx", false, include_str!("../../scripts/Create Null at Selected Layers.jsx")), + ("Rename Layers.jsx", false, include_str!("../../scripts/Rename Layers.jsx")), + ("Render Queue Batch.jsx", false, include_str!("../../scripts/Render Queue Batch.jsx")), + ("Sort Layers by In Point.jsx", false, include_str!("../../scripts/Sort Layers by In Point.jsx")), + ("Layer Tools.jsx", true, include_str!("../../scripts/ScriptUI Panels/Layer Tools.jsx")), +]; + +/// A script File ▸ Scripts (or the Window menu, for panels) offers. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ScriptEntry { + /// File name (`Rename Layers.jsx`): what the menus show and commands take. + pub name: String, + /// In the ScriptUI Panels folder: opens as a dockable panel. + pub panel: bool, + /// `installed` or `sample`. + pub source: &'static str, +} + +fn is_script(name: &str) -> bool { + let l = name.to_ascii_lowercase(); + l.ends_with(".jsx") || l.ends_with(".js") +} + +/// Installed scripts and panels, then the samples not shadowed by an installed one, by name. +pub fn scripts(s: &Session) -> Vec { + let mut v: Vec = vec![]; + if let Some(cfg) = &s.config { + for (dir, panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + for n in cfg.list(dir).into_iter().filter(|n| is_script(n)) { + v.push(ScriptEntry { name: n, panel, source: "installed" }); + } + } + } + for (name, panel, _) in SAMPLES { + if !v.iter().any(|e| e.name == *name && e.panel == *panel) { + v.push(ScriptEntry { name: name.to_string(), panel: *panel, source: "sample" }); + } + } + v.sort_by(|a, b| a.panel.cmp(&b.panel).then(a.name.to_lowercase().cmp(&b.name.to_lowercase()))); + v +} + +/// The code of an installed or sample script / panel. +pub fn script_code(s: &Session, name: &str, panel: Option) -> Option<(bool, String)> { + if let Some(cfg) = &s.config { + for (dir, is_panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + if panel.is_some_and(|p| p != is_panel) { + continue; + } + if let Some(code) = cfg.read(&format!("{dir}/{name}")) { + return Some((is_panel, code)); + } + } + } + SAMPLES.iter().find(|(n, p, _)| *n == name && panel.is_none_or(|w| w == *p)).map(|(_, p, c)| (*p, c.to_string())) +} + +fn install(s: &mut Session, p: &Value, panel: bool) -> Result { + let c = if panel { "file.installScriptUIPanel" } else { "file.installScript" }; + let path = str_p(p, "path").ok_or_else(|| bad(c, "missing `path` (a .jsx / .js script)"))?; + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + if !is_script(&name) { + return Err(bad(c, "scripts are .jsx or .js files (.jsxbin is not supported)")); + } + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let code = String::from_utf8(bytes).map_err(|_| bad(c, "the script is not UTF-8 text"))?; + let cfg = s.config.clone().ok_or_else(|| EngineError::Other("this session has no settings folder to install scripts into".into()))?; + let dir = if panel { PANELS_DIR } else { SCRIPTS_DIR }; + cfg.write(&format!("{dir}/{name}"), &code).map_err(|e| EngineError::Other(format!("cannot install {name}: {e}")))?; + // After Effects asks for a restart; our menus pick it up at once. + s.toast(if panel { + format!("Installed ScriptUI panel {name}: find it at the bottom of the Window menu") + } else { + format!("Installed script {name}: find it in File ▸ Scripts") + }); + Ok(json!({"name": name, "panel": panel})) +} + +fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!(scripts(s))) +} + +fn uninstall(s: &mut Session, p: &Value) -> Result { + let c = "file.uninstallScript"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name`"))?; + let cfg = s.config.clone().ok_or_else(|| bad(c, "no settings folder"))?; + let mut removed = false; + for dir in [SCRIPTS_DIR, PANELS_DIR] { + let key = format!("{dir}/{name}"); + if cfg.read(&key).is_some() { + cfg.remove(&key).map_err(|e| EngineError::Other(e.to_string()))?; + removed = true; + } + } + if removed { Ok(json!({"removed": name})) } else { Err(bad(c, format!("`{name}` is not installed"))) } +} + +/// Run an installed or sample script by name (File ▸ Scripts ▸ ). A ScriptUI panel run +/// this way gets `this` = a floating palette-like panel window. +pub(crate) fn run_named(s: &mut Session, name: &str) -> Result { + let (panel, code) = script_code(s, name, None).ok_or_else(|| bad("file.runScript", format!("no installed or sample script `{name}`")))?; + if panel { + return open_panel(s, &json!({"name": name})); + } + super::file_more::run_js_named(s, &code, name) +} + +/// The code a ScriptUI panel runs with: `this` is a dockable Panel titled after the script. +pub fn panel_wrapper(title: &str, code: &str) -> String { + // On one line, so line numbers in errors stay those of the file. + format!("(function () {{ {code}\n}}).call(__uiDockPanel({}));", Value::String(title.to_string())) +} + +/// Window ▸ : run the panel script (or bring its panel forward). +fn open_panel(s: &mut Session, p: &Value) -> Result { + let c = "window.scriptPanel"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name` (a script in the ScriptUI Panels folder)"))?.to_string(); + let title = name.trim_end_matches(".jsx").trim_end_matches(".js").to_string(); + if let Some(w) = s.script_ui.windows.iter().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name) { + let id = w.id; + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + return Ok(json!({"window": id, "open": true})); + } + let (_, code) = script_code(s, &name, Some(true)).or_else(|| script_code(s, &name, None)).ok_or_else(|| bad(c, format!("no ScriptUI panel `{name}`")))?; + let out = super::file_more::run_js_named(s, &panel_wrapper(&title, &code), &name)?; + let id = s.script_ui.windows.iter().rev().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name).map(|w| w.id); + if let Some(id) = id { + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + } + Ok(json!({"window": id, "result": out})) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "file.installScript", + "Install Script File...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the Scripts folder; it appears in File ▸ Scripts", + always, + |s, p| install(s, p, false) + ), + cmd!( + "file.installScriptUIPanel", + "Install ScriptUI Panel...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the ScriptUI Panels folder; it appears in the Window menu", + always, + |s, p| install(s, p, true) + ), + cmd!("file.uninstallScript", "Uninstall Script", [], None, "{name}", always, uninstall), + crate::query!("file.scripts.list", "List Scripts", "{} → [{name, panel, source: installed|sample}]", list), + cmd!( + "window.scriptPanel", + "ScriptUI Panel", + [], + None, + "{name (a script in the ScriptUI Panels folder, e.g. `Layer Tools.jsx`)} → opens it as a dockable panel", + always, + open_panel + ), + crate::query!("scriptui.list", "List Script Windows", "{} → [{window, title, kind: dialog|palette|window|panel, script, modal, size}]", scriptui::list), + crate::query!( + "scriptui.get", + "Script Window Controls", + "{window?: id | title} → {id, title, kind, root: {id, type, name, text, value, checked, items, selection, bounds, enabled, children…}}", + scriptui::get + ), + cmd!( + "scriptui.click", + "Click Script Window Control", + [], + None, + "{window?: id | title, widget: id | \"#id\" | properties.name | text} (buttons, checkboxes, radio buttons, tabs)", + always, + scriptui::click + ), + cmd!( + "scriptui.set", + "Set Script Window Control", + [], + None, + "{window?, widget, value: text | number | bool | item index | item text} (edit text, sliders, checkboxes, lists) → fires onChanging / onChange", + always, + scriptui::set + ), + cmd!( + "scriptui.close", + "Close Script Window", + [], + None, + "{window?, result? (a dialog's show() returns it; default 2 = Cancel)}", + always, + scriptui::close + ), + ] +} diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs index ae7cfe09..0608dc6c 100644 --- a/crates/engine/src/config.rs +++ b/crates/engine/src/config.rs @@ -16,6 +16,12 @@ pub trait ConfigStore: Send + Sync { fn read(&self, name: &str) -> Option; fn write(&self, name: &str, data: &str) -> std::io::Result<()>; fn remove(&self, name: &str) -> std::io::Result<()>; + /// Names of the entries directly in `dir` (a `/`-separated name prefix such as `Scripts`), + /// sorted. Stores that can't list return nothing. + fn list(&self, dir: &str) -> Vec { + let _ = dir; + vec![] + } /// The folder next to the configuration (default Auto-Save folder for untitled projects), /// in the namespace of [`ConfigStore::files`]; `None` when there is none. fn dir(&self) -> Option { @@ -86,6 +92,11 @@ impl ConfigStore for MemoryConfig { } Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{}/", dir.trim_end_matches('/')); + let Ok(m) = self.files.lock() else { return vec![] }; + m.keys().filter_map(|k| k.strip_prefix(&pre)).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string).collect() + } } /// Files in one directory, written atomically. @@ -104,8 +115,15 @@ impl ConfigStore for DirConfig { std::fs::read_to_string(self.dir.join(name)).ok() } fn write(&self, name: &str, data: &str) -> std::io::Result<()> { - std::fs::create_dir_all(&self.dir)?; - atomic_write(&self.dir.join(name), data.as_bytes()) + let path = self.dir.join(name); + std::fs::create_dir_all(path.parent().unwrap_or(&self.dir))?; + atomic_write(&path, data.as_bytes()) + } + fn list(&self, dir: &str) -> Vec { + let Ok(rd) = std::fs::read_dir(self.dir.join(dir)) else { return vec![] }; + let mut v: Vec = rd.flatten().filter(|e| e.path().is_file()).map(|e| e.file_name().to_string_lossy().to_string()).collect(); + v.sort(); + v } fn remove(&self, name: &str) -> std::io::Result<()> { match std::fs::remove_file(self.dir.join(name)) { diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs new file mode 100644 index 00000000..36630969 --- /dev/null +++ b/crates/engine/src/history.rs @@ -0,0 +1,346 @@ +//! Branching undo history (the History panel; better than After Effects, whose undo is a +//! straight line). +//! +//! [`History`] keeps After Effects' linear model as its working line — `undo` (the states +//! before each step, oldest first) and `redo` (undone states, next first) — and adds +//! **branches**: when you undo and then do something new, the undone states are not thrown +//! away but kept as a [`Branch`] hanging off the state they grew from. [`History::tree`] lists +//! every state as a tree and [`Session::goto_history`] jumps to any of them, on any branch: the +//! path to the target becomes the working line, and the line you left becomes a branch. +//! +//! States are whole-project snapshots (`Arc`), identified by their address while the +//! history holds them ([`HistoryNode::id`]), so ids stay valid across undo, redo and jumps. + +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_project::Project; +use serde::Serialize; + +use crate::Session; + +/// Label of the oldest state in the history (the project as opened, or the oldest state kept). +pub const ROOT_LABEL: &str = "Original"; + +/// Undo history of whole-project snapshots. +#[derive(Clone, Default)] +pub struct History { + /// The states before each step on the working line, oldest first: (step label, state). + pub undo: Vec<(String, Arc)>, + /// Undone steps, the next one last: (step label, the state it leads to). + pub redo: Vec<(String, Arc)>, + /// Key of the last merged step: a continuous gesture with the same key folds into one step. + pub merge_key: Option, + /// Alternative futures kept when a new step was made after undoing. + pub branches: Vec, +} + +/// A line of states that grows from another state of the history. +#[derive(Clone)] +pub struct Branch { + /// The state the branch grows from. + pub parent: Arc, + /// The branch's states in order: (label of the step that made it, state). + pub steps: Vec<(String, Arc)>, +} + +/// One state of the history tree (`edit.history.list`). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryNode { + /// Position in the listing (depth-first, the working line's continuation first). + pub index: usize, + /// Stable id of the state while the history keeps it. + pub id: String, + /// The step that made this state ([`ROOT_LABEL`] for the oldest). + pub label: String, + /// Index of the state it was made from. + pub parent: Option, + /// Branch nesting (0 = the working line). + pub depth: usize, + /// The current state. + pub current: bool, + /// On the working line (Undo / Redo walk it). + pub line: bool, + /// On the working line after the current state (Redo reaches it). + pub future: bool, +} + +fn key(p: &Arc) -> usize { + Arc::as_ptr(p) as usize +} + +/// Id of a state (its address while the history holds it). +pub fn state_id(p: &Arc) -> String { + format!("s{:x}", key(p)) +} + +/// The whole tree as an arena (internal). +struct Arena { + label: Vec, + state: Vec>, + parent: Vec>, + children: Vec>, + /// Index of each working-line state, oldest first, and the current state's position in it. + line: Vec, + current: usize, +} + +impl Arena { + fn build(h: &History, current: &Arc) -> Arena { + let mut a = Arena { label: vec![], state: vec![], parent: vec![], children: vec![], line: vec![], current: 0 }; + let add = |a: &mut Arena, label: String, st: Arc, parent: Option| { + let i = a.state.len(); + a.label.push(label); + a.state.push(st); + a.parent.push(parent); + a.children.push(vec![]); + if let Some(p) = parent { + a.children[p].push(i); + } + i + }; + // The working line: undo states, the current state, then the redo states. + let mut prev: Option = None; + let mut next_label = ROOT_LABEL.to_string(); + for (label, st) in &h.undo { + let i = add(&mut a, std::mem::take(&mut next_label), st.clone(), prev); + a.line.push(i); + prev = Some(i); + next_label = label.clone(); + } + let cur = add(&mut a, next_label, current.clone(), prev); + a.line.push(cur); + a.current = cur; + prev = Some(cur); + for (label, st) in h.redo.iter().rev() { + let i = add(&mut a, label.clone(), st.clone(), prev); + a.line.push(i); + prev = Some(i); + } + // Branches, attached to whichever state they grow from (branches of branches need the + // parent branch first, hence the passes). Orphans (their state was dropped) are skipped. + let mut index: HashMap = a.state.iter().enumerate().map(|(i, s)| (key(s), i)).collect(); + let mut pending: Vec<&Branch> = h.branches.iter().collect(); + loop { + let before = pending.len(); + pending.retain(|b| { + let Some(&p) = index.get(&key(&b.parent)) else { return true }; + let mut prev = p; + for (label, st) in &b.steps { + if index.contains_key(&key(st)) { + // Already in the tree (shared state): continue from it. + prev = index[&key(st)]; + continue; + } + let i = add(&mut a, label.clone(), st.clone(), Some(prev)); + index.insert(key(st), i); + prev = i; + } + false + }); + if pending.is_empty() || pending.len() == before { + break; + } + } + a + } + + fn on_line(&self) -> Vec { + let mut v = vec![false; self.state.len()]; + for &i in &self.line { + v[i] = true; + } + v + } + + /// Depth-first listing: the working line's child first, then branches in creation order. + fn listing(&self) -> Vec { + let on_line = self.on_line(); + let pos_in_line: HashMap = self.line.iter().enumerate().map(|(k, &i)| (i, k)).collect(); + let cur_pos = pos_in_line[&self.current]; + let mut out: Vec = vec![]; + let mut order: HashMap = HashMap::new(); + // (node, depth) + let mut stack: Vec<(usize, usize)> = vec![(self.line[0], 0)]; + while let Some((n, depth)) = stack.pop() { + order.insert(n, out.len()); + let line = on_line[n]; + out.push(HistoryNode { + index: out.len(), + id: state_id(&self.state[n]), + label: self.label[n].clone(), + parent: self.parent[n].and_then(|p| order.get(&p).copied()), + depth, + current: n == self.current, + line, + future: line && pos_in_line[&n] > cur_pos, + }); + // Push in reverse so the first child pops first; the line child continues at the + // same depth, branch children one deeper; a branch's own continuation (its first + // child) stays at the branch's depth. + for (k, &c) in self.children[n].iter().enumerate().rev() { + let cont = if line { on_line[c] } else { k == 0 }; + stack.push((c, if cont { depth } else { depth + 1 })); + } + } + out + } +} + +impl History { + /// Record a new step made from `before`: the redo line (if any) is kept as a branch. + pub fn record(&mut self, label: &str, before: Arc, levels: usize) { + self.abandon_redo(&before); + self.undo.push((label.to_string(), before.clone())); + self.merge_key = None; + self.trim(levels, &before); + } + + /// Keep the undone states as a branch growing from `from` (the state a new step starts from). + pub fn abandon_redo(&mut self, from: &Arc) { + if self.redo.is_empty() { + return; + } + let steps: Vec<(String, Arc)> = self.redo.drain(..).rev().collect(); + self.branches.push(Branch { parent: from.clone(), steps }); + } + + /// Apply the undo-levels limit: drop the oldest working-line states, the branches that grew + /// from them, and the oldest branches when branches hold more states than `levels`. + pub fn trim(&mut self, levels: usize, current: &Arc) { + let levels = levels.max(1); + if self.undo.len() > levels { + let extra = self.undo.len() - levels; + self.undo.drain(..extra); + } + let mut total: usize = self.branches.iter().map(|b| b.steps.len()).sum(); + while total > levels && !self.branches.is_empty() { + total -= self.branches.remove(0).steps.len(); + } + self.prune(current); + } + + /// Drop branches whose parent state is no longer in the history (`current` = the current + /// state, which the history doesn't store). + pub fn prune(&mut self, current: &Arc) { + if self.branches.is_empty() { + return; + } + let mut live: std::collections::HashSet = self.undo.iter().chain(self.redo.iter()).map(|(_, p)| key(p)).collect(); + live.insert(key(current)); + let mut keep = vec![false; self.branches.len()]; + loop { + let mut changed = false; + for (i, b) in self.branches.iter().enumerate() { + if !keep[i] && live.contains(&key(&b.parent)) { + keep[i] = true; + live.extend(b.steps.iter().map(|(_, p)| key(p))); + changed = true; + } + } + if !changed { + break; + } + } + let mut k = keep.into_iter(); + self.branches.retain(|_| k.next().unwrap_or(false)); + } + + /// Drop everything (Edit ▸ Purge ▸ Undo, New / Open Project). + pub fn clear(&mut self) { + *self = History::default(); + } + + /// The history as a tree, given the current state. + pub fn tree(&self, current: &Arc) -> Vec { + Arena::build(self, current).listing() + } + + /// Number of states on branches (not on the working line). + pub fn branch_states(&self) -> usize { + self.branches.iter().map(|b| b.steps.len()).sum() + } +} + +impl Session { + /// The undo history as a tree (the History panel, `edit.history.list`). + pub fn history_tree(&self) -> Vec { + self.history.tree(&self.project) + } + + /// Jump to any state of the history (`id` from [`HistoryNode::id`]), including states on + /// other branches: the path to it becomes the working line (its undone continuation the + /// redo line), and the line left behind is kept as a branch. Returns false for an unknown + /// id. + pub fn goto_history(&mut self, id: &str) -> bool { + let a = Arena::build(&self.history, &self.project); + let Some(target) = a.state.iter().position(|s| state_id(s) == id) else { return false }; + if target == a.current { + return true; + } + let on_line = a.on_line(); + // Path root → target. + let mut path = vec![target]; + while let Some(p) = a.parent[*path.last().unwrap_or(&target)] { + path.push(p); + } + path.reverse(); + // Continue past the target: the working line where it runs on, else each branch's own + // continuation (its first child). + let mut new_line = path.clone(); + let mut n = target; + loop { + let kids = &a.children[n]; + let next = if on_line[n] { kids.iter().copied().find(|&c| on_line[c]) } else { kids.first().copied() }; + match next { + Some(c) => { + new_line.push(c); + n = c; + } + None => break, + } + } + let mut in_new = vec![false; a.state.len()]; + for &i in &new_line { + in_new[i] = true; + } + // Every state off the new line goes into branches: a chain per first child, the other + // children starting branches of their own. + fn chain(a: &Arena, start: usize, parent: usize, in_new: &[bool], out: &mut Vec) { + let mut steps = vec![]; + let mut n = start; + let mut extra: Vec<(usize, usize)> = vec![]; + loop { + steps.push((a.label[n].clone(), a.state[n].clone())); + let kids: Vec = a.children[n].iter().copied().filter(|&c| !in_new[c]).collect(); + let Some((&first, rest)) = kids.split_first() else { break }; + extra.extend(rest.iter().map(|&c| (c, n))); + n = first; + } + out.push(Branch { parent: a.state[parent].clone(), steps }); + for (c, p) in extra { + chain(a, c, p, in_new, out); + } + } + let mut branches = vec![]; + for &n in &new_line { + for &c in &a.children[n] { + if !in_new[c] { + chain(&a, c, n, &in_new, &mut branches); + } + } + } + let k = new_line.iter().position(|&i| i == target).unwrap_or(0); + let undo: Vec<(String, Arc)> = (0..k).map(|i| (a.label[new_line[i + 1]].clone(), a.state[new_line[i]].clone())).collect(); + let redo: Vec<(String, Arc)> = new_line[k + 1..].iter().rev().map(|&i| (a.label[i].clone(), a.state[i].clone())).collect(); + self.project = a.state[target].clone(); + self.history.undo = undo; + self.history.redo = redo; + self.history.branches = branches; + self.history.merge_key = None; + self.sanitize_state(); + self.bump(); + true + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9333329a..ad6ae4fb 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,6 +13,7 @@ pub mod camera_track; pub mod commands; pub mod config; pub mod demo; +pub mod history; pub mod links; pub mod mask_track; pub mod menus; @@ -21,6 +22,7 @@ pub mod prefs; pub mod psd_import; pub mod render_queue; pub mod roto; +pub mod scriptui; mod session_settings; pub mod shortcuts; pub mod tracking; @@ -47,6 +49,7 @@ pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; pub use effectcraft_track as track; +pub use history::{Branch, History, HistoryNode}; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -90,15 +93,6 @@ pub trait Importer: Send + Sync { fn probe(&self, path: &str) -> std::result::Result; } -/// Undo history of whole-project snapshots. -#[derive(Clone, Default)] -pub struct History { - pub undo: Vec<(String, Arc)>, - pub redo: Vec<(String, Arc)>, - /// Key of the last merged step: a continuous gesture with the same key folds into one step. - pub merge_key: Option, -} - /// A keyframe reference: layer, property uid, key time (layer time). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct KeyRef { @@ -320,6 +314,11 @@ pub struct Session { /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. pub script: Option, + /// Loads WebAssembly effect plug-ins (set by the host that links `effectcraft-plugin` with + /// its runtime): `effect.plugins.load`. + pub plugin_loader: Option, + /// ScriptUI windows and panels opened by scripts (see [`scriptui`]). + pub script_ui: scriptui::ScriptUi, /// Runs renders and analyses off the UI thread where there are no threads (the web app's /// Web Workers, see [`offload`]); `None` = threads (desktop) or inline (wasm32). pub offload: Option>, @@ -341,6 +340,10 @@ pub struct ScriptRequest<'a> { /// column, file} | null}`. pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; +/// Loads one WebAssembly effect plug-in (`bytes`, `source` path) into the effect registry and +/// describes it (`{id, name, category, version, api, params}`). +pub type PluginLoader = fn(&[u8], &str) -> std::result::Result; + impl Default for Session { fn default() -> Self { Session { @@ -379,6 +382,8 @@ impl Default for Session { autosave: autosave::AutoSaveState::default(), snapshot: None, script: None, + plugin_loader: None, + script_ui: scriptui::ScriptUi::default(), offload: None, offloaded: vec![], } @@ -453,15 +458,13 @@ impl Session { } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { - self.history.undo.push((label.to_string(), before)); + // Undone steps aren't lost: they stay in the History panel as a branch. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + self.history.record(label, before, levels); + } else { + self.history.abandon_redo(&before); } self.history.merge_key = merge.map(str::to_string); - self.history.redo.clear(); self.project = Arc::new(p); self.state = st; self.bump(); @@ -748,6 +751,8 @@ pub fn text_families() -> Vec { effectcraft_text::families().into_iter().map(|(f, _)| f).collect() } #[cfg(test)] +mod tests_history; +#[cfg(test)] mod tests_paint; #[cfg(test)] mod tests_roto; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..7d41fa71 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -247,9 +247,8 @@ impl Session { return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Track Mask".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Track Mask", self.project.clone(), levels); let shared = Arc::new(MaskTrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 51fa4e02..7f14a988 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -280,7 +280,7 @@ pub fn parse(text: &str, mac: bool) -> Result, String> { /// Effect ▸ category submenus from the effect registry. fn effect_categories() -> Vec { let mut cats: Vec<(&str, Vec)> = vec![]; - for e in crate::effects::registry() { + for e in crate::effects::all() { let entry = MenuNode::Item(MenuEntry { label: e.name.into(), command: "effect.apply".into(), params: serde_json::json!({"effect": e.id}), shortcut: None }); match cats.iter_mut().find(|(c, _)| *c == e.category) { @@ -371,6 +371,8 @@ File Find Missing Footage | file.findMissing {"what":"footage"} --- Scripts + Install Script File... | file.installScript + Install ScriptUI Panel... | file.installScriptUIPanel Run Script File... | file.runScript --- Create Proxy diff --git a/crates/engine/src/scriptui.rs b/crates/engine/src/scriptui.rs new file mode 100644 index 00000000..2a76ba04 --- /dev/null +++ b/crates/engine/src/scriptui.rs @@ -0,0 +1,572 @@ +//! ScriptUI: the windows, dialogs and dockable panels scripts build with After Effects' +//! ScriptUI object model (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, +//! `orientation`, `alignChildren`, `onClick`/`onChange`, `show()`, `close()`, +//! `layout.layout()`…). +//! +//! The scripting engine (`effectcraft-script`) owns the live JavaScript objects; after every +//! script step it publishes a serde description of each open window here ([`ScriptWindow`], +//! a [`Widget`] tree with computed [`Widget::bounds`]). Frontends draw that description (the egui +//! UI renders dialogs, palettes and dockable panels from it) and report what the user does as +//! [`ScriptUiEvent`]s through the `scriptui.*` commands, which hand them to the script's +//! handlers ([`ScriptUi::dispatch`]). Agents use the same commands: list the open script +//! windows, read their widget trees, click buttons and set values. +//! +//! [`layout`] is ScriptUI's automatic layout (orientation, alignChildren / alignment, spacing, +//! margins, preferredSize), so every frontend places widgets the same way. + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{EngineError, Result, Session}; + +/// What kind of window a script made. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WindowKind { + /// Modal: the script waits in `show()` until it closes. + #[default] + Dialog, + /// Floating, non-modal. + Palette, + /// A non-modal document window. + Window, + /// A dockable panel (a script from the ScriptUI Panels folder, opened from the Window menu). + Panel, +} + +/// ScriptUI control types (the `type` passed to `add()`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WidgetKind { + /// The window itself (the tree's root). + #[default] + Window, + Panel, + Group, + Button, + IconButton, + StaticText, + EditText, + Checkbox, + RadioButton, + Slider, + Scrollbar, + Progressbar, + DropDownList, + ListBox, + TabbedPanel, + Tab, + Image, + /// A type we don't draw (it still lays out as an empty box). + #[serde(other)] + Unknown, +} + +impl WidgetKind { + pub fn is_container(self) -> bool { + matches!(self, WidgetKind::Window | WidgetKind::Panel | WidgetKind::Group | WidgetKind::TabbedPanel | WidgetKind::Tab) + } +} + +/// One control (or container) of a script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Widget { + /// Unique within its window (the window itself is 0). + pub id: u32, + #[serde(rename = "type")] + pub kind: WidgetKind, + /// `properties.name` (agents can address controls by it). + pub name: String, + pub text: String, + /// Slider / scrollbar / progressbar value. + pub value: f64, + pub min: f64, + pub max: f64, + /// Checkbox / radio button state. + pub checked: bool, + /// Drop-down list / list box items. + pub items: Vec, + /// Selected item indices. + pub selection: Vec, + /// `column` (default for windows and panels), `row` (groups) or `stack`. + pub orientation: String, + /// `[horizontal, vertical]`: left / center / right / fill, top / center / bottom / fill. + pub align_children: Vec, + /// This control's own alignment in its parent (overrides the parent's alignChildren). + pub alignment: Vec, + pub spacing: Option, + /// `[left, top, right, bottom]`. + pub margins: Option<[f64; 4]>, + pub preferred_size: Option<[f64; 2]>, + /// Bounds set by the script (`[left, top, right, bottom]` relative to the parent). + pub fixed_bounds: Option<[f64; 4]>, + /// Laid-out bounds `[x, y, width, height]` relative to the window's content. + pub bounds: [f64; 4], + pub enabled: bool, + pub visible: bool, + pub help_tip: String, + pub multiline: bool, + pub read_only: bool, + /// Event handlers the script attached (`onClick`, `onChange`, `onChanging`…). + pub handlers: Vec, + /// The active tab of a tabbed panel (child index). + pub active_tab: usize, + pub children: Vec, +} + +impl Widget { + pub fn find(&self, id: u32) -> Option<&Widget> { + if self.id == id { + return Some(self); + } + self.children.iter().find_map(|c| c.find(id)) + } + pub fn find_mut(&mut self, id: u32) -> Option<&mut Widget> { + if self.id == id { + return Some(self); + } + self.children.iter_mut().find_map(|c| c.find_mut(id)) + } + /// Every control, depth first. + pub fn walk<'a>(&'a self, out: &mut Vec<&'a Widget>) { + out.push(self); + for c in &self.children { + c.walk(out); + } + } + /// A control by id (number or `"#12"`), `properties.name` or text. + pub fn lookup(&self, key: &Value) -> Option<&Widget> { + let mut all = vec![]; + self.walk(&mut all); + match key { + Value::Number(n) => n.as_u64().and_then(|n| self.find(n as u32)), + Value::String(s) => { + if let Some(n) = s.strip_prefix('#').and_then(|n| n.parse::().ok()) { + return self.find(n); + } + all.iter().find(|w| !w.name.is_empty() && w.name == *s).or_else(|| all.iter().find(|w| w.id != 0 && w.text == *s)).copied() + } + _ => None, + } + } +} + +/// An open script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ScriptWindow { + /// Unique in the process. + pub id: u32, + /// The script context that owns it (its handlers run there). + pub host: u32, + pub kind: WindowKind, + pub title: String, + /// The script that made it. + pub script: String, + /// Shown and not closed. + pub visible: bool, + /// The script is waiting in this dialog's `show()`. + pub modal: bool, + /// The window and its controls (`root.bounds` is the content size). + pub root: Widget, +} + +/// A user (or agent) action on a script window control. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptUiEvent { + pub window: u32, + pub widget: u32, + /// `click`, `change`, `changing` or `close`. + pub kind: String, + /// The new value: text, number, bool, or selected index(es); for `close`, the result. + pub value: Value, +} + +/// Hands an event to the script that owns the window; returns what the handler reported +/// (`{ok, output, error}`) and updates [`Session::script_ui`]. +pub type ScriptUiDispatch = fn(&mut Session, &ScriptUiEvent) -> std::result::Result; + +/// The open script windows. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct ScriptUi { + pub windows: Vec, + /// Bumped whenever a window changes. + pub revision: u64, + /// The scripting engine's event hand-off (set with [`Session::script`]). + #[serde(skip)] + pub dispatch: Option, +} + +impl ScriptUi { + pub fn window(&self, id: u32) -> Option<&ScriptWindow> { + self.windows.iter().find(|w| w.id == id) + } + + /// Replace the windows a script host published (laying them out). + pub fn publish(&mut self, host: u32, mut windows: Vec) { + for w in &mut windows { + w.host = host; + layout(w); + } + windows.retain(|w| w.visible); + let mut out: Vec = vec![]; + // Keep the order windows first appeared in. + for old in self.windows.drain(..) { + if old.host != host { + out.push(old); + } else if let Some(i) = windows.iter().position(|w| w.id == old.id) { + out.push(windows.remove(i)); + } + } + out.extend(windows); + self.windows = out; + self.revision += 1; + } +} + +// ---------------------------------------------------------------- layout + +/// Average character width and line height of the default dialog font (points). +const CHAR_W: f64 = 7.0; +const LINE_H: f64 = 16.0; + +fn text_w(s: &str) -> f64 { + s.lines().map(|l| l.chars().count()).max().unwrap_or(0) as f64 * CHAR_W +} + +fn lines(s: &str) -> f64 { + s.lines().count().max(1) as f64 +} + +fn orientation(w: &Widget) -> &str { + match w.orientation.as_str() { + "" => match w.kind { + WidgetKind::Group => "row", + WidgetKind::TabbedPanel => "stack", + _ => "column", + }, + o => o, + } +} + +fn margins(w: &Widget) -> [f64; 4] { + w.margins.unwrap_or(match w.kind { + WidgetKind::Window => [15.0; 4], + WidgetKind::Panel => [10.0, 15.0, 10.0, 10.0], + WidgetKind::TabbedPanel => [0.0, 24.0, 0.0, 0.0], + WidgetKind::Tab => [10.0; 4], + _ => [0.0; 4], + }) +} + +fn spacing(w: &Widget) -> f64 { + w.spacing.unwrap_or(10.0) +} + +fn align_children(w: &Widget) -> (String, String) { + let d = match w.kind { + WidgetKind::Group => ("center", "center"), + _ => ("center", "top"), + }; + let a = |i: usize, d: &str| w.align_children.get(i).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| d.to_string()); + // A single value applies to the axis it names. + if w.align_children.len() == 1 { + let v = w.align_children[0].clone(); + return match v.as_str() { + "top" | "bottom" => (d.0.into(), v), + "fill" if orientation(w) == "row" => (d.0.into(), v), + _ => (v, d.1.into()), + }; + } + (a(0, d.0), a(1, d.1)) +} + +/// Preferred size of a control (its own content; containers include their children). +pub fn preferred(w: &Widget) -> [f64; 2] { + let ps = w.preferred_size.unwrap_or([0.0; 2]); + let content = if w.kind.is_container() { + let kids: Vec<[f64; 2]> = w.children.iter().filter(|c| c.visible).map(preferred).collect(); + let m = margins(w); + let sp = spacing(w); + let n = kids.len() as f64; + let (cw, ch) = match orientation(w) { + "row" => (kids.iter().map(|k| k[0]).sum::() + sp * (n - 1.0).max(0.0), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + "stack" => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + _ => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).sum::() + sp * (n - 1.0).max(0.0)), + }; + let title_w = if matches!(w.kind, WidgetKind::Panel) { text_w(&w.text) + 20.0 } else { 0.0 }; + [(cw + m[0] + m[2]).max(title_w), ch + m[1] + m[3]] + } else { + match w.kind { + WidgetKind::Button | WidgetKind::IconButton => [(text_w(&w.text) + 24.0).max(80.0), 25.0], + WidgetKind::StaticText => [text_w(&w.text) + 4.0, LINE_H * lines(&w.text)], + WidgetKind::EditText => { + if w.multiline { + [(text_w(&w.text) + 12.0).max(160.0), (LINE_H * lines(&w.text) + 8.0).max(60.0)] + } else { + [(text_w(&w.text) + 12.0).max(40.0), 22.0] + } + } + WidgetKind::Checkbox | WidgetKind::RadioButton => [text_w(&w.text) + 24.0, 18.0], + WidgetKind::Slider | WidgetKind::Scrollbar => [100.0, 22.0], + WidgetKind::Progressbar => [100.0, 10.0], + WidgetKind::DropDownList => [w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 36.0, 22.0], + WidgetKind::ListBox => [(w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 24.0).max(80.0), (w.items.len().clamp(3, 10) as f64 * 18.0 + 4.0)], + _ => [20.0, 20.0], + } + }; + [if ps[0] > 0.0 { ps[0] } else { content[0] }, if ps[1] > 0.0 { ps[1] } else { content[1] }] +} + +fn place(w: &mut Widget, x: f64, y: f64, width: f64, height: f64) { + w.bounds = [x, y, width, height]; + if !w.kind.is_container() { + return; + } + let m = margins(w); + let sp = spacing(w); + let (ah, av) = align_children(w); + let inner = [m[0], m[1], (width - m[0] - m[2]).max(0.0), (height - m[1] - m[3]).max(0.0)]; + let orient = orientation(w).to_string(); + let mut cursor = 0.0; + for c in w.children.iter_mut() { + if !c.visible { + c.bounds = [0.0; 4]; + continue; + } + if let Some(b) = c.fixed_bounds { + let (cx, cy, cw, ch) = (x + b[0], y + b[1], b[2] - b[0], b[3] - b[1]); + place(c, cx, cy, cw, ch); + continue; + } + let pref = preferred(c); + let h_align = c.alignment.first().filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| ah.clone()); + let v_align = c.alignment.get(1).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| av.clone()); + let along = |a: &str, free: f64| match a { + "center" => free / 2.0, + "right" | "bottom" => free, + _ => 0.0, + }; + let (cx, cy, cw, ch) = match orient.as_str() { + "row" => { + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + let cy = inner[1] + along(&v_align, inner[3] - ch); + let r = (inner[0] + cursor, cy, pref[0], ch); + cursor += pref[0] + sp; + r + } + "stack" => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + (inner[0] + along(&h_align, inner[2] - cw), inner[1] + along(&v_align, inner[3] - ch), cw, ch) + } + _ => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let cx = inner[0] + along(&h_align, inner[2] - cw); + let r = (cx, inner[1] + cursor, cw, pref[1]); + cursor += pref[1] + sp; + r + } + }; + place(c, x + cx, y + cy, cw, ch); + } + // Rows centre their content run as a whole when it is narrower than the row. + if orient == "row" && (ah == "center" || ah == "right") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&ah, inner[2] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, shift, 0.0); + } + } + } + // Columns aligned to the bottom / centre move the whole run. + if orient == "column" && (av == "center" || av == "bottom") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&av, inner[3] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, 0.0, shift); + } + } + } +} + +fn along_shift(a: &str, free: f64) -> f64 { + match a { + "center" => (free / 2.0).max(0.0), + "right" | "bottom" => free.max(0.0), + _ => 0.0, + } +} + +fn shift_all(w: &mut Widget, dx: f64, dy: f64) { + w.bounds[0] += dx; + w.bounds[1] += dy; + for c in &mut w.children { + shift_all(c, dx, dy); + } +} + +/// ScriptUI automatic layout: size the window to its preferred size (or the size the script +/// gave it) and place every control (`bounds` relative to the window's top-left). +pub fn layout(w: &mut ScriptWindow) { + let pref = preferred(&w.root); + let (width, height) = match w.root.fixed_bounds { + Some(b) if b[2] > b[0] && b[3] > b[1] => (b[2] - b[0], b[3] - b[1]), + _ => (pref[0], pref[1]), + }; + place(&mut w.root, 0.0, 0.0, width, height); +} + +// ---------------------------------------------------------------- commands + +fn window_p(s: &Session, p: &Value, cmd: &str) -> Result { + let key = p.get("window"); + let w = match key { + Some(Value::Number(n)) => n.as_u64().map(|n| n as u32).filter(|n| s.script_ui.window(*n).is_some()), + Some(Value::String(t)) => s.script_ui.windows.iter().find(|w| w.title == *t).map(|w| w.id), + None if s.script_ui.windows.len() == 1 => s.script_ui.windows.first().map(|w| w.id), + _ => None, + }; + w.ok_or_else(|| crate::commands::bad(cmd, "no such script window (give `window`: id or title from scriptui.list)")) +} + +fn widget_p(s: &Session, win: u32, p: &Value, cmd: &str) -> Result { + let w = s.script_ui.window(win).ok_or_else(|| crate::commands::bad(cmd, "window closed"))?; + let key = p.get("widget").ok_or_else(|| crate::commands::bad(cmd, "missing `widget` (id, `#id`, properties.name or text)"))?; + w.root.lookup(key).map(|w| w.id).ok_or_else(|| crate::commands::bad(cmd, format!("no control {key} in window {win}"))) +} + +fn dispatch(s: &mut Session, ev: ScriptUiEvent) -> Result { + let f = s.script_ui.dispatch.ok_or_else(|| EngineError::Other("scripting is not available in this build".into()))?; + f(s, &ev).map_err(EngineError::Other) +} + +/// `scriptui.list`: the open script windows. +pub(crate) fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!( + s.script_ui + .windows + .iter() + .map(|w| json!({"window": w.id, "title": w.title, "kind": w.kind, "script": w.script, "modal": w.modal, "size": [w.root.bounds[2], w.root.bounds[3]]})) + .collect::>() + )) +} + +/// `scriptui.get`: one window's control tree. +pub(crate) fn get(s: &mut Session, p: &Value) -> Result { + let id = window_p(s, p, "scriptui.get")?; + Ok(serde_json::to_value(s.script_ui.window(id)).unwrap_or(Value::Null)) +} + +/// `scriptui.click`: press a button / toggle a checkbox / pick a radio button. +pub(crate) fn click(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.click"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "click".into(), value: Value::Null }) +} + +/// `scriptui.set`: set a control's value (edit text, slider, checkbox, list selection) as the +/// user would, firing onChanging / onChange. +pub(crate) fn set(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.set"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + let mut value = p.get("value").cloned().ok_or_else(|| crate::commands::bad(c, "missing `value`"))?; + // List selections by item text. + if matches!(w.kind, WidgetKind::DropDownList | WidgetKind::ListBox) + && let Value::String(t) = &value + { + let i = w.items.iter().position(|it| it == t).ok_or_else(|| crate::commands::bad(c, format!("no item `{t}` (items: {:?})", w.items)))?; + value = json!(i); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "change".into(), value }) +} + +/// `scriptui.close`: close a window (a dialog's `show()` returns `result`, default 2 = Cancel). +pub(crate) fn close(s: &mut Session, p: &Value) -> Result { + let window = window_p(s, p, "scriptui.close")?; + let result = p.get("result").cloned().unwrap_or(json!(2)); + dispatch(s, ScriptUiEvent { window, widget: 0, kind: "close".into(), value: result }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn w(kind: WidgetKind, text: &str) -> Widget { + Widget { kind, text: text.into(), enabled: true, visible: true, ..Default::default() } + } + + #[test] + fn column_and_row_layout() { + let mut row = w(WidgetKind::Group, ""); + row.id = 1; + let mut ok = w(WidgetKind::Button, "OK"); + ok.id = 2; + let mut cancel = w(WidgetKind::Button, "Cancel"); + cancel.id = 3; + row.children = vec![ok, cancel]; + let mut label = w(WidgetKind::StaticText, "Name:"); + label.id = 4; + let mut root = w(WidgetKind::Window, "Dialog"); + root.children = vec![label, row]; + let mut win = ScriptWindow { id: 1, root, visible: true, ..Default::default() }; + layout(&mut win); + let r = &win.root; + // Window: 15 px margins, children stacked with 10 px spacing. + let row = r.find(1).unwrap(); + assert_eq!(row.bounds[2], 170.0, "two 80 px buttons + 10 spacing"); + assert_eq!(r.bounds[2], 200.0); + assert_eq!(r.find(4).unwrap().bounds[1], 15.0); + assert_eq!(row.bounds[1], 15.0 + 16.0 + 10.0); + // Centred (the default alignChildren of windows). + assert_eq!(row.bounds[0], 15.0); + let label = r.find(4).unwrap(); + assert!((label.bounds[0] + label.bounds[2] / 2.0 - 100.0).abs() < 1e-9); + assert_eq!(r.find(3).unwrap().bounds[0], 15.0 + 80.0 + 10.0); + // alignChildren left / fill. + win.root.align_children = vec!["fill".into(), "top".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[2], 170.0); + win.root.align_children = vec!["left".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[0], 15.0); + // preferredSize wins. + win.root.find_mut(2).unwrap().preferred_size = Some([120.0, 30.0]); + layout(&mut win); + assert_eq!(win.root.find(2).unwrap().bounds[2..], [120.0, 30.0]); + // Lookup by name / text / id. + win.root.find_mut(2).unwrap().name = "ok".into(); + assert_eq!(win.root.lookup(&json!("ok")).unwrap().id, 2); + assert_eq!(win.root.lookup(&json!("Cancel")).unwrap().id, 3); + assert_eq!(win.root.lookup(&json!("#4")).unwrap().id, 4); + assert_eq!(win.root.lookup(&json!(1)).unwrap().id, 1); + } + + #[test] + fn publish_replaces_a_hosts_windows() { + let mut ui = ScriptUi::default(); + let win = |id: u32, visible: bool| ScriptWindow { id, visible, root: w(WidgetKind::Window, ""), ..Default::default() }; + ui.publish(7, vec![win(1, true), win(2, true)]); + ui.publish(8, vec![win(3, true)]); + assert_eq!(ui.windows.iter().map(|w| (w.id, w.host)).collect::>(), [(1, 7), (2, 7), (3, 8)]); + ui.publish(7, vec![win(2, true), win(1, false)]); + assert_eq!(ui.windows.iter().map(|w| w.id).collect::>(), [2, 3]); + let j = serde_json::to_value(&ui).unwrap(); + assert_eq!(j["windows"][0]["root"]["type"], "window"); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index 1a3370e4..155204d4 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -30,10 +30,8 @@ impl Session { self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + let current = self.project.clone(); + self.history.trim(levels, ¤t); self.prefs_revision += 1; } diff --git a/crates/engine/src/tests_history.rs b/crates/engine/src/tests_history.rs new file mode 100644 index 00000000..326c37c0 --- /dev/null +++ b/crates/engine/src/tests_history.rs @@ -0,0 +1,135 @@ +//! Branching history: undo then a new step keeps the undone line as a branch; the tree lists +//! every state; jumping to any state (on any branch) restores it and keeps everything else. + +use serde_json::{Value, json}; + +use crate::Session; + +fn names(s: &Session) -> Vec { + s.project.comps().next().map(|(_, c)| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +fn solid(s: &mut Session, name: &str) { + s.execute("layer.newSolid", json!({"name": name, "color": "#ff0000", "width": 10, "height": 10})).unwrap(); +} + +fn list(s: &mut Session) -> Vec { + s.execute("edit.history.list", json!({})).unwrap()["states"].as_array().cloned().unwrap() +} + +fn index_of(states: &[Value], label: &str, nth: usize) -> u64 { + states.iter().filter(|n| n["label"] == label).nth(nth).unwrap_or_else(|| panic!("no state {label} in {states:?}"))["index"].as_u64().unwrap() +} + +#[test] +fn undo_then_new_step_keeps_a_branch() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + solid(&mut s, "C"); + // Linear so far: Original, New Composition, 3 solids. + let st = list(&mut s); + assert_eq!(st.len(), 5, "{st:?}"); + assert!(st.iter().all(|n| n["depth"] == 0)); + assert_eq!(st.last().unwrap()["current"], true); + // Undo twice (B and C undone), then a new step: AE would forget B and C. + assert!(s.undo() && s.undo()); + let st = list(&mut s); + assert_eq!(st.iter().filter(|n| n["future"] == true).count(), 2); + solid(&mut s, "D"); + assert!(s.history.redo.is_empty(), "the new step clears Redo, as in After Effects"); + assert_eq!(s.history.branches.len(), 1); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6, "the undone states are still listed: {st:?}"); + let branch: Vec<&Value> = st.iter().filter(|n| n["depth"] == 1).collect(); + assert_eq!(branch.len(), 2, "B and C sit on a branch: {st:?}"); + // Jump to C on the old branch. + let c = st.iter().filter(|n| n["depth"] == 1).last().unwrap()["index"].as_u64().unwrap(); + let r = s.execute("edit.history.goto", json!({"index": c})).unwrap(); + assert_eq!(r["label"], "New Solid", "{r}"); + assert_eq!(names(&s), ["C", "B", "A"]); + // Now D is the branch, and Undo walks the B/C line. + assert_eq!(s.history.branches.len(), 1); + assert_eq!(s.history.branches[0].steps.len(), 1); + assert!(s.undo()); + assert_eq!(names(&s), ["B", "A"]); + assert!(s.redo()); + // ...and back to D by id. + let st = list(&mut s); + let d = st.iter().find(|n| n["depth"] == 1).unwrap()["id"].as_str().unwrap().to_string(); + s.execute("edit.history.goto", json!({"id": d})).unwrap(); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6); + assert_eq!(st.iter().filter(|n| n["current"] == true).count(), 1); + // The original state. + let root = index_of(&st, "Original", 0); + s.execute("edit.history.goto", json!({"index": root})).unwrap(); + assert!(s.project.comps().next().is_none()); + assert!(s.history.undo.is_empty()); + assert_eq!(list(&mut s).len(), 6, "nothing is lost by jumping"); + // Relative steps along the line. + s.execute("edit.history.goto", json!({"steps": 2})).unwrap(); + assert_eq!(names(&s), ["A"]); + assert!(s.execute("edit.history.goto", json!({"id": "nope"})).is_err()); + assert!(s.execute("edit.history.goto", json!({})).is_err()); +} + +#[test] +fn branches_of_branches_and_purge() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + s.undo(); + solid(&mut s, "C"); // branch 1: B + s.undo(); + solid(&mut s, "D"); // branch 2: C + let st = list(&mut s); + assert_eq!(st.len(), 6, "{st:?}"); + // Go to B, undo to A, make E: D's line becomes a branch, and a branch of the new line. + let b = st.iter().find(|n| n["depth"] == 1 && n["label"] == "New Solid").unwrap()["index"].as_u64().unwrap(); + s.execute("edit.history.goto", json!({"index": b})).unwrap(); + let which = names(&s); + assert!(which == ["B", "A"] || which == ["C", "A"], "{which:?}"); + s.undo(); + solid(&mut s, "E"); + let st = list(&mut s); + assert_eq!(st.len(), 7, "{st:?}"); + // Every state is reachable and restores its own layers. + let mut seen = std::collections::BTreeSet::new(); + for i in 0..st.len() { + s.execute("edit.history.goto", json!({"index": i})).unwrap(); + seen.insert(names(&s).join(",")); + assert_eq!(list(&mut s).len(), 7); + } + for want in ["", "A", "B,A", "C,A", "D,A", "E,A"] { + assert!(seen.contains(want), "{want:?} not reachable: {seen:?}"); + } + s.execute("edit.purgeUndo", json!({})).unwrap(); + assert_eq!(list(&mut s).len(), 1); +} + +#[test] +fn undo_levels_trim_branches() { + let mut s = Session::default(); + s.prefs.general.undo_levels = 3; + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + for n in ["A", "B", "C", "D", "E"] { + solid(&mut s, n); + } + assert_eq!(s.history.undo.len(), 3); + s.undo(); + s.undo(); + solid(&mut s, "F"); + // The branch (D, E) grew from a state still kept. + assert_eq!(s.history.branch_states(), 2); + for n in ["G", "H", "I"] { + solid(&mut s, n); + } + // Its parent fell off the end of the undo levels: the branch goes too. + assert_eq!(s.history.branch_states(), 0); + assert_eq!(list(&mut s).len(), 4); +} diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index 9926e04c..9407ddd1 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -250,3 +250,56 @@ fn liquify_stroke_adds_the_effect_and_warps() { assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); assert!(px(&s, 0.0, 55, 50)[3] > 0.99); } + +fn pin_keys(s: &Session, id: u64, pin: u64) -> Vec<(f64, [f64; 2])> { + let l = layer(s, id); + let pos = l.props.find_group(pin).unwrap().get("position").unwrap().clone(); + pos.keys.iter().map(|k| (k.time.seconds(), k.value.as_vec2())).collect() +} + +#[test] +fn puppet_pin_recording_makes_keys_at_the_frame_rate() { + let (mut s, id) = setup(); + let a = s.execute("puppet.addPin", json!({"layer": id, "position": [10, 30]})).unwrap(); + s.execute("puppet.addPin", json!({"layer": id, "position": [90, 30]})).unwrap(); + let pin = a["pin"].as_u64().unwrap(); + s.set_time(Tick::from_seconds_f64(0.5)); + // A 0.5 s drag sampled at irregular wall-clock times: right 30 px, linear in time. + let samples: Vec = [0.0, 0.07, 0.2, 0.33, 0.41, 0.5].iter().map(|t| json!([t, 10.0 + 60.0 * t, 30.0])).collect(); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "smoothing": 0})).unwrap(); + // 0.5 s at 30 fps from 0.5 s: frames 15..=30, one key per frame. + assert_eq!(r["frames"], 16, "{r}"); + let keys = pin_keys(&s, id, pin); + let rec: Vec<&(f64, [f64; 2])> = keys.iter().filter(|k| k.0 >= 0.5 - 1e-9).collect(); + assert_eq!(rec.len(), 16, "{keys:?}"); + for (i, (t, v)) in rec.iter().enumerate() { + assert!((t - (0.5 + i as f64 / 30.0)).abs() < 1e-6, "{t}"); + assert!((v[0] - (10.0 + 60.0 * i as f64 / 30.0)).abs() < 1e-6, "{v:?}"); + } + assert_eq!(keys.len(), 17, "the pin's first key (0 s) is kept"); + assert_eq!(s.time().seconds(), 0.5, "the current time returns to where recording began"); + // The recording deforms the layer: by 1 s the left edge has followed the pin 30 px right. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + assert!(px(&s, 1.0, 55, 50)[3] < 0.5); + // Speed 200 %: the same drag plays back over twice the frames; recording again replaces + // the keys in the span. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "speed": 200, "smoothing": 0})).unwrap(); + assert_eq!(r["frames"], 31); + let keys = pin_keys(&s, id, pin); + assert_eq!(keys.len(), 32); + let k = keys.iter().find(|k| (k.0 - (0.5 + 10.0 / 30.0)).abs() < 1e-6).unwrap(); + assert!((k.1[0] - (10.0 + 60.0 * 5.0 / 30.0)).abs() < 1e-6, "{k:?}"); + // Smoothing removes the extraneous keys of a straight-line drag. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert!(r["keys"].as_u64().unwrap() < 6, "{r}"); + // One undo step per recording. + s.undo(); + assert_eq!(pin_keys(&s, id, pin).len(), 32); + // Record Options are tool state (serde for agents). + let o = s.execute("puppet.recordOptions", json!({"speed": 50, "smoothing": 0, "useDraftDeformation": true, "showMesh": true})).unwrap(); + assert_eq!(o["speed"], 50.0); + assert!(s.state.puppet.record_draft && s.state.puppet.record_show_mesh); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert_eq!(r["frames"], 9, "50 %: half the frames"); + assert!(s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": []})).is_err()); +} diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index b4863d54..a4386199 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -358,9 +358,8 @@ impl Session { return self.offload_analysis(crate::offload::WorkerJob::Track { comp, layer, tracker, direction, settings, points, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Analyze Track".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Analyze Track", self.project.clone(), levels); let shared = Arc::new(TrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index 635e3260..b6e9ff55 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -16,6 +16,11 @@ effectcraft-media = { workspace = true } effectcraft-expr = { workspace = true } effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } +effectcraft-plugin = { workspace = true } + +# WebAssembly effect plug-ins run in wasmi on native builds. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +effectcraft-plugin = { workspace = true, features = ["wasm"] } [dev-dependencies] serde_json = { workspace = true } @@ -23,3 +28,4 @@ effectcraft-psd = { workspace = true } effectcraft-svg = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-time = { workspace = true } +rayon = { workspace = true } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index b5de7714..c379d091 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -58,6 +58,8 @@ pub fn session() -> Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), script: Some(effectcraft_script::runner), + script_ui: effectcraft_engine::scriptui::ScriptUi { dispatch: Some(effectcraft_script::dispatch_ui), ..Default::default() }, + plugin_loader: effectcraft_plugin::wasm_available().then_some(effectcraft_plugin::loader as effectcraft_engine::PluginLoader), ..Default::default() } } diff --git a/crates/host/tests/determinism.rs b/crates/host/tests/determinism.rs new file mode 100644 index 00000000..58a1b933 --- /dev/null +++ b/crates/host/tests/determinism.rs @@ -0,0 +1,74 @@ +//! Deterministic rendering (better than After Effects): the same project renders bit-identical +//! frames every time and whatever the number of render threads. Renders the demo comps (and a +//! comp of seeded-random effects: particles, grain, noise, wiggle expressions, Shatter) twice +//! on one thread and on several, and compares every float of every pixel. + +use effectcraft_engine::Session; +use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::{RenderOpts, Renderer}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +fn render(s: &Session, p: &Project, comp: ItemId, t: f64) -> Vec { + let mut r = Renderer::new(p, s.footage.as_ref(), RenderOpts { scale: 0.25, ..Default::default() }); + r.expr = s.expr.as_deref(); + // No layer cache: every render computes everything. + let img = r.comp_frame(comp, Tick::from_seconds_f64(t)); + img.data.iter().flat_map(|px| px.iter().map(|c| c.to_bits())).collect() +} + +fn pool(n: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new().num_threads(n).build().expect("thread pool") +} + +/// A comp of effects that draw random numbers (all seeded per instance). +fn random_comp(s: &mut Session) -> ItemId { + s.execute("comp.new", json!({"name": "Random", "width": 480, "height": 270, "frameRate": 24, "duration": 4})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let solid = |s: &mut Session, name: &str, color: &str| { + s.execute("layer.newSolid", json!({"name": name, "color": color, "width": 480, "height": 270})).unwrap()["layer"].as_u64().unwrap() + }; + let a = solid(s, "Particles", "#000000"); + for fx in ["ec.sim.ccparticleworld", "ec.noise.addgrain"] { + s.execute("effect.apply", json!({"layer": a, "effect": fx})).unwrap(); + } + let b = solid(s, "Noise", "#336699"); + for fx in ["ec.noise.fractal", "ec.sim.ccsnowfall", "ec.noise.noisehls"] { + s.execute("effect.apply", json!({"layer": b, "effect": fx})).unwrap(); + } + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/position", "expression": "wiggle(3, 40)"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/opacity", "expression": "random(40, 90)"})).unwrap(); + let c = solid(s, "Shatter", "#cc8844"); + s.execute("layer.setTransform", json!({"layer": c, "prop": "scale", "value": [50, 50]})).unwrap(); + s.execute("effect.apply", json!({"layer": c, "effect": "ec.sim.shatter"})).unwrap(); + cid +} + +#[test] +fn renders_are_bit_identical_across_runs_and_thread_counts() { + let mut s = effectcraft_host::session(); + s.replace_project(effectcraft_engine::demo::demo_project(), None); + let random = random_comp(&mut s); + let p = (*s.project).clone(); + let mut comps: Vec<(ItemId, f64)> = p.comps().map(|(id, c)| (*id, c.duration.seconds())).collect(); + comps.sort_by_key(|c| c.0); + assert!(comps.len() >= 3, "demo comps + the random comp"); + assert!(comps.iter().any(|c| c.0 == random)); + let (one, many) = (pool(1), pool(6)); + let mut frames = 0; + for (comp, dur) in comps { + for t in [0.0, dur * 0.37, dur * 0.81] { + let a = one.install(|| render(&s, &p, comp, t)); + let b = many.install(|| render(&s, &p, comp, t)); + let c = many.install(|| render(&s, &p, comp, t)); + let d = one.install(|| render(&s, &p, comp, t)); + let name = &p.item(comp).unwrap().name; + assert!(!a.is_empty()); + assert!(a == b, "{name} @ {t:.2}s: 1 thread vs 6 threads differ"); + assert!(b == c, "{name} @ {t:.2}s: two 6-thread renders differ"); + assert!(a == d, "{name} @ {t:.2}s: two 1-thread renders differ"); + frames += 1; + } + } + assert!(frames >= 9); +} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml new file mode 100644 index 00000000..4081b61b --- /dev/null +++ b/crates/plugin/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "effectcraft-plugin" +description = "EffectCraft effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1) into the effect registry" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +# The WebAssembly runtime (wasmi, a pure-Rust interpreter). Hosts enable it on native; without +# it `load_wasm` reports that WebAssembly plug-ins are unavailable. +wasm = ["dep:wasmi"] + +[dependencies] +effectcraft-effects = { workspace = true } +serde_json = { workspace = true } +wasmi = { workspace = true, optional = true } + +[dev-dependencies] +effectcraft-keyframe = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-raster = { workspace = true } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs new file mode 100644 index 00000000..1231d171 --- /dev/null +++ b/crates/plugin/src/lib.rs @@ -0,0 +1,66 @@ +//! # effectcraft-plugin (L3) +//! +//! Loads **WebAssembly effect plug-ins** (plug-in API v1, see `docs/plugins.md`) into the effect +//! registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or `.wat`) module with +//! no imports that exports: +//! +//! | export | signature | | +//! |---|---|---| +//! | `memory` | memory | linear memory shared with the host | +//! | `ec_api_version` | `() -> i32` | must return 1 | +//! | `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | a UTF-8 JSON [`PluginManifest`] in memory | +//! | `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +//! | `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; 0 = ok | +//! +//! For each render the host asks `ec_alloc` for one buffer holding the parameters (`nparams` +//! little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s, +//! premultiplied RGBA, row-major), calls `ec_render`, and reads the pixels back. +//! +//! Modules run in the [wasmi](https://github.com/wasmi-labs/wasmi) interpreter (pure Rust, +//! MIT/Apache-2.0) with deterministic floats, no imports (no files, network or clock) and a +//! fuel limit per frame, so a plug-in can't escape its sandbox, hang the app, or render +//! differently on another machine. The runtime is the `wasm` feature (on for the desktop app and +//! the CLI); without it [`load_wasm`] says plug-ins are unavailable. + +use effectcraft_effects::EffectSpec; +pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, register_plugin}; + +#[cfg(feature = "wasm")] +mod wasm; + +/// Whether this build can load WebAssembly plug-ins. +pub fn wasm_available() -> bool { + cfg!(feature = "wasm") +} + +/// Load and register a WebAssembly plug-in (`bytes` = a `.wasm` binary or `.wat` text; +/// `source` = where it came from, for listings). Returns the registered effect. +pub fn load_wasm(bytes: &[u8], source: &str) -> Result<&'static EffectSpec, String> { + #[cfg(feature = "wasm")] + { + let p = wasm::WasmPlugin::new(bytes, source)?; + register_plugin(std::sync::Arc::new(p)) + } + #[cfg(not(feature = "wasm"))] + { + let _ = (bytes, source); + Err("WebAssembly plug-ins are not available in this build".into()) + } +} + +/// [`load_wasm`] for the engine's plug-in loader hook: `{id, name, category, version, api}`. +pub fn loader(bytes: &[u8], source: &str) -> Result { + let spec = load_wasm(bytes, source)?; + let m = effectcraft_effects::plugin::plugin(spec.id).map(|p| p.manifest().clone()); + Ok(serde_json::json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()).unwrap_or_default(), + "api": PLUGIN_API_VERSION, + "params": spec.params.len(), + })) +} + +#[cfg(all(test, feature = "wasm"))] +mod tests; diff --git a/crates/plugin/src/tests.rs b/crates/plugin/src/tests.rs new file mode 100644 index 00000000..9363ab3e --- /dev/null +++ b/crates/plugin/src/tests.rs @@ -0,0 +1,164 @@ +//! WebAssembly plug-ins: load, register, render through the effect registry, sandbox limits. + +use effectcraft_effects::{Buf, EffectCtx, EffectEnv, Image}; +use effectcraft_project::build::Ids; + +use crate::load_wasm; + +fn load_err(r: Result<&'static effectcraft_effects::EffectSpec, String>) -> String { + match r { + Err(e) => e, + Ok(s) => panic!("unexpectedly loaded {}", s.id), + } +} + +/// A plug-in in WebAssembly text: multiplies RGB by `gain`, adds `lift` × `tint` when `on`. +fn gain_module(id: &str, render_body: &str) -> String { + let manifest = format!( + r#"{{"api":1,"id":"{id}","name":"Gain {id}","category":"Test Plug-ins","version":"1.0","params":[{{"id":"gain","name":"Gain","type":"slider","default":0.5,"min":0,"max":4}},{{"id":"on","name":"Lift","type":"checkbox","default":false}},{{"id":"tint","name":"Tint","type":"color","default":[0,0,1,1]}}]}}"# + ); + let len = manifest.len(); + let escaped = manifest.replace('\\', "\\\\").replace('"', "\\\""); + format!( + r#"(module + (memory (export "memory") 2) + (data (i32.const 16) "{escaped}") + (func (export "ec_api_version") (result i32) i32.const 1) + (func (export "ec_manifest_ptr") (result i32) i32.const 16) + (func (export "ec_manifest_len") (result i32) i32.const {len}) + (func (export "ec_alloc") (param $n i32) (result i32) + (local $need i32) + local.get $n + i32.const 131071 + i32.add + i32.const 16 + i32.shr_u + memory.size + i32.sub + local.tee $need + i32.const 0 + i32.gt_s + if + local.get $need + memory.grow + drop + end + i32.const 65536) + (func (export "ec_render") (param $px i32) (param $w i32) (param $h i32) (param $params i32) (param $np i32) (param $t f64) (param $scale f64) (result i32) + (local $i i32) (local $end i32) (local $g f32) (local $on f64) + {render_body} + local.get $params + f64.load + f32.demote_f64 + local.set $g + local.get $params + f64.load offset=8 + local.set $on + local.get $px + local.get $w + local.get $h + i32.mul + i32.const 16 + i32.mul + i32.add + local.set $end + local.get $px + local.set $i + block $done + loop $l + local.get $i + local.get $end + i32.ge_u + br_if $done + local.get $i + local.get $i + f32.load + local.get $g + f32.mul + f32.store + local.get $i + local.get $i + f32.load offset=4 + local.get $g + f32.mul + f32.store offset=4 + local.get $i + local.get $i + f32.load offset=8 + local.get $g + f32.mul + local.get $on + local.get $params + f64.load offset=32 + f64.mul + f32.demote_f64 + f32.add + f32.store offset=8 + local.get $i + i32.const 16 + i32.add + local.set $i + br $l + end + end + i32.const 0))"# + ) +} + +fn render(spec: &effectcraft_effects::EffectSpec, gain: f64, on: bool) -> Image { + let mut next = 1; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [4.0, 2.0]); + let mut params = effectcraft_effects::flatten_params(&g, &mut |p| p.value.clone()); + params.values.insert("gain".into(), effectcraft_keyframe::Value::Scalar(gain)); + params.values.insert("on".into(), effectcraft_keyframe::Value::Bool(on)); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [4.0, 2.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + let buf = Buf { img: Image::filled(4, 2, [0.8, 0.6, 0.2, 1.0]), offset: [0.0; 2], scale: 1.0 }; + (spec.render)(&ctx, buf).img +} + +#[test] +fn wasm_plugin_registers_and_renders() { + let spec = load_wasm(gain_module("org.test.gain", "").as_bytes(), "gain.wat").unwrap(); + assert_eq!(spec.id, "org.test.gain"); + assert_eq!(spec.category, "Test Plug-ins"); + assert_eq!(spec.params.len(), 3); + // Found like a built-in, listed under its own category. + assert!(effectcraft_effects::find("org.test.gain").is_some()); + assert!(effectcraft_effects::lookup("Gain org.test.gain").is_some()); + assert!(effectcraft_effects::all().iter().any(|e| e.id == "org.test.gain")); + assert!(effectcraft_effects::categories().contains(&"Test Plug-ins")); + let img = render(spec, 0.5, false); + assert_eq!(img.get(1, 1), [0.4, 0.3, 0.1, 1.0]); + // The checkbox and colour parameters arrive flattened (tint blue = 1 → +1 on blue). + let img = render(spec, 1.0, true); + assert_eq!(img.get(3, 0), [0.8, 0.6, 1.2, 1.0]); + // Deterministic, and safe to run on many threads at once (each takes an instance). + let a: Vec = (0..8).map(|_| render(spec, 0.25, false)).collect(); + std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| render(spec, 0.25, false))).collect(); + for h in hs { + assert_eq!(h.join().unwrap(), a[0]); + } + }); + assert!(a.iter().all(|i| *i == a[0])); + // The same id can't be registered twice. + assert!(load_err(load_wasm(gain_module("org.test.gain", "").as_bytes(), "again.wat")).contains("already registered")); +} + +#[test] +fn wasm_plugins_are_sandboxed() { + // A runaway loop runs out of fuel: the frame comes back unchanged instead of hanging. + let spec = load_wasm(gain_module("org.test.spin", "(loop $spin br $spin)").as_bytes(), "spin.wat").unwrap(); + let img = render(spec, 0.5, false); + assert_eq!(img.get(0, 0), [0.8, 0.6, 0.2, 1.0]); + // No imports (files, clock, network…). + let imports = r#"(module (import "env" "now" (func $now (result f64))) (memory (export "memory") 1))"#; + assert!(load_err(load_wasm(imports.as_bytes(), "imports.wat")).contains("import")); + // Garbage, wrong API version, bad manifests. + assert!(load_wasm(b"\0asm nope", "bad.wasm").is_err()); + let v2 = gain_module("org.test.v2", "") + .replace("(func (export \"ec_api_version\") (result i32) i32.const 1)", "(func (export \"ec_api_version\") (result i32) i32.const 2)"); + assert!(load_err(load_wasm(v2.as_bytes(), "v2.wat")).contains("API 2")); + let reserved = gain_module("ec.blur.mine", ""); + assert!(load_err(load_wasm(reserved.as_bytes(), "reserved.wat")).contains("reserved")); +} diff --git a/crates/plugin/src/wasm.rs b/crates/plugin/src/wasm.rs new file mode 100644 index 00000000..8015f7ee --- /dev/null +++ b/crates/plugin/src/wasm.rs @@ -0,0 +1,151 @@ +//! The wasmi-backed [`EffectPlugin`] (plug-in API v1 ABI, see the crate docs). + +use std::sync::Mutex; + +use wasmi::{Config, Engine, Instance, Linker, Memory, Module, Store, TypedFunc, WasmParams, WasmResults}; + +use crate::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams}; + +/// Instruction budget per frame: a fixed allowance plus some per pixel. Generous for per-pixel +/// work, but a runaway loop stops instead of hanging the render. +const FUEL_BASE: u64 = 50_000_000; +const FUEL_PER_PIXEL: u64 = 20_000; + +type RenderFn = TypedFunc<(i32, i32, i32, i32, i32, f64, f64), i32>; + +/// One instantiated module (renders running on several threads each take their own). +struct Inst { + store: Store<()>, + instance: Instance, + memory: Memory, + alloc: TypedFunc, + render: RenderFn, +} + +impl Inst { + fn func(&self, name: &str) -> Result, String> { + self.instance.get_typed_func::(&self.store, name).map_err(|_| missing(name)) + } +} + +fn missing(name: &str) -> String { + format!("the module exports no `{name}` (or with the wrong signature)") +} + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +fn instantiate(engine: &Engine, module: &Module) -> Result { + let mut store = Store::new(engine, ()); + store.set_fuel(FUEL_BASE).map_err(err)?; + let linker = Linker::<()>::new(engine); + let instance = linker.instantiate_and_start(&mut store, module).map_err(|e| format!("instantiation failed: {e}"))?; + let memory = instance.get_memory(&store, "memory").ok_or("the module exports no `memory`")?; + let alloc = instance.get_typed_func::(&store, "ec_alloc").map_err(|_| missing("ec_alloc"))?; + let render = instance.get_typed_func::<(i32, i32, i32, i32, i32, f64, f64), i32>(&store, "ec_render").map_err(|_| missing("ec_render"))?; + Ok(Inst { store, instance, memory, alloc, render }) +} + +/// A loaded WebAssembly effect plug-in. +pub struct WasmPlugin { + manifest: PluginManifest, + source: String, + engine: Engine, + module: Module, + /// Idle instances. + pool: Mutex>, +} + +impl WasmPlugin { + pub fn new(bytes: &[u8], source: &str) -> Result { + let mut config = Config::default(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, bytes).map_err(|e| format!("not a valid WebAssembly module: {e}"))?; + if let Some(i) = module.imports().next() { + return Err(format!("plug-ins can't import anything (the module imports `{}.{}`)", i.module(), i.name())); + } + let mut inst = instantiate(&engine, &module)?; + let version = inst.func::<(), i32>("ec_api_version")?; + let v = version.call(&mut inst.store, ()).map_err(err)?; + if v != PLUGIN_API_VERSION as i32 { + return Err(format!("the module implements plug-in API {v}, this build implements API {PLUGIN_API_VERSION}")); + } + let ptr = inst.func::<(), i32>("ec_manifest_ptr")?.call(&mut inst.store, ()).map_err(err)?; + let len = inst.func::<(), i32>("ec_manifest_len")?.call(&mut inst.store, ()).map_err(err)?; + let mut text = vec![0u8; len.max(0) as usize]; + inst.memory.read(&inst.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; + let manifest: PluginManifest = serde_json::from_slice(&text).map_err(|e| format!("bad manifest JSON: {e}"))?; + manifest.validate()?; + Ok(WasmPlugin { manifest, source: source.to_string(), engine, module, pool: Mutex::new(vec![inst]) }) + } + + fn take(&self) -> Result { + if let Some(i) = self.pool.lock().ok().and_then(|mut p| p.pop()) { + return Ok(i); + } + instantiate(&self.engine, &self.module) + } + + fn give_back(&self, inst: Inst) { + if let Ok(mut p) = self.pool.lock() { + p.push(inst); + } + } +} + +impl EffectPlugin for WasmPlugin { + fn manifest(&self) -> &PluginManifest { + &self.manifest + } + + fn source(&self) -> String { + self.source.clone() + } + + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + let n = frame.pixels.len(); + let params_bytes = params.flat.len() * 8; + let total = params_bytes + n * 16; + if total > i32::MAX as usize { + return Err("frame too large for a 32-bit plug-in".into()); + } + let mut inst = self.take()?; + inst.store.set_fuel(FUEL_BASE + FUEL_PER_PIXEL * n as u64).map_err(err)?; + let base = inst.alloc.call(&mut inst.store, total as i32).map_err(err)? as u32 as usize; + // Parameters, then pixels (little-endian, as WebAssembly memory is). + let mut buf: Vec = Vec::with_capacity(total); + for v in params.flat { + buf.extend_from_slice(&v.to_le_bytes()); + } + for px in frame.pixels.iter() { + for c in px { + buf.extend_from_slice(&c.to_le_bytes()); + } + } + inst.memory.write(&mut inst.store, base, &buf).map_err(|e| format!("ec_alloc returned a buffer out of bounds: {e}"))?; + let px_ptr = (base + params_bytes) as i32; + let r = inst + .render + .call(&mut inst.store, (px_ptr, frame.width as i32, frame.height as i32, base as i32, params.flat.len() as i32, time, frame.scale)) + .map_err(|e| format!("ec_render trapped: {e}")); + let r = match r { + Ok(0) => { + inst.memory.read(&inst.store, base + params_bytes, &mut buf[params_bytes..]).map_err(err)?; + for (px, b) in frame.pixels.iter_mut().zip(buf[params_bytes..].chunks_exact(16)) { + for (c, v) in px.iter_mut().zip(b.chunks_exact(4)) { + *c = f32::from_le_bytes([v[0], v[1], v[2], v[3]]); + } + } + Ok(()) + } + Ok(code) => Err(format!("ec_render returned error {code}")), + Err(e) => Err(e), + }; + if r.is_ok() { + self.give_back(inst); + } + r + } +} diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs index a1e9e5d4..a7a85213 100644 --- a/crates/script/src/lib.rs +++ b/crates/script/src/lib.rs @@ -30,6 +30,14 @@ //! * **Security**: scripts can import/open/save projects and render, but `File` reads are limited //! to the project's folder and writes (and the network) are off unless Preferences ▸ Scripting //! & Expressions ▸ Allow Scripts to Write Files and Access Network is on. +//! * **ScriptUI** (`scriptui.js`, `ui.rs`): `Window` (dialog / palette / window), `Panel`, +//! `Group`, `Button`, `StaticText`, `EditText`, `Checkbox`, `RadioButton`, `Slider`, +//! `Progressbar`, `DropDownList`, `ListBox`, `TabbedPanel`/`Tab`, with `add()`, `orientation`, +//! `alignChildren`, `alignment`, `margins`, `spacing`, `preferredSize`, `onClick` / +//! `onChange` / `onChanging` / `onClose`, `show()` / `close()` and `layout.layout()`. Windows +//! are published to [`Session::script_ui`](effectcraft_engine::Session::script_ui) for +//! frontends and agents (`scriptui.*` commands); modal dialogs block `show()` until they close. +//! Scripts in the ScriptUI Panels folder run with `this` = a dockable `Panel`. //! //! Entry points: [`install`] sets [`Session::script`](effectcraft_engine::Session::script), which //! the `script.run` command, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`), the Script Console @@ -40,19 +48,22 @@ pub mod matchnames; mod model; mod runtime; +mod ui; use effectcraft_engine::{ScriptRequest, Session}; pub use runtime::{Outcome, ScriptError, run}; +pub use ui::dispatch_ui; /// The [`effectcraft_engine::ScriptRunner`] this crate provides. pub fn runner(s: &mut Session, req: &ScriptRequest) -> serde_json::Value { run(s, req).to_json() } -/// Enable scripting on a session. +/// Enable scripting (and ScriptUI) on a session. pub fn install(s: &mut Session) { s.script = Some(runner); + s.script_ui.dispatch = Some(dispatch_ui); } /// Run `code` as a one-off script named `name`. @@ -62,3 +73,5 @@ pub fn run_code(s: &mut Session, code: &str, name: &str) -> Outcome { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_ui; diff --git a/crates/script/src/model.rs b/crates/script/src/model.rs index 03e80bb5..18270590 100644 --- a/crates/script/src/model.rs +++ b/crates/script/src/model.rs @@ -59,8 +59,8 @@ pub fn query(s: &mut Session, kind: &str, a: &J) -> R { } "labels" => Ok(json!(Label::ALL.iter().map(|l| l.name()).collect::>())), "effects" => Ok(json!( - effectcraft_engine::effects::registry() - .iter() + effectcraft_engine::effects::all() + .into_iter() .map(|e| json!({"id": e.id, "name": e.name, "matchName": matchnames::effect(e.id), "category": e.category})) .collect::>() )), diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs index 61b49595..750918f4 100644 --- a/crates/script/src/runtime.rs +++ b/crates/script/src/runtime.rs @@ -26,6 +26,9 @@ pub struct Outcome { /// `writeLn` / `$.writeln` / `alert` output, one entry per line. pub output: Vec, pub error: Option, + /// The script is waiting in a modal ScriptUI dialog's `show()` (it goes on when the dialog + /// closes; see [`crate::dispatch_ui`]). + pub waiting: bool, } /// An uncaught script error. @@ -42,36 +45,49 @@ impl Outcome { self.error.is_none() } pub fn to_json(&self) -> J { - json!({ + let mut j = json!({ "ok": self.ok(), "result": self.result, "output": self.output.join("\n"), "error": self.error.as_ref().map(|e| json!({"message": e.message, "line": e.line, "column": e.column, "file": e.file})), - }) + }); + if self.waiting { + j["waiting"] = json!(true); + } + j } } /// The session and per-run state while a script runs (the natives reach it here; nothing /// borrowed ever enters the JS heap). -struct Active { - session: Session, - output: Vec, +pub(crate) struct Active { + pub(crate) session: Session, + pub(crate) output: Vec, /// Open undo groups: (name, project snapshot when the outermost group began). - groups: Vec<(String, Arc)>, + pub(crate) groups: Vec<(String, Arc)>, /// `app.scheduleTask` queue: (id, code, repeat). - tasks: Vec<(u32, String, bool)>, - next_task: u32, - name: String, + pub(crate) tasks: Vec<(u32, String, bool)>, + pub(crate) next_task: u32, + pub(crate) name: String, + /// Running on a script-host thread: modal dialogs hand the session back to the caller + /// through this link while they wait. + pub(crate) link: Option, +} + +impl Active { + pub(crate) fn new(session: Session, name: &str) -> Active { + Active { session, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: name.to_string(), link: None } + } } thread_local! { - static ACTIVE: RefCell> = const { RefCell::new(None) }; + pub(crate) static ACTIVE: RefCell> = const { RefCell::new(None) }; /// The Script Console's persistent context (never dropped, like the expression runtime: boa's /// own thread-local heap may be torn down first at thread exit). - static CONSOLE: RefCell>> = const { RefCell::new(None) }; + pub(crate) static CONSOLE: RefCell>> = const { RefCell::new(None) }; } -fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { +pub(crate) fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { ACTIVE.with(|a| match a.try_borrow_mut() { Ok(mut g) => match g.as_mut() { Some(act) => f(act), @@ -81,11 +97,11 @@ fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result) -> boa_engine::JsError { +pub(crate) fn throw(msg: impl Into) -> boa_engine::JsError { JsNativeError::error().with_message(msg.into()).into() } -fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { +pub(crate) fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { Ok(args.get(i).cloned().unwrap_or_default().to_string(ctx)?.to_std_string_escaped()) } @@ -97,7 +113,7 @@ fn arg_json(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { serde_json::from_str(&s).map_err(|e| throw(format!("bad parameters: {e}"))) } -fn js_str(s: &str) -> JsValue { +pub(crate) fn js_str(s: &str) -> JsValue { JsValue::from(JsString::from(s)) } @@ -205,7 +221,7 @@ fn native_undo(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult) { +pub(crate) fn collapse(s: &mut Session, name: &str, snap: &Arc) { let h = &mut s.history; let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { // Nothing was recorded (or history was reset by New / Open Project). @@ -349,28 +365,32 @@ fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult Result { +pub(crate) fn new_context() -> Result { let mut ctx = Context::default(); let limits = ctx.runtime_limits_mut(); limits.set_loop_iteration_limit(200_000_000); limits.set_recursion_limit(2000); - let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 6] = [ + let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 9] = [ (js_string!("__exec"), 2, native_exec), (js_string!("__query"), 2, native_query), (js_string!("__print"), 1, native_print), (js_string!("__undo"), 2, native_undo), (js_string!("__task"), 3, native_task), (js_string!("__file"), 3, native_file), + (js_string!("__uiNewId"), 0, crate::ui::native_new_id), + (js_string!("__uiModal"), 1, crate::ui::native_modal), + (js_string!("__uiLayoutNative"), 1, crate::ui::native_layout), ]; for (name, len, f) in natives { ctx.register_global_callable(name, len, NativeFunction::from_fn_ptr(f)).map_err(|e| e.to_string())?; } ctx.eval(Source::from_bytes(PRELUDE)).map_err(|e| format!("script prelude: {e}"))?; + ctx.eval(Source::from_bytes(crate::ui::SCRIPTUI)).map_err(|e| format!("ScriptUI prelude: {e}"))?; Ok(ctx) } /// Turn a boa error into a message with line and column (`code` is the user's script). -fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { +pub(crate) fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { // Thrown values: `throw new Error("x")`, `throw "x"`, native errors. let display = e.to_string(); let message = match e.as_opaque() { @@ -442,7 +462,7 @@ fn position(msg: &str, file: &str, code: &str) -> (Option, Option) { } /// Evaluate `code` in `ctx` and convert the result. -fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { +pub(crate) fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { let src = Source::from_bytes(code.as_bytes()).with_path(std::path::Path::new(file)); let script = boa_engine::Script::parse(src, None, ctx).map_err(|e| script_error(&e, ctx, file, code))?; let v = script.evaluate(ctx).map_err(|e| script_error(&e, ctx, file, code))?; @@ -466,8 +486,8 @@ fn result_json(v: &JsValue, ctx: &mut Context) -> J { } /// Restores the caller's session when a run ends (on every exit path). -struct Restore<'a> { - target: &'a mut Session, +pub(crate) struct Restore<'a> { + pub(crate) target: &'a mut Session, } impl Drop for Restore<'_> { @@ -478,8 +498,54 @@ impl Drop for Restore<'_> { } } +/// End of a script step: close undo groups left open, drop selections that no longer exist, +/// and take the output so far. +pub(crate) fn settle() -> Vec { + with_active(|a| { + if let Some((name, snap)) = a.groups.first().cloned() { + collapse(&mut a.session, &name, &snap); + a.groups.clear(); + } + a.session.sanitize_state(); + Ok(std::mem::take(&mut a.output)) + }) + .unwrap_or_default() +} + +/// Evaluate a script file's code, then the one-shot tasks it scheduled (`app.scheduleTask`). +pub(crate) fn eval_with_tasks(ctx: &mut Context, code: &str, file: &str) -> Result { + let mut r = eval_in(ctx, code, file); + let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); + for (_, code, repeat) in tasks { + if repeat { + continue; + } + if let Err(e) = eval_in(ctx, &code, "scheduled task") + && r.is_ok() + { + r = Err(e); + } + } + r +} + +pub(crate) fn outcome(output: Vec, result: Result) -> Outcome { + let mut out = Outcome { output, ..Default::default() }; + match result { + Ok(v) => out.result = v, + Err(e) => out.error = Some(e), + } + out +} + /// Run a script against `session`: the object model sees and edits that session through engine /// commands. Script errors come back in [`Outcome::error`]. +/// +/// On native builds a script file runs on its own script-host thread, so a ScriptUI dialog's +/// `show()` can wait for the user (the session comes back here meanwhile, and the dialog's +/// events reach the host through [`crate::dispatch_ui`]); palettes and panels keep their host +/// alive for their handlers. The Script Console (and the web build) run inline, where dialogs +/// don't block. pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { let running = ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)); if running { @@ -488,59 +554,49 @@ pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { ..Default::default() }; } + #[cfg(not(target_arch = "wasm32"))] + if !req.console { + return crate::ui::run_threaded(session, req); + } + run_inline(session, req) +} + +/// Run on this thread (the Script Console's persistent context, or a fresh one). +pub(crate) fn run_inline(session: &mut Session, req: &ScriptRequest) -> Outcome { + let host = if req.console { crate::ui::CONSOLE_HOST } else { crate::ui::new_host_id() }; let s = std::mem::take(session); - ACTIVE.with(|a| *a.borrow_mut() = Some(Active { session: s, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: req.name.to_string() })); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, req.name))); let guard = Restore { target: session }; let file = if req.name.is_empty() { "script" } else { req.name }; - let mut result = if req.console { + let (result, windows) = if req.console { CONSOLE.with(|c| { let mut c = c.borrow_mut(); if c.is_none() { match new_context() { Ok(ctx) => *c = Some(ManuallyDrop::new(ctx)), - Err(e) => return Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => return (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } } let ctx = c.as_mut().map(|m| &mut **m).expect("console context"); - eval_in(ctx, req.code, file) + let r = eval_in(ctx, req.code, file); + (r, crate::ui::snapshot(ctx)) }) } else { match new_context() { Ok(mut ctx) => { - let r = eval_in(&mut ctx, req.code, file); - // app.scheduleTask: run the queued one-shot tasks after the script, in order. - let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); - let mut r = r; - for (_, code, repeat) in tasks { - if repeat { - continue; - } - if let Err(e) = eval_in(&mut ctx, &code, "scheduled task") - && r.is_ok() - { - r = Err(e); - } + let r = eval_with_tasks(&mut ctx, req.code, file); + let windows = crate::ui::snapshot(&mut ctx); + if windows.iter().any(|w| w.visible) { + // Palettes and panels outlive the run: keep their context for the handlers. + crate::ui::keep_inline(host, ctx); } - r + (r, windows) } - Err(e) => Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } }; - // Close undo groups left open; drop selections that no longer exist. - let output = with_active(|a| { - if let Some((name, snap)) = a.groups.first().cloned() { - collapse(&mut a.session, &name, &snap); - a.groups.clear(); - } - a.session.sanitize_state(); - Ok(std::mem::take(&mut a.output)) - }) - .unwrap_or_default(); + let output = settle(); drop(guard); - let mut out = Outcome { output, ..Default::default() }; - match result.as_mut() { - Ok(v) => out.result = std::mem::take(v), - Err(e) => out.error = Some(e.clone()), - } - out + crate::ui::publish(session, host, req.name, windows); + outcome(output, result) } diff --git a/crates/script/src/scriptui.js b/crates/script/src/scriptui.js new file mode 100644 index 00000000..e214be30 --- /dev/null +++ b/crates/script/src/scriptui.js @@ -0,0 +1,544 @@ +// EffectCraft scripting: ScriptUI, the dialog / panel object model After Effects scripts use +// (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, `onClick`, `show()`…). +// +// Written from the behaviour Adobe's public JavaScript Tools Guide documents. Controls are plain +// JavaScript objects; after each script step the host reads `__uiSnapshot()` (a JSON +// description of the open windows) and lays them out, frontends draw them, and user actions come +// back through `__uiDispatch(window, control, kind, valueJson)`. Modal dialogs block in +// `show()` (`__uiModal`) until they close, as in After Effects. + +var __uiWins = []; +// show() results of closed dialogs, by window id. +var __uiResults = {}; + +function __uiModalResult(id) { + var r = __uiResults[id]; + if (r !== undefined) { + delete __uiResults[id]; + return r; + } + return __uiWin(id) ? null : 2; +} + +function __dim(w, h) { + var d = [w, h]; + Object.defineProperty(d, "width", { get: function () { return this[0]; }, set: function (v) { this[0] = v; } }); + Object.defineProperty(d, "height", { get: function () { return this[1]; }, set: function (v) { this[1] = v; } }); + return d; +} +function __bounds4(l, t, r, b) { + var a = [l, t, r, b]; + Object.defineProperty(a, "left", { get: function () { return this[0]; } }); + Object.defineProperty(a, "top", { get: function () { return this[1]; } }); + Object.defineProperty(a, "right", { get: function () { return this[2]; } }); + Object.defineProperty(a, "bottom", { get: function () { return this[3]; } }); + Object.defineProperty(a, "x", { get: function () { return this[0]; } }); + Object.defineProperty(a, "y", { get: function () { return this[1]; } }); + Object.defineProperty(a, "width", { get: function () { return this[2] - this[0]; } }); + Object.defineProperty(a, "height", { get: function () { return this[3] - this[1]; } }); + return a; +} +function __toBounds(b) { + if (b === undefined || b === null) return null; + if (b.length === 4) return [__num(b[0]), __num(b[1]), __num(b[2]), __num(b[3])]; + if (b.width !== undefined) { + var x = b.x || b.left || 0, y = b.y || b.top || 0; + return [x, y, x + __num(b.width), y + __num(b.height)]; + } + return null; +} + +var __uiTypes = { + window: "Window", panel: "Panel", group: "Group", button: "Button", iconbutton: "IconButton", + statictext: "StaticText", edittext: "EditText", checkbox: "Checkbox", radiobutton: "RadioButton", + slider: "Slider", scrollbar: "Scrollbar", progressbar: "Progressbar", dropdownlist: "DropDownList", + listbox: "ListBox", tabbedpanel: "TabbedPanel", tab: "Tab", image: "ScriptUIImage" +}; +var __uiContainers = { window: 1, panel: 1, group: 1, tabbedpanel: 1, tab: 1 }; + +function Window(type, title, bounds, props) { + if (!(this instanceof Window)) return new Window(type, title, bounds, props); + type = String(type === undefined ? "window" : type).toLowerCase(); + if (type.indexOf("{") >= 0) throw __err("ScriptUI resource strings are not supported: build the window with add()"); + if (type !== "dialog" && type !== "palette" && type !== "window") throw __err("Bad window type " + type); + __uiInit(this, "window", null, null, bounds, title, props); + this.__kind = type; + this.__root = true; + __uiWins.push(this); +} +Window.alert = function (s) { alert(s); }; +Window.confirm = function (s) { return confirm(s); }; +Window.prompt = function (m, d) { return prompt(m, d); }; +Window.find = function (type, title) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].text === title) return __uiWins[i]; + return null; +}; + +function Panel() {} +function Group() {} +function Button() {} +function IconButton() {} +function StaticText() {} +function EditText() {} +function Checkbox() {} +function RadioButton() {} +function Slider() {} +function Scrollbar() {} +function Progressbar() {} +function DropDownList() {} +function ListBox() {} +function TabbedPanel() {} +function Tab() {} +function ScriptUIImage() {} +function ListItem() {} + +var ScriptUI = { + version: "6.2.2", + frameworkName: "EffectCraft", + environment: { keyboardState: { shiftKey: false, ctrlKey: false, altKey: false, metaKey: false, keyName: "" } }, + FontStyle: { REGULAR: 0, BOLD: 1, ITALIC: 2, BOLDITALIC: 3 }, + BrushType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + PenType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + Alignment: { LEFT: "left", RIGHT: "right", TOP: "top", BOTTOM: "bottom", CENTER: "center", FILL: "fill" }, + newFont: function (name, style, size) { return { name: name, style: style, size: size }; }, + newImage: function () { return {}; }, + getResourceText: function (t) { return t; }, + events: { createEvent: function (t) { return { type: t }; } } +}; + +function __uiGraphics() { + return { + newBrush: function (t, c) { return { type: t, color: c }; }, + newPen: function (t, c, w) { return { type: t, color: c, lineWidth: w }; }, + newPath: function () {}, moveTo: function () {}, lineTo: function () {}, rectPath: function () {}, + ellipsePath: function () {}, fillPath: function () {}, strokePath: function () {}, closePath: function () {}, + drawString: function () {}, drawOSControl: function () {}, drawImage: function () {}, drawFocusRing: function () {}, + measureString: function (s) { return __dim(String(s).length * 7, 16); }, + font: null, foregroundColor: null, backgroundColor: null, disabledForegroundColor: null, disabledBackgroundColor: null + }; +} + +function __uiInit(o, type, win, parent, bounds, text, props) { + o.type = type; + o.window = win || o; + o.parent = parent || null; + o.__id = win ? win.__next++ : 0; + if (!win) { + o.__next = 1; + o.__wid = __uiNewId(); + } + o.properties = props || {}; + o.text = text === undefined || text === null ? "" : String(text); + o.enabled = true; + o.visible = true; + o.helpTip = ""; + o.children = []; + o.__ub = __toBounds(bounds); + o.__lb = [0, 0, 0, 0]; + o.preferredSize = __dim(-1, -1); + o.minimumSize = __dim(0, 0); + o.maximumSize = __dim(10000, 10000); + o.alignment = null; + o.graphics = __uiGraphics(); + o.__listeners = {}; + if (o.properties.name) o.name = o.properties.name; + if (__uiContainers[type]) { + o.orientation = type === "group" ? "row" : type === "tabbedpanel" ? "stack" : "column"; + o.alignChildren = type === "group" ? ["center", "center"] : ["center", "top"]; + o.spacing = 10; + o.margins = type === "window" ? [15, 15, 15, 15] : type === "panel" ? [10, 15, 10, 10] : type === "tab" ? [10, 10, 10, 10] : [0, 0, 0, 0]; + var self = o; + o.layout = { + layout: function () { __uiLayout(self.window); }, + resize: function () { __uiLayout(self.window); }, + margins: o.margins, + spacing: o.spacing + }; + } + if (type === "checkbox" || type === "radiobutton") o.value = !!o.properties.value; + if (type === "slider" || type === "scrollbar" || type === "progressbar") { + o.value = 0; + o.minvalue = 0; + o.maxvalue = 100; + } + if (type === "edittext") { + o.characters = 0; + o.textselection = ""; + o.active = false; + } + if (type === "dropdownlist" || type === "listbox") { + o.items = []; + o.__sel = []; + var it = o.properties.items; + if (it) for (var i = 0; i < it.length; i++) o.add("item", it[i]); + } + if (type === "tabbedpanel") o.__sel = []; + return o; +} + +function __uiProto() { + return { + add: function (type, bounds, text, props) { + var t = String(type).toLowerCase(); + if (t === "item" || t === "separator") return this.__addItem(t === "separator" ? "-" : bounds); + if (!__uiContainers[this.type]) throw __err("add(): a " + this.type + " can't contain controls"); + var ctor = __uiTypes[t]; + if (!ctor || t === "window") throw __err("add(): unknown control type " + type); + var c = Object.create(__ecGlobal[ctor].prototype); + // Value arguments after `text`: slider (value, min, max), progressbar (value, max), lists (items). + __uiInit(c, t, this.window, this, bounds, typeof text === "object" && text !== null && !(text instanceof Array) ? "" : text, props); + if ((t === "dropdownlist" || t === "listbox") && text instanceof Array) { + for (var i = 0; i < text.length; i++) c.add("item", text[i]); + c.text = ""; + } + if (t === "slider" || t === "scrollbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (arguments.length > 3 && typeof props === "number") { + c.minvalue = props; + c.maxvalue = arguments[4] === undefined ? 100 : __num(arguments[4]); + c.properties = arguments[5] || {}; + } + } + if (t === "progressbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (typeof props === "number") { + c.maxvalue = props; + c.properties = arguments[4] || {}; + } + } + if (t === "checkbox" || t === "radiobutton") c.value = false; + if (t === "edittext" && c.properties.multiline) c.__multiline = true; + if (t === "edittext" && c.properties.readonly) c.__readonly = true; + if (c.properties && c.properties.name) c.name = c.properties.name; + this.children.push(c); + if (t === "tab" && this.type === "tabbedpanel" && this.__sel.length === 0) this.__sel = [this.children.length - 1]; + return c; + }, + remove: function (what) { + var list = this.items && this.type !== "tabbedpanel" ? this.items : this.children; + var i = typeof what === "number" ? what : list.indexOf(what); + if (i < 0 || i >= list.length) return; + list.splice(i, 1); + if (this.items) { + for (var k = 0; k < this.items.length; k++) this.items[k].index = k; + this.__sel = this.__sel.filter(function (s) { return s !== i; }).map(function (s) { return s > i ? s - 1 : s; }); + } + }, + removeAll: function () { + if (this.items) { + this.items = []; + this.__sel = []; + } else this.children = []; + }, + find: function (text) { + var list = this.items || this.children; + for (var i = 0; i < list.length; i++) if (list[i].text === text) return list[i]; + return null; + }, + __addItem: function (text) { + if (!this.items) throw __err("add(\"item\"): only lists have items"); + var it = Object.create(ListItem.prototype); + it.text = String(text === undefined ? "" : text); + it.type = text === "-" ? "separator" : "item"; + it.index = this.items.length; + it.parent = this; + it.image = null; + it.checked = false; + this.items.push(it); + return it; + }, + show: function () { + if (!this.__root) { + this.visible = true; + return; + } + this.__shown = true; + this.__closed = false; + this.__fire("onShow"); + if (this.__kind === "dialog") { + this.__modal = true; + var r = __uiModal(this.__wid); + this.__modal = false; + return r === null ? undefined : r; + } + return undefined; + }, + hide: function () { + if (this.__root) { + this.close(2); + } else this.visible = false; + }, + close: function (result) { + if (!this.__root) return; + if (this.__fire("onClose") === false) return; + this.__closed = true; + this.__shown = false; + this.__result = result === undefined ? 0 : result; + if (this.__modal) __uiResults[this.__wid] = this.__result; + }, + center: function () {}, + notify: function (ev) { + var e = ev || (this.type === "edittext" || this.type === "slider" || this.type === "dropdownlist" || this.type === "listbox" ? "onChange" : "onClick"); + if (e.indexOf("on") !== 0) e = "on" + e.charAt(0).toUpperCase() + e.slice(1); + this.__fire(e); + }, + addEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + (this.__listeners[k] = this.__listeners[k] || []).push(fn); + }, + removeEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + if (this.__listeners[k]) this.__listeners[k] = this.__listeners[k].filter(function (f) { return f !== fn; }); + }, + dispatchEvent: function (e) { this.__fire("on" + String(e.type || e).charAt(0).toUpperCase() + String(e.type || e).slice(1)); }, + __fire: function (name) { + var r; + if (typeof this[name] === "function") r = this[name].call(this); + var l = this.__listeners[name.slice(2).toLowerCase()]; + if (l) for (var i = 0; i < l.length; i++) l[i].call(this, { type: name.slice(2).toLowerCase(), target: this }); + return r; + }, + __handlers: function () { + var out = []; + var names = ["onClick", "onChange", "onChanging", "onDoubleClick", "onClose", "onShow", "onActivate", "onDeactivate", "onResize"]; + for (var i = 0; i < names.length; i++) { + var n = names[i]; + if (typeof this[n] === "function" || (this.__listeners[n.slice(2).toLowerCase()] || []).length) out.push(n); + } + return out; + }, + toString: function () { return "[object " + (__uiTypes[this.type] || "Window") + "]"; } + }; +} + +(function () { + var all = ["Window", "Panel", "Group", "Button", "IconButton", "StaticText", "EditText", "Checkbox", "RadioButton", "Slider", + "Scrollbar", "Progressbar", "DropDownList", "ListBox", "TabbedPanel", "Tab", "ScriptUIImage"]; + for (var i = 0; i < all.length; i++) { + var C = __ecGlobal[all[i]]; + var p = __uiProto(); + for (var k in p) C.prototype[k] = p[k]; + Object.defineProperty(C.prototype, "bounds", { + get: function () { var b = this.__ub || this.__lb; return __bounds4(b[0], b[1], b[2], b[3]); }, + set: function (v) { this.__ub = __toBounds(v); }, + configurable: true + }); + Object.defineProperty(C.prototype, "size", { + get: function () { var b = this.bounds; return __dim(b[2] - b[0], b[3] - b[1]); }, + set: function (v) { + var b = this.bounds; + var w = v.width !== undefined ? v.width : v[0], h = v.height !== undefined ? v.height : v[1]; + this.__ub = [b[0], b[1], b[0] + __num(w), b[1] + __num(h)]; + }, + configurable: true + }); + Object.defineProperty(C.prototype, "location", { + get: function () { var b = this.bounds; return __dim(b[0], b[1]); }, + set: function (v) { + var b = this.bounds; + var x = v.x !== undefined ? v.x : v[0], y = v.y !== undefined ? v.y : v[1]; + this.__ub = [__num(x), __num(y), __num(x) + b[2] - b[0], __num(y) + b[3] - b[1]]; + }, + configurable: true + }); + } + var sel = { + get: function () { + var self = this; + if (this.type === "tabbedpanel") return this.__sel.length ? this.children[this.__sel[0]] : null; + var items = this.__sel.map(function (i) { return self.items[i]; }); + if (this.properties.multiselect) return items.length ? items : null; + return items.length ? items[0] : null; + }, + set: function (v) { + var self = this; + var list = this.type === "tabbedpanel" ? this.children : this.items; + var idx = function (x) { return typeof x === "number" ? x : list.indexOf(x); }; + if (v === null || v === undefined) this.__sel = []; + else if (v instanceof Array) this.__sel = v.map(idx).filter(function (i) { return i >= 0 && i < list.length; }); + else { + var i = idx(v); + this.__sel = i >= 0 && i < list.length ? [i] : []; + } + if (this.items) for (var k = 0; k < this.items.length; k++) this.items[k].selected = this.__sel.indexOf(k) >= 0; + }, + configurable: true + }; + Object.defineProperty(DropDownList.prototype, "selection", sel); + Object.defineProperty(ListBox.prototype, "selection", sel); + Object.defineProperty(TabbedPanel.prototype, "selection", sel); + Object.defineProperty(ListItem.prototype, "toString", { value: function () { return this.text; } }); +})(); + +// A dockable ScriptUI panel (scripts in the ScriptUI Panels folder run with `this` = it). +function __uiDockPanel(title) { + var p = Object.create(Panel.prototype); + __uiInit(p, "panel", null, null, undefined, title, {}); + p.type = "panel"; + p.__kind = "panel"; + p.__root = true; + p.__shown = true; + p.margins = [10, 10, 10, 10]; + __uiWins.push(p); + return p; +} + +function __uiAlign(v) { + if (v === null || v === undefined) return []; + if (v instanceof Array) return v.map(function (x) { return String(x).toLowerCase(); }); + return [String(v).toLowerCase()]; +} +function __uiMargins(m) { + if (m === null || m === undefined) return null; + if (typeof m === "number") return [m, m, m, m]; + if (m.length === 4) return [m[0], m[1], m[2], m[3]]; + return null; +} +function __uiSize(d) { + if (!d) return null; + var w = d.width !== undefined ? d.width : d[0], h = d.height !== undefined ? d.height : d[1]; + w = Number(w); + h = Number(h); + if (!(w > 0) && !(h > 0)) return null; + return [w > 0 ? w : 0, h > 0 ? h : 0]; +} + +function __uiJson(c) { + var ps = __uiSize(c.preferredSize); + if (c.type === "edittext" && c.characters > 0) ps = [c.characters * 7 + 12, ps ? ps[1] : 0]; + var o = { + id: c.__id, + type: c.type, + name: String(c.name || c.properties.name || ""), + text: String(c.text === undefined || c.text === null ? "" : c.text), + enabled: c.enabled !== false, + visible: c.visible !== false, + helpTip: String(c.helpTip || ""), + handlers: c.__handlers(), + children: [] + }; + if (ps) o.preferredSize = ps; + if (c.__ub && !c.__root) o.fixedBounds = c.__ub; + if (c.__root && c.__ub) o.fixedBounds = [0, 0, c.__ub[2] - c.__ub[0], c.__ub[3] - c.__ub[1]]; + if (__uiContainers[c.type]) { + o.orientation = String(c.orientation || "").toLowerCase(); + o.alignChildren = __uiAlign(c.alignChildren); + var m = __uiMargins(c.margins); + if (m) o.margins = m; + if (typeof c.spacing === "number") o.spacing = c.spacing; + var kids = c.children; + for (var i = 0; i < kids.length; i++) o.children.push(__uiJson(kids[i])); + } + if (c.alignment !== null && c.alignment !== undefined) o.alignment = __uiAlign(c.alignment); + if (c.type === "checkbox" || c.type === "radiobutton") o.checked = !!c.value; + if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") { + o.value = Number(c.value) || 0; + o.min = Number(c.minvalue) || 0; + o.max = Number(c.maxvalue); + if (!(o.max === o.max)) o.max = 100; + } + if (c.items) { + o.items = c.items.map(function (it) { return String(it.text); }); + o.selection = c.__sel.slice(); + } + if (c.type === "tabbedpanel") o.activeTab = c.__sel.length ? c.__sel[0] : 0; + if (c.type === "edittext") { + o.multiline = !!c.__multiline; + o.readOnly = !!c.__readonly; + } + return o; +} + +// The open windows of this script, for the host. +function __uiSnapshot() { + var out = []; + for (var i = 0; i < __uiWins.length; i++) { + var w = __uiWins[i]; + out.push({ id: w.__wid, kind: w.__kind, title: String(w.text || ""), visible: !!w.__shown && !w.__closed, modal: !!w.__modal, root: __uiJson(w) }); + } + // Closed windows are forgotten. + __uiWins = __uiWins.filter(function (w) { return !w.__closed; }); + return JSON.stringify(out); +} + +function __uiOpenCount() { + var n = 0; + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__shown && !__uiWins[i].__closed) n++; + return n; +} + +function __uiWin(id) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__wid === id) return __uiWins[i]; + return null; +} +function __uiFind(c, id) { + if (c.__id === id) return c; + for (var i = 0; i < (c.children || []).length; i++) { + var r = __uiFind(c.children[i], id); + if (r) return r; + } + return null; +} + +// layout.layout(): lay the window out now (the host does it before drawing anyway) so scripts +// can read bounds and sizes. +function __uiLayout(win) { + var b = JSON.parse(__uiLayoutNative(JSON.stringify({ id: win.__wid, kind: win.__kind, title: String(win.text || ""), visible: true, root: __uiJson(win) }))); + // Bounds are relative to the parent container, as in ScriptUI. + (function apply(c, ox, oy) { + var r = b[c.__id]; + if (!r) return; + c.__lb = [r[0] - ox, r[1] - oy, r[0] - ox + r[2], r[1] - oy + r[3]]; + for (var i = 0; i < (c.children || []).length; i++) apply(c.children[i], r[0], r[1]); + })(win, 0, 0); +} + +// A user action from the frontend. +function __uiDispatch(winId, id, kind, valueJson) { + var w = __uiWin(winId); + if (!w) return; + var v = valueJson === undefined || valueJson === "" ? null : JSON.parse(valueJson); + if (kind === "close") { + w.close(v === null ? 2 : v); + return; + } + var c = __uiFind(w, id); + if (!c || c.enabled === false) return; + var radio = function () { + var sib = c.parent ? c.parent.children : []; + for (var i = 0; i < sib.length; i++) if (sib[i] !== c && sib[i].type === "radiobutton") sib[i].value = false; + c.value = true; + }; + if (kind === "click") { + if (c.type === "checkbox") c.value = !c.value; + else if (c.type === "radiobutton") radio(); + else if (c.type === "tab" && c.parent) { + c.parent.selection = c; + c.parent.__fire("onChange"); + return; + } + var handled = c.__handlers().indexOf("onClick") >= 0; + c.__fire("onClick"); + // Dialog buttons without handlers: OK closes with 1, Cancel with 2. + if (!handled && c.type === "button" && w.__kind === "dialog") { + var n = String(c.name || c.properties.name || c.text).toLowerCase(); + if (c === w.defaultElement || n === "ok") w.close(1); + else if (c === w.cancelElement || n === "cancel") w.close(2); + } + return; + } + // change / changing + if (c.type === "edittext") c.text = v === null ? "" : String(v); + else if (c.type === "statictext") c.text = v === null ? "" : String(v); + else if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") c.value = Number(v); + else if (c.type === "checkbox") c.value = !!v; + else if (c.type === "radiobutton") { if (v) radio(); else c.value = false; } + else if (c.type === "dropdownlist" || c.type === "listbox" || c.type === "tabbedpanel") c.selection = v; + if (c.type === "edittext" || c.type === "slider" || c.type === "scrollbar") c.__fire("onChanging"); + if (kind === "change") { + if (c.type === "checkbox" || c.type === "radiobutton") c.__fire("onClick"); + else c.__fire("onChange"); + } +} diff --git a/crates/script/src/tests_ui.rs b/crates/script/src/tests_ui.rs new file mode 100644 index 00000000..667a3242 --- /dev/null +++ b/crates/script/src/tests_ui.rs @@ -0,0 +1,299 @@ +//! ScriptUI: windows built by scripts are published to the session, drawn by frontends and +//! driven through the `scriptui.*` commands; dialogs block `show()` until they close; dockable +//! panels; File ▸ Scripts (installed and sample scripts). + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{WidgetKind, WindowKind}; +use serde_json::{Value, json}; + +use crate::run_code; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), ..Default::default() }; + crate::install(&mut s); + s.config = Some(Arc::new(effectcraft_engine::config::MemoryConfig::default())); + s +} + +fn comp_names(s: &Session) -> Vec { + s.project.items.values().filter(|i| i.as_comp().is_some()).map(|i| i.name.clone()).collect() +} + +fn layer_names(s: &Session) -> Vec { + s.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +const PALETTE: &str = r#" +var made = 0; +var w = new Window("palette", "Maker"); +w.orientation = "column"; +w.alignChildren = ["fill", "top"]; +var name = w.add("edittext", undefined, "Box"); +name.characters = 12; +var big = w.add("checkbox", undefined, "Big"); +var row = w.add("group"); +var make = row.add("button", undefined, "Make", { name: "make" }); +var label = row.add("statictext", undefined, "none yet"); +var size = w.add("slider", undefined, 50, 10, 200); +var kind = w.add("dropdownlist", undefined, ["Solid", "Null"]); +kind.selection = 0; +var echo = ""; +name.onChange = function () { echo = name.text; }; +make.onClick = function () { + var c = app.project.activeItem; + var s = big.value ? 200 : Math.round(size.value); + if (kind.selection.index === 0) c.layers.addSolid([1, 0, 0], name.text, s, s, 1); + else c.layers.addNull().name = name.text; + made++; + label.text = "made " + made; + writeLn("made " + name.text + " " + s); +}; +w.show(); +"palette up" +"#; + +#[test] +fn palette_controls_run_their_handlers() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 2})).unwrap(); + let o = run_code(&mut s, PALETTE, "maker.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!("palette up")); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert_eq!((w.kind, w.title.as_str(), w.script.as_str()), (WindowKind::Palette, "Maker", "maker.jsx")); + let kinds: Vec = w.root.children.iter().map(|c| c.kind).collect(); + assert_eq!(kinds, [WidgetKind::EditText, WidgetKind::Checkbox, WidgetKind::Group, WidgetKind::Slider, WidgetKind::DropDownList]); + // Laid out: the window has a size, the fill-aligned controls span its width. + assert!(w.root.bounds[2] > 100.0 && w.root.bounds[3] > 100.0, "{:?}", w.root.bounds); + let edit = &w.root.children[0]; + assert_eq!(edit.bounds[2], w.root.bounds[2] - 30.0); + assert_eq!(w.root.children[3].value, 50.0); + assert_eq!((w.root.children[3].min, w.root.children[3].max), (10.0, 200.0)); + assert_eq!(w.root.children[4].items, ["Solid", "Null"]); + assert_eq!(w.root.children[4].selection, [0]); + assert!(w.root.children[2].children[0].handlers.contains(&"onClick".to_string())); + // Agents: list, read, click. + let list = s.execute("scriptui.list", json!({})).unwrap(); + assert_eq!(list[0]["title"], "Maker"); + let tree = s.execute("scriptui.get", json!({"window": w.id})).unwrap(); + assert_eq!(tree["root"]["children"][2]["children"][0]["name"], "make"); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Box 50", "{r}"); + assert_eq!(layer_names(&s), ["Box"]); + // Handlers' edits are undoable engine commands. + assert!(s.undo()); + assert!(layer_names(&s).is_empty()); + // Set values like a user: text (onChange), checkbox, slider, list by item text. + s.execute("scriptui.set", json!({"widget": "#1", "value": "Crate"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Big"})).unwrap(); + s.execute("scriptui.set", json!({"window": "Maker", "widget": 7, "value": "Null"})).unwrap(); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Crate 200"); + assert_eq!(layer_names(&s), ["Crate"]); + assert!(s.active_comp().unwrap().layers[0].source == effectcraft_engine::project::LayerSource::Null); + // The handler updated a label: the published tree follows. + let w = s.script_ui.windows[0].clone(); + assert_eq!(w.root.children[2].children[1].text, "made 2"); + assert!(w.root.children[1].checked); + assert_eq!(w.root.children[4].selection, [1]); + // Errors. + assert!(s.execute("scriptui.click", json!({"widget": "nope"})).is_err()); + assert!(s.execute("scriptui.get", json!({"window": 999_999})).is_err()); + // Closing the window ends the script host. + s.execute("scriptui.close", json!({})).unwrap(); + assert!(s.script_ui.windows.is_empty()); + assert!(s.execute("scriptui.list", json!({})).unwrap().as_array().unwrap().is_empty()); +} + +const DIALOG: &str = r#" +var d = new Window("dialog", "New Comp"); +d.alignChildren = "left"; +var e = d.add("edittext", undefined, "Untitled"); +var fps = d.add("radiobutton", undefined, "24 fps"); +var fps30 = d.add("radiobutton", undefined, "30 fps"); +fps.value = true; +var g = d.add("group"); +g.add("button", undefined, "Cancel", { name: "cancel" }); +var ok = g.add("button", undefined, "OK", { name: "ok" }); +writeLn("before"); +var r = d.show(); +writeLn("result " + r + " " + e.text + " " + (fps30.value ? 30 : 24)); +if (r === 1) app.project.items.addComp(e.text, 100, 100, 1, 1, fps30.value ? 30 : 24); +r +"#; + +#[test] +fn dialogs_block_show_until_they_close() { + let mut s = session(); + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + // The script waits in show(); the session is back here meanwhile. + assert!(o.waiting, "{o:?}"); + assert_eq!(o.output, ["before"]); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert!(w.modal && w.kind == WindowKind::Dialog); + // The app keeps working while the dialog is up. + s.execute("comp.new", json!({"name": "Other", "width": 64, "height": 64, "duration": 1})).unwrap(); + s.execute("scriptui.set", json!({"widget": "#1", "value": "Shot 010"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "30 fps"})).unwrap(); + let w = s.script_ui.windows[0].clone(); + assert!(!w.root.children[1].checked && w.root.children[2].checked, "radio buttons are exclusive"); + // OK closes the dialog with 1: the script runs on. + let r = s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + assert_eq!(r["output"], "result 1 Shot 010 30", "{r}"); + assert_eq!(r["result"], 1); + assert!(r.get("waiting").is_none()); + assert!(s.script_ui.windows.is_empty()); + let names = comp_names(&s); + assert!(names.contains(&"Shot 010".to_string()) && names.contains(&"Other".to_string()), "{names:?}"); + // Cancel (or closing the window) returns 2. + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + assert!(o.waiting); + let r = s.execute("scriptui.close", json!({})).unwrap(); + assert_eq!(r["output"], "result 2 Untitled 24"); + assert_eq!(comp_names(&s).len(), 2); +} + +#[test] +fn layout_and_handler_errors() { + let mut s = session(); + let code = r#" + var w = new Window("palette", "L"); + var g = w.add("group"); + g.orientation = "row"; + var a = g.add("button", undefined, "A"); + var b = g.add("button", undefined, "B"); + b.preferredSize = [120, 30]; + var p = w.add("panel", undefined, "Options"); + p.add("checkbox", undefined, "One"); + var bad = w.add("button", undefined, "Bad"); + bad.onClick = function () { nope(); }; + w.layout.layout(true); + w.show(); + [a.bounds.width, b.size.width, b.size.height, b.bounds[0] - a.bounds[0], w.bounds.width > 200, p.bounds.top > a.bounds.bottom] + "#; + let o = run_code(&mut s, code, "layout.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!([80, 120, 30, 90, true, true])); + // An error in a handler is reported, the window stays. + let r = s.execute("scriptui.click", json!({"widget": "Bad"})).unwrap(); + assert_eq!(r["ok"], false); + assert!(r["error"]["message"].as_str().unwrap().contains("nope"), "{r}"); + assert_eq!(s.script_ui.windows.len(), 1); + // Resource strings aren't supported (a clear error). + let o = run_code(&mut s, "new Window(\"dialog { text: 'x' }\")", "res.jsx"); + assert!(o.error.unwrap().message.contains("resource")); + s.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn console_windows_run_inline() { + let mut s = session(); + let req = effectcraft_engine::ScriptRequest { + code: "var cw = new Window('palette', 'Console UI'); var cb = cw.add('button', undefined, 'Hi'); var hits = 0; cb.onClick = function () { hits++; writeLn('hit ' + hits); }; cw.show(); 1", + name: "Script Console", + console: true, + }; + let o = crate::run(&mut s, &req); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(s.script_ui.windows.len(), 1); + let r = s.execute("scriptui.click", json!({"widget": "Hi"})).unwrap(); + assert_eq!(r["output"], "hit 1"); + // The console context sees the handler's state. + let o = crate::run(&mut s, &effectcraft_engine::ScriptRequest { code: "hits", name: "Script Console", console: true }); + assert_eq!(o.result, json!(1)); + // Dialogs don't block in the console. + let o = crate::run( + &mut s, + &effectcraft_engine::ScriptRequest { code: "var dd = new Window('dialog', 'D'); dd.show(); 'went on'", name: "Script Console", console: true }, + ); + assert_eq!(o.result, json!("went on")); + assert_eq!(s.script_ui.windows.len(), 2); + s.execute("scriptui.close", json!({"window": "D"})).unwrap(); + s.execute("scriptui.close", json!({"window": "Console UI"})).unwrap(); + assert!(s.script_ui.windows.is_empty()); +} + +fn solid(s: &mut Session, name: &str, at: f64) -> u64 { + let id = s.execute("layer.newSolid", json!({"name": name, "color": "#808080", "width": 20, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.timing", json!({"layers": [id], "in": at})).unwrap(); + id +} + +#[test] +fn sample_scripts_and_installing() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 400, "height": 200, "duration": 10})).unwrap(); + for (n, at) in [("C", 2.0), ("A", 0.0), ("B", 1.0)] { + solid(&mut s, n, at); + } + // File ▸ Scripts lists the samples. + let list = s.execute("file.scripts.list", json!({})).unwrap(); + let names: Vec<&str> = list.as_array().unwrap().iter().map(|e| e["name"].as_str().unwrap()).collect(); + for want in ["Rename Layers.jsx", "Sort Layers by In Point.jsx", "Create Null at Selected Layers.jsx", "Render Queue Batch.jsx", "Layer Tools.jsx"] { + assert!(names.contains(&want), "{names:?}"); + } + // Sort Layers by In Point (all layers when none are selected). + s.execute("edit.deselectAll", json!({})).unwrap(); + s.execute("file.runScript", json!({"name": "Sort Layers by In Point.jsx"})).unwrap(); + assert_eq!(layer_names(&s), ["A", "B", "C"]); + assert!(s.undo()); + assert_eq!(layer_names(&s), ["B", "A", "C"]); + // Create Null at Selected Layers. + let ids: Vec = s.active_comp().unwrap().layers.iter().take(2).map(|l| l.id.0).collect(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("file.runScript", json!({"name": "Create Null at Selected Layers.jsx"})).unwrap(); + let c = s.active_comp().unwrap(); + let null = c.layers.iter().find(|l| l.name == "Centroid").unwrap(); + assert!(c.layers.iter().filter(|l| ids.contains(&l.id.0)).all(|l| l.parent == Some(null.id))); + // Rename Layers: a dialog; fill it in and press OK. + s.execute("layer.select", json!({"layers": ids})).unwrap(); + let r = s.execute("file.runScript", json!({"name": "Rename Layers.jsx"})).unwrap(); + assert_eq!(r["waiting"], true); + s.execute("scriptui.set", json!({"widget": "#11", "value": "Shot_"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Number the layers"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + let names = layer_names(&s); + assert!(names.contains(&"Shot_B 1".to_string()) && names.contains(&"Shot_A 2".to_string()), "{names:?}"); + // Install a script and a ScriptUI panel (they live in the settings store). + let dir = std::env::temp_dir().join(format!("ec-scripts-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("Hello.jsx"); + std::fs::write(&file, "writeLn('hello from ' + $.fileName); 42").unwrap(); + s.execute("file.installScript", json!({"path": file.to_string_lossy()})).unwrap(); + let panel = dir.join("Mini Panel.jsx"); + std::fs::write(&panel, "var p = this; p.add('button', undefined, 'Go').onClick = function () { writeLn('go'); }; p.layout.layout(true);").unwrap(); + s.execute("file.installScriptUIPanel", json!({"path": panel.to_string_lossy()})).unwrap(); + let list = s.execute("file.scripts.list", json!({})).unwrap(); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Hello.jsx" && e["source"] == "installed" && e["panel"] == false)); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Mini Panel.jsx" && e["panel"] == true)); + let r = s.execute("file.runScript", json!({"name": "Hello.jsx"})).unwrap(); + assert_eq!(r["result"], 42); + // Window ▸ Mini Panel.jsx: a dockable panel; `this` is the panel. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + let wid = r["window"].as_u64().unwrap(); + let w = s.script_ui.window(wid as u32).unwrap().clone(); + assert_eq!((w.kind, w.title.as_str()), (WindowKind::Panel, "Mini Panel")); + let r = s.execute("scriptui.click", json!({"window": wid, "widget": "Go"})).unwrap(); + assert_eq!(r["output"], "go"); + // Opening it again brings the same panel forward. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + assert_eq!(r["window"], wid); + // The sample panel docks too, and works. + let r = s.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + let tools = r["window"].as_u64().unwrap(); + let before = s.active_comp().unwrap().layers.len(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("scriptui.click", json!({"window": tools, "widget": "Null at Centroid"})).unwrap(); + assert_eq!(s.active_comp().unwrap().layers.len(), before + 1); + s.execute("scriptui.close", json!({"window": tools})).unwrap(); + s.execute("scriptui.close", json!({"window": wid})).unwrap(); + assert!(s.execute("file.installScript", json!({"path": "/nope/x.txt"})).is_err()); + s.execute("file.uninstallScript", json!({"name": "Hello.jsx"})).unwrap(); + assert!(s.execute("file.runScript", json!({"name": "Hello.jsx"})).is_err()); + let _ = std::fs::remove_dir_all(&dir); + let _: Value = json!(null); +} diff --git a/crates/script/src/ui.rs b/crates/script/src/ui.rs new file mode 100644 index 00000000..66574ce0 --- /dev/null +++ b/crates/script/src/ui.rs @@ -0,0 +1,339 @@ +//! ScriptUI hosts: who owns a script window's JavaScript objects, and how user actions reach +//! its handlers. +//! +//! * **Script files** (native builds) run on a *script-host thread*. The caller's session moves +//! to that thread while JavaScript runs and comes back whenever the script stops: at the end, +//! or while a modal dialog waits in `show()`. Each [`ScriptUiEvent`] moves the session there +//! again, runs the handler, and brings it back with the windows' new state. The host ends +//! when its last window closes. Only one side holds the session at a time, so nothing runs +//! concurrently and every edit is still an ordinary undoable engine command. +//! * **The Script Console** (and every run on the web) runs inline: its windows' handlers run in +//! the console's (or a kept) context on the caller's thread, and dialogs don't block. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::mem::ManuallyDrop; +use std::sync::atomic::{AtomicU32, Ordering}; + +use boa_engine::{Context, JsResult, JsValue, js_string}; +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{ScriptUiEvent, ScriptWindow, layout}; +use serde_json::{Value as J, json}; + +use crate::runtime::{ACTIVE, Active, CONSOLE, Restore, ScriptError, arg_string, js_str, outcome, script_error, settle, throw, with_active}; + +pub(crate) const SCRIPTUI: &str = include_str!("scriptui.js"); + +/// The Script Console's host id. +pub(crate) const CONSOLE_HOST: u32 = u32::MAX; + +static NEXT_WINDOW: AtomicU32 = AtomicU32::new(1); +static NEXT_HOST: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn new_host_id() -> u32 { + NEXT_HOST.fetch_add(1, Ordering::Relaxed) +} + +thread_local! { + /// Contexts of inline runs whose palettes / panels are still open (dropped when they close). + static INLINE: RefCell>> = const { RefCell::new(BTreeMap::new()) }; +} + +pub(crate) fn keep_inline(host: u32, ctx: Context) { + INLINE.with(|m| m.borrow_mut().insert(host, ManuallyDrop::new(ctx))); +} + +fn drop_inline(host: u32) { + if let Some(ctx) = INLINE.with(|m| m.borrow_mut().remove(&host)) { + drop(ManuallyDrop::into_inner(ctx)); + } +} + +// ---------------------------------------------------------------- JavaScript side + +fn call_global(ctx: &mut Context, name: &str, args: &[JsValue]) -> JsResult { + let f = ctx.global_object().get(js_string!(name), ctx)?; + let f = f.as_object().ok_or_else(|| throw(format!("{name} is not defined")))?; + f.call(&JsValue::undefined(), args, ctx) +} + +/// The open windows of a context (closed ones are reported once, invisible, then forgotten). +pub(crate) fn snapshot(ctx: &mut Context) -> Vec { + let Ok(v) = call_global(ctx, "__uiSnapshot", &[]) else { return vec![] }; + let s = v.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + serde_json::from_str(&s).unwrap_or_default() +} + +/// Run a window event's handlers in `ctx`. +fn dispatch_js(ctx: &mut Context, ev: &ScriptUiEvent) -> Result { + let args = [JsValue::from(ev.window), JsValue::from(ev.widget), js_str(&ev.kind), js_str(&ev.value.to_string())]; + match call_global(ctx, "__uiDispatch", &args) { + Ok(_) => Ok(J::Null), + Err(e) => { + let mut err = script_error(&e, ctx, "ScriptUI handler", ""); + err.message = format!("{} handler: {}", ev.kind, err.message); + Err(err) + } + } +} + +/// `__uiNewId()`: a window id unique in the process. +pub(crate) fn native_new_id(_: &JsValue, _: &[JsValue], _: &mut Context) -> JsResult { + Ok(JsValue::from(NEXT_WINDOW.fetch_add(1, Ordering::Relaxed))) +} + +/// `__uiLayoutNative(windowJson)` → `{controlId: [x, y, width, height]}`: ScriptUI's automatic +/// layout ([`effectcraft_engine::scriptui::layout`]) for `layout.layout()`. +pub(crate) fn native_layout(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let s = arg_string(args, 0, ctx)?; + let mut w: ScriptWindow = serde_json::from_str(&s).map_err(|e| throw(format!("layout: {e}")))?; + layout(&mut w); + let mut all = vec![]; + w.root.walk(&mut all); + let map: serde_json::Map = all.iter().map(|c| (c.id.to_string(), json!(c.bounds))).collect(); + Ok(js_str(&J::Object(map).to_string())) +} + +/// `__uiModal(windowId)`: a dialog's `show()`. On a script-host thread it hands the session back +/// to the caller and waits for the dialog's events until it closes, returning its result. Inline +/// (Script Console, web) it returns `null` at once: the dialog stays open like a palette. +pub(crate) fn native_modal(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let id = args.first().cloned().unwrap_or_default().to_number(ctx)? as u32; + let linked = with_active(|a| Ok(a.link.is_some())).unwrap_or(false); + if !linked { + return Ok(JsValue::null()); + } + loop { + let r = call_global(ctx, "__uiModalResult", &[JsValue::from(id)])?; + if !r.is_null() && !r.is_undefined() { + return Ok(r); + } + let windows = snapshot(ctx); + let out = crate::Outcome { output: settle(), waiting: true, ..Default::default() }; + let Some(ev) = yield_wait(out, windows, false) else { + // The caller went away: cancel the dialog. + return Ok(JsValue::from(2)); + }; + if let Err(e) = dispatch_js(ctx, &ev) { + let line = e.line.map(|l| format!(" (line {l})")).unwrap_or_default(); + let _ = with_active(|a| { + a.output.push(format!("Error{line}: {}", e.message)); + Ok(()) + }); + } + } +} + +// ---------------------------------------------------------------- script-host threads + +pub(crate) enum ToHost { + Start(Session), + Event(Session, ScriptUiEvent), +} + +pub(crate) struct FromHost { + session: Session, + outcome: crate::Outcome, + windows: Vec, + name: String, + /// The host thread has ended (no windows left). + done: bool, +} + +/// The host thread's end of the hand-off. +pub(crate) struct Link { + rx: std::sync::mpsc::Receiver, + tx: std::sync::mpsc::Sender, +} + +/// The caller's end. +struct HostEnd { + tx: std::sync::mpsc::Sender, + rx: std::sync::mpsc::Receiver, +} + +fn hosts() -> &'static std::sync::Mutex> { + static H: std::sync::OnceLock>> = std::sync::OnceLock::new(); + H.get_or_init(Default::default) +} + +// The session crosses to script-host threads and back. +const _: fn() = || { + fn send() {} + send::(); +}; + +/// On the host thread: hand the session (with the windows' state and what the script printed) +/// back to the caller, and wait for the next event. `None` when the caller has gone. +fn yield_wait(outcome: crate::Outcome, windows: Vec, done: bool) -> Option { + let (session, link, name) = ACTIVE.with(|a| { + let mut a = a.borrow_mut(); + let a = a.as_mut()?; + Some((std::mem::take(&mut a.session), a.link.take()?, a.name.clone())) + })?; + if link.tx.send(FromHost { session, outcome, windows, name, done }).is_err() { + return None; + } + if done { + return None; + } + match link.rx.recv() { + Ok(ToHost::Event(s, ev)) => { + ACTIVE.with(|a| { + if let Some(a) = a.borrow_mut().as_mut() { + a.session = s; + a.link = Some(link); + } + }); + Some(ev) + } + _ => None, + } +} + +fn host_main(name: String, code: String, link: Link) { + let Ok(ToHost::Start(session)) = link.rx.recv() else { return }; + let mut a = Active::new(session, &name); + a.link = Some(link); + ACTIVE.with(|c| *c.borrow_mut() = Some(a)); + let file = if name.is_empty() { "script".to_string() } else { name.clone() }; + let body = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut ctx = match crate::runtime::new_context() { + Ok(c) => c, + Err(e) => { + yield_wait(outcome(settle(), Err(ScriptError { message: e, file: file.clone(), ..Default::default() })), vec![], true); + return; + } + }; + let r = crate::runtime::eval_with_tasks(&mut ctx, &code, &file); + let mut out = outcome(settle(), r); + loop { + let windows = snapshot(&mut ctx); + let open = windows.iter().any(|w| w.visible); + match yield_wait(out, windows, !open) { + Some(ev) => { + let r = dispatch_js(&mut ctx, &ev); + out = outcome(settle(), r); + } + None => return, + } + } + })); + // A panic: give the session back all the same. + let left = ACTIVE.with(|c| c.borrow_mut().take()); + if let Some(mut a) = left + && let Some(link) = a.link.take() + { + let message = if body.is_err() { "the script engine stopped unexpectedly" } else { "the script ended" }; + let out = crate::Outcome { error: Some(ScriptError { message: message.into(), file, ..Default::default() }), ..Default::default() }; + let _ = link.tx.send(FromHost { session: std::mem::take(&mut a.session), outcome: out, windows: vec![], name, done: true }); + } +} + +/// Wait for the host to hand the session back; keep its link while it has windows open. +fn receive(session: &mut Session, host: u32, end: HostEnd) -> crate::Outcome { + match end.rx.recv() { + Ok(FromHost { session: s, outcome, windows, name, done }) => { + *session = s; + publish(session, host, &name, windows); + if done { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + } else if let Ok(mut h) = hosts().lock() { + h.insert(host, end); + } + outcome + } + Err(_) => crate::Outcome { error: Some(ScriptError { message: "the script host stopped".into(), ..Default::default() }), ..Default::default() }, + } +} + +/// Run a script file on a new script-host thread (see the module docs). +pub(crate) fn run_threaded(session: &mut Session, req: &effectcraft_engine::ScriptRequest) -> crate::Outcome { + let host = new_host_id(); + let (to_tx, to_rx) = std::sync::mpsc::channel(); + let (from_tx, from_rx) = std::sync::mpsc::channel(); + let (name, code) = (req.name.to_string(), req.code.to_string()); + let spawned = + std::thread::Builder::new().name(format!("script-{host}")).stack_size(64 << 20).spawn(move || host_main(name, code, Link { rx: to_rx, tx: from_tx })); + if spawned.is_err() { + return crate::runtime::run_inline(session, req); + } + if let Err(std::sync::mpsc::SendError(ToHost::Start(s) | ToHost::Event(s, _))) = to_tx.send(ToHost::Start(std::mem::take(session))) { + *session = s; + return crate::Outcome { error: Some(ScriptError { message: "the script host did not start".into(), ..Default::default() }), ..Default::default() }; + } + receive(session, host, HostEnd { tx: to_tx, rx: from_rx }) +} + +/// Record the windows a host published (named after the script that made them). +pub(crate) fn publish(session: &mut Session, host: u32, name: &str, mut windows: Vec) { + for w in &mut windows { + if w.script.is_empty() { + w.script = name.to_string(); + } + } + let had = session.script_ui.windows.iter().any(|w| w.host == host); + if windows.is_empty() && !had { + return; + } + session.script_ui.publish(host, windows); + if host != CONSOLE_HOST && !session.script_ui.windows.iter().any(|w| w.host == host) { + drop_inline(host); + } +} + +/// The [`effectcraft_engine::scriptui::ScriptUiDispatch`] this crate provides: run a window +/// event's handlers in the script that made the window, then publish its windows' new state. +/// Returns the handler run's `{ok, output, error, waiting?}`. +pub fn dispatch_ui(session: &mut Session, ev: &ScriptUiEvent) -> Result { + if ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)) { + return Err("scripts can't drive script windows".into()); + } + let w = session.script_ui.window(ev.window).ok_or_else(|| format!("no script window {}", ev.window))?; + let (host, name) = (w.host, w.script.clone()); + let inline = host == CONSOLE_HOST || INLINE.with(|m| m.borrow().contains_key(&host)); + if inline { + return Ok(dispatch_inline(session, host, &name, ev).to_json()); + } + let end = hosts().lock().ok().and_then(|mut h| h.remove(&host)); + let Some(end) = end else { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + }; + if let Err(std::sync::mpsc::SendError(ToHost::Event(s, _) | ToHost::Start(s))) = end.tx.send(ToHost::Event(std::mem::take(session), ev.clone())) { + *session = s; + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + } + Ok(receive(session, host, end).to_json()) +} + +fn dispatch_inline(session: &mut Session, host: u32, name: &str, ev: &ScriptUiEvent) -> crate::Outcome { + let s = std::mem::take(session); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, name))); + let guard = Restore { target: session }; + let (r, windows) = if host == CONSOLE_HOST { + CONSOLE.with(|c| match c.borrow_mut().as_mut() { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + } else { + INLINE.with(|m| match m.borrow_mut().get_mut(&host) { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + }; + let output = settle(); + drop(guard); + publish(session, host, name, windows); + outcome(output, r) +} diff --git a/crates/ui-egui/src/header.rs b/crates/ui-egui/src/header.rs index b2fa142e..b3f5908f 100644 --- a/crates/ui-egui/src/header.rs +++ b/crates/ui-egui/src/header.rs @@ -221,7 +221,21 @@ fn puppet_options(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter } x = r.max.x + 12.0; } - x + 4.0 + // Record Options… (⌘/Ctrl-drag a pin records its motion in real time). + let label = "Record Options..."; + let w = 104.0; + let r = Rect::from_min_size(pos2(x, cy - 10.0), vec2(w, 20.0)); + let resp = ui.interact(r, egui::Id::new("puppet-record-options"), egui::Sense::click()).on_hover_text("⌘/Ctrl-drag a pin to record its motion"); + p.rect_stroke(r, 3.0, egui::Stroke::new(1.0, if resp.hovered() { t.accent } else { t.field_border }), egui::StrokeKind::Inside); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.5), t.text); + app.auto.add("header.puppet.recordOptions", r, label); + if resp.clicked() { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "puppet.recordOptions", json!({})) { + app.ui.status = e; + } + } + r.max.x + 16.0 } fn ws_menu(app: &mut EffectcraftApp, ui: &mut egui::Ui) { diff --git a/crates/ui-egui/src/panels/effects_presets.rs b/crates/ui-egui/src/panels/effects_presets.rs index 888f9d86..21d63dec 100644 --- a/crates/ui-egui/src/panels/effects_presets.rs +++ b/crates/ui-egui/src/panels/effects_presets.rs @@ -322,7 +322,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let searching = !query.is_empty(); let mut y = list.min.y - scroll; let mut apply: Option = None; - let reg = effectcraft_engine::effects::registry(); + let reg = effectcraft_engine::effects::all(); let keep = |e: &effectcraft_engine::effects::EffectSpec| depth_keeps(&view.depth, e) && (query.is_empty() || e.name.to_lowercase().contains(&query)); let lookup = |id: &str| reg.iter().find(|e| e.id == id); @@ -381,7 +381,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effect_row(app, ui, &lp, list, &mut y, e, -16.0, &query, "effects.item", &mut apply); } } else { - let mut cats: Vec<&str> = effectcraft_engine::effects::CATEGORIES.to_vec(); + let mut cats: Vec<&str> = effectcraft_engine::effects::categories(); cats.retain(|c| reg.iter().any(|e| e.category == *c)); for cat in cats { let items: Vec<_> = reg.iter().filter(|e| e.category == cat && keep(e)).collect(); diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 39640ef6..50f43f73 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -136,6 +136,19 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { 1, )], ), + // Puppet tool ▸ Record Options. + "puppet.recordOptions" if !has(p, &["speed", "smoothing", "useDraftDeformation", "showMesh"]) => { + let o = &s.state.puppet; + ( + "Puppet Record Options".into(), + vec![ + Field::num("speed", "Speed (%)", o.record_speed), + Field::num("smoothing", "Smoothing", o.record_smoothing), + Field::bool("useDraftDeformation", "Use Draft Deformation", o.record_draft), + Field::bool("showMesh", "Show Mesh", o.record_show_mesh), + ], + ) + } "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 69d95f66..ea12591c 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -172,34 +172,43 @@ pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { pub fn history(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let t = app.tokens; let p = ui.painter().with_clip_rect(rect); + // The branching history (EffectCraft's History panel): every state, branches indented under + // the state they grew from; click any state to go there (`edit.history.goto`). + let nodes = app.session.history_tree(); + let row_h = 22.0; + let cur = nodes.iter().position(|n| n.current).unwrap_or(0); + // Keep the current state in view. + let visible = ((rect.height() - 12.0) / row_h).max(1.0) as usize; + let first = (cur + 2).saturating_sub(visible).min(nodes.len().saturating_sub(visible)); let mut y = rect.min.y + 6.0; - let undo: Vec = app.session.history.undo.iter().map(|u| u.0.clone()).collect(); - let redo: Vec = app.session.history.redo.iter().rev().map(|u| u.0.clone()).collect(); - let n = undo.len(); - let mut jump: Option = None; - for (i, label) in undo.iter().chain(redo.iter()).enumerate() { - let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), 22.0)); - y += 22.0; - let current = i + 1 == n; - let future = i >= n; - let resp = ui.interact(r, egui::Id::new(("hist", i)), Sense::click()); - if current { + let mut jump: Option = None; + for n in nodes.iter().skip(first).take(visible + 1) { + let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), row_h)); + y += row_h; + let resp = ui.interact(r, egui::Id::new(("hist", n.id.as_str())), Sense::click()); + if n.current { p.rect_filled(r, 0.0, t.row_selected); } else if resp.hovered() { p.rect_filled(r, 0.0, t.hover); } - p.text(pos2(r.min.x + 12.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(12.0), if future { t.text_faint } else { t.text }); + let x = r.min.x + 12.0 + n.depth as f32 * 14.0; + if n.depth > 0 { + // Branch marker. + let c = pos2(x - 7.0, r.center().y); + p.line_segment([pos2(c.x, r.min.y), c], egui::Stroke::new(1.0, t.text_faint)); + p.line_segment([c, pos2(c.x + 5.0, c.y)], egui::Stroke::new(1.0, t.text_faint)); + } + let col = if !n.current && (n.future || !n.line) { t.text_faint } else { t.text }; + p.text(pos2(x, r.center().y), Align2::LEFT_CENTER, &n.label, Tokens::ui(12.0), col); + app.auto.add(&format!("history.state.{}", n.index), r, &n.label); if resp.clicked() { - jump = Some(i as i64 + 1 - n as i64); + jump = Some(n.index); } } - if let Some(d) = jump { - for _ in 0..d.unsigned_abs() { - if d < 0 { - app.session.undo(); - } else { - app.session.redo(); - } + if let Some(i) = jump { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "edit.history.goto", serde_json::json!({"index": i})) { + app.ui.status = e; } } } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 04999a90..54643bd4 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -121,6 +121,17 @@ enum Gesture { pin: u64, inv: Mat3, }, + /// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds + /// since it began, layer space) and becomes keyframes on release (`puppet.recordPin`); the + /// current time runs along while recording. + PuppetRecord { + layer: LayerId, + pin: u64, + inv: Mat3, + began: f64, + cti: Tick, + samples: Vec<(f64, [f64; 2])>, + }, /// Pen tool: pulling the tangents of the vertex just placed. Pen { layer: LayerId, @@ -861,7 +872,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { t if t.is_shape() => Some(Gesture::Create { tool: t, start: cpt }), t if t.puppet_kind().is_some() => pin_hits.iter().rev().find(|h| h.pos.distance(press) < 8.0).and_then(|h| { let l = comp.layer(h.layer)?; - Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv: l2c(&ectx, l).0.inverse()? }) + let inv = l2c(&ectx, l).0.inverse()?; + if mods.command { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let began = ui.input(|i| i.time); + return Some(Gesture::PuppetRecord { layer: h.layer, pin: h.pin, inv, began, cti: time, samples: vec![(0.0, [lp.x, lp.y])] }); + } + Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv }) }), Tool::Orbit | Tool::PanCamera | Tool::Dolly => Some(Gesture::Camera { tool }), Tool::Selection if super::tracker::hit_at(&track_hits, press).is_some() => { @@ -1097,6 +1114,20 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let _ = app.session.execute("mask.setVertex", params); } + Gesture::PuppetRecord { layer, pin, inv, began, cti, mut samples } => { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let now = ui.input(|i| i.time) - began; + if samples.last().is_none_or(|s| now > s.0) { + samples.push((now, [lp.x, lp.y])); + } + // The current time runs with the recording (Record Options ▸ Speed). + let speed = app.session.state.puppet.record_speed.max(1.0) / 100.0; + app.session.set_time(cti + Tick::from_seconds_f64(now / speed)); + painter.circle_filled(pos, 6.0, Color32::from_rgb(0xe0, 0x30, 0x30)); + painter.text(pos + vec2(10.0, -10.0), Align2::LEFT_BOTTOM, "Recording", Tokens::ui(11.0), Color32::from_rgb(0xff, 0x60, 0x60)); + ui.data_mut(|dd| dd.insert_temp(gid, Gesture::PuppetRecord { layer, pin, inv, began, cti, samples })); + ui.ctx().request_repaint(); + } Gesture::PuppetPin { layer, pin, inv } => { let lp = inv.apply(gv2(cpt[0], cpt[1])); if let Err(e) = app.session.execute("puppet.movePin", json!({"layer": layer.0, "pin": pin, "position": [lp.x, lp.y], "merge": merge})) { @@ -1129,6 +1160,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if resp.drag_stopped() { if let (Some(g), Some(pos)) = (gesture, resp.interact_pointer_pos()) { match g { + Gesture::PuppetRecord { layer, pin, cti, samples, .. } => { + let pts: Vec = samples.iter().map(|(t, p)| json!([t, p[0], p[1]])).collect(); + if let Err(e) = app.session.execute("puppet.recordPin", json!({"layer": layer.0, "pin": pin, "samples": pts, "start": cti.seconds()})) { + app.ui.status = e.to_string(); + } + } Gesture::Create { tool, start } => { let end = map.to_comp(pos); create_shape(app, tool, start, end, mods.shift); diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..6ff400dc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,9 +34,10 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | -| L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | -| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) whose edits run engine commands; AE match names | -| L4 | `host` | A fully wired `Session` (media, expressions, scripting, exporter) for the frontends | +| L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | +| L4 | `engine` | `Session`: project, branching undo history, editor state, the command registry and menus, the ScriptUI window model (`scriptui`) | +| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) and ScriptUI (dialogs, palettes, dockable panels; script-host threads so dialogs block `show()`), whose edits run engine commands; AE match names | +| L4 | `host` | A fully wired `Session` (media, expressions, scripting, WebAssembly plug-ins, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 21928b90..dfedbea3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -43,6 +43,7 @@ const LAYERS: &[(&str, u8)] = &[ ("gpu", 3), ("lottie", 3), ("format", 3), + ("plugin", 3), ("engine", 4), ("script", 4), ("host", 4), From 0bce3f2e4938eec54ef0036ef5c8c3a36c542649 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:08:23 -0400 Subject: [PATCH 22/36] M13.3: Subspace Warp content-preserving mesh warp, face tracker library Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/warp_stab.rs | 40 +- crates/track/src/camtrack/features.rs | 13 +- crates/track/src/face.rs | 1142 +++++++++++++++++++++++++ crates/track/src/lib.rs | 2 + crates/track/src/solve.rs | 6 + crates/track/src/stabilize.rs | 358 +++++++- crates/track/src/subspace.rs | 447 ++++++++++ 7 files changed, 1971 insertions(+), 37 deletions(-) create mode 100644 crates/track/src/face.rs create mode 100644 crates/track/src/subspace.rs diff --git a/crates/effects/src/warp_stab.rs b/crates/effects/src/warp_stab.rs index c0bbced8..ff535c41 100644 --- a/crates/effects/src/warp_stab.rs +++ b/crates/effects/src/warp_stab.rs @@ -17,8 +17,11 @@ //! Edge Cropping ignores bad edge pixels of the input frames; //! - **Show Track Points** draws the analysed background features. //! -//! *Subspace Warp* is approximated by a per-frame perspective warp (no content-preserving -//! mesh warp yet), and *Rolling Shutter Ripple* is stored but has no effect. +//! *Subspace Warp* warps each frame with a content-preserving mesh fitted to the analysis' +//! subspace-smoothed feature trajectories ([`effectcraft_track::subspace`]); analyses made +//! before trajectories were stored fall back to a perspective warp. *Rolling Shutter Ripple* +//! picks the mesh: Automatic Reduction (12 × 8, stiffer) or Enhanced Reduction (twice the rows, +//! softer), which follows the row-wise wobble of rolling-shutter shake more closely. //! //! Without an analysis (or with one made for another layer size) the input passes through. @@ -30,7 +33,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::Homography; -use effectcraft_track::stabilize::{Framing, Method, Plan, StabResult, StabSettings, WarpAnalysis, plan}; +use effectcraft_track::stabilize::{Framing, Method, Plan, Ripple, StabResult, StabSettings, WarpAnalysis, plan}; use rayon::prelude::*; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -145,6 +148,7 @@ pub fn settings(params: &Params, fps: f64) -> StabSettings { // AE's 0…100 slider (50 = balanced) onto the planner's −100…100. crop_less_smooth_more: (f(params, "advanced/cropLessSmoothMore") - 50.0) * 2.0, fps, + ripple: if e(params, "advanced/rollingShutterRipple") == 1 { Ripple::Enhanced } else { Ripple::Automatic }, } } @@ -175,7 +179,7 @@ fn settings_key(s: &StabSettings) -> u64 { for v in [s.smoothness, s.max_scale, s.action_safe, s.additional_scale, s.crop_less_smooth_more, s.fps] { h.write_u64(v.to_bits()); } - h.write(format!("{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing).as_bytes()); + h.write(format!("{:?}{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing, s.ripple).as_bytes()); h.finish() } @@ -275,6 +279,13 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { } let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; let Some(inv) = pl.warps[k].inverse() else { return buf }; + // Output point → source point of this frame (a mesh lookup for Subspace Warp). + let mesh = pl.meshes.get(k); + let finv = pl.framing.inverse().unwrap_or(Homography::IDENTITY); + let src_of = |p: [f64; 2]| match mesh { + Some(m) => m.sample(finv.apply(p)), + None => inv.apply(p), + }; let (ow, oh) = (((size[0] * sc).ceil() as u32).max(1), ((size[1] * sc).ceil() as u32).max(1)); let synth = matches!(settings(ctx.params, 30.0).framing, Framing::SynthesizeEdges); let crop = [ @@ -306,7 +317,7 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { { continue; } - let q = inv.apply(p); + let q = src_of(p); if !synth { *px = sample(src, q); continue; @@ -337,12 +348,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { }); let need = need_m.into_inner().unwrap_or_else(|e| e.into_inner()); if synth && need.iter().any(|w| *w > 0.0) { - synthesize(ctx, &a, k, &inv, &crop, sc, &mut out, &need); + synthesize(ctx, &a, k, &src_of, &crop, sc, &mut out, &need); } if b(ctx.params, "advanced/showTrackPoints") { let r = (2.5 * f(ctx.params, "advanced/trackPointSize") / 100.0 * sc).max(0.5); for q in &a.frames[k].points { - let o = pl.warps[k].apply([q[0] as f64, q[1] as f64]); + let Some(o) = pl.output_of(k, [q[0] as f64, q[1] as f64]) else { continue }; dot(&mut out, [o[0] * sc, o[1] * sc], r, [0.25, 0.75, 1.0, 1.0]); } } @@ -352,7 +363,16 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { /// Fill the pixels flagged in `need` (weight = how much synthesized content they take) from /// neighbouring frames, nearest first. #[allow(clippy::too_many_arguments)] -fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, crop: &[f64; 4], sc: f64, out: &mut Image, need: &[f32]) { +fn synthesize( + ctx: &EffectCtx, + a: &WarpAnalysis, + k: usize, + src_of: &(dyn Fn([f64; 2]) -> [f64; 2] + Sync), + crop: &[f64; 4], + sc: f64, + out: &mut Image, + need: &[f32], +) { let Some(host) = ctx.env.host else { return }; let n = a.frames.len(); let range = ((f(ctx.params, "advanced/synthesizeInputRange").max(0.0) / a.frame_duration.max(1e-6)).round() as usize).min(60); @@ -376,7 +396,7 @@ fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, cro .filter(|i| !filled[**i]) .filter_map(|i| { let (x, y) = (i % ow, i / ow); - let q = inv.apply([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); + let q = src_of([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); let qj = m.apply(q); inside(crop, qj).then(|| (*i, nb.img.sample_bilinear_clamped(qj[0] * nb.scale + nb.offset[0], qj[1] * nb.scale + nb.offset[1]))) }) @@ -412,7 +432,7 @@ mod tests { use effectcraft_track::stabilize::FrameMotion; fn analysis_json(n: usize, w: f64, h: f64, shift: impl Fn(usize) -> [f64; 2]) -> String { - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![], tracks: vec![] }; for k in 0..n { let mut fm = FrameMotion::default(); if k > 0 { diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 30b019e5..1775a304 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -28,11 +28,19 @@ pub struct TrackAnalyzer { frame: u32, size: [f64; 2], factor: f64, + /// (analysis long side, wanted features) overriding the options' defaults. + budget: Option<(u32, usize)>, } impl TrackAnalyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { - TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0, budget: None } + } + + /// An analyzer working at most at `max_side` pixels with about `want` live features (the + /// Warp Stabilizer's trajectories for Subspace Warp). + pub fn with_budget(size: [f64; 2], max_side: u32, want: usize) -> TrackAnalyzer { + TrackAnalyzer { budget: Some((max_side, want.max(8))), ..TrackAnalyzer::new(size, AnalyzeOpts::default()) } } pub fn frames(&self) -> u32 { @@ -45,6 +53,9 @@ impl TrackAnalyzer { } fn params(&self) -> (u32, usize) { + if let Some(b) = self.budget { + return b; + } if self.opts.detailed { (1280, 900) } else { (720, 500) } } diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs new file mode 100644 index 00000000..78344a14 --- /dev/null +++ b/crates/track/src/face.rs @@ -0,0 +1,1142 @@ +//! Face tracking (Tracker panel ▸ Method ▸ Face Tracking (Outline Only) / (Detailed Features)), +//! a classical pipeline with no learned weights: +//! +//! 1. **Skin model**: on the first frame, the colour of the middle of the mask (luma and +//! normalised r, g chromaticity: median and median absolute deviation) defines skin; every +//! later frame is classified against it inside a search window around the previous face. +//! 2. **Face region and outline**: the skin component containing the face centre is flood-filled; +//! its outer boundary is found along 48 rays from its centroid (so the non-skin holes of eyes, +//! brows, nostrils and mouth don't matter), median-smoothed, and an ellipse is fitted to the +//! outline polygon from its area moments (centre, half-height, roll). The outline is what +//! *Outline Only* keys into the mask. +//! 3. **Facial features**: the non-skin connected components inside the face are matched to the +//! eyes, brows, nostrils and mouth by their position in the face frame (predicted from the +//! previous frame); landmark points are measured on each component (extremes along the face +//! axes for corners and lids, a darkness-weighted centroid for the pupil). Chin and jaw points +//! come from the outline. +//! 4. **Shape model**: an active-shape point distribution model (Cootes, Taylor, Cooper & Graham, +//! "Active Shape Models — Their Training and Application", CVIU 61(1), 1995) regularises the +//! landmarks: a weighted similarity alignment and shape-parameter fit (parameters clamped to +//! ±3 standard deviations) fills landmarks whose feature was not found (a closed eye, a +//! merged component) and replaces detections that disagree with the model. The model is +//! *trained on synthetic shapes* produced by our own parametric face generator +//! ([`synth_shape`]: eye and mouth openness, mouth width, brow raise, head yaw and pitch) by +//! principal component analysis — no external data or weights. +//! +//! Coordinates are layer pixels; the face frame is centred on the face ellipse, with the +//! half-height as unit, `u` right and `v` down (image-left is "Left"). + +use std::sync::OnceLock; + +use effectcraft_raster::Image; + +use crate::Frame; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; + +/// Landmarks: (id, display name, canonical position in the face frame). +pub const LANDMARKS: [(&str, &str, [f64; 2]); 26] = [ + ("leftEyebrowInner", "Left Eyebrow Inner", [-0.16, -0.44]), + ("leftEyebrowMiddle", "Left Eyebrow Middle", [-0.32, -0.50]), + ("leftEyebrowOuter", "Left Eyebrow Outer", [-0.48, -0.44]), + ("rightEyebrowInner", "Right Eyebrow Inner", [0.16, -0.44]), + ("rightEyebrowMiddle", "Right Eyebrow Middle", [0.32, -0.50]), + ("rightEyebrowOuter", "Right Eyebrow Outer", [0.48, -0.44]), + ("leftEyeInner", "Left Eye Inner", [-0.17, -0.22]), + ("leftEyeOuter", "Left Eye Outer", [-0.45, -0.22]), + ("leftEyeTop", "Left Eye Top", [-0.31, -0.292]), + ("leftEyeBottom", "Left Eye Bottom", [-0.31, -0.148]), + ("leftPupil", "Left Pupil", [-0.31, -0.22]), + ("rightEyeInner", "Right Eye Inner", [0.17, -0.22]), + ("rightEyeOuter", "Right Eye Outer", [0.45, -0.22]), + ("rightEyeTop", "Right Eye Top", [0.31, -0.292]), + ("rightEyeBottom", "Right Eye Bottom", [0.31, -0.148]), + ("rightPupil", "Right Pupil", [0.31, -0.22]), + ("noseTip", "Nose Tip", [0.0, 0.14]), + ("leftNostril", "Left Nostril", [-0.08, 0.20]), + ("rightNostril", "Right Nostril", [0.08, 0.20]), + ("mouthLeft", "Mouth Left", [-0.24, 0.50]), + ("mouthRight", "Mouth Right", [0.24, 0.50]), + ("mouthTop", "Mouth Top", [0.0, 0.435]), + ("mouthBottom", "Mouth Bottom", [0.0, 0.565]), + ("chin", "Chin", [0.0, 1.0]), + ("leftJaw", "Left Jaw", [-0.614, 0.574]), + ("rightJaw", "Right Jaw", [0.614, 0.574]), +]; + +/// Number of landmarks. +pub const N: usize = LANDMARKS.len(); + +/// Landmark index by id. +pub fn landmark(id: &str) -> Option { + LANDMARKS.iter().position(|l| l.0 == id) +} + +const BROW: [[usize; 3]; 2] = [[0, 1, 2], [3, 4, 5]]; +/// Eye landmarks: inner, outer, top, bottom, pupil. +const EYE: [[usize; 5]; 2] = [[6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]; +const NOSE_TIP: usize = 16; +const NOSTRIL: [usize; 2] = [17, 18]; +/// Mouth: left, right, top, bottom. +const MOUTH: [usize; 4] = [19, 20, 21, 22]; +const CHIN: usize = 23; +const JAW: [usize; 2] = [24, 25]; +/// Half-width of the face relative to its half-height. +const ASPECT: f64 = 0.75; + +// ---------------------------------------------------------------- synthetic faces + +/// Parameters of a synthetic face (the shape model's training data and the tests' footage). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FaceParams { + /// Face centre (layer pixels) and half-height (pixels). + pub center: [f64; 2], + pub height: f64, + /// Head rotation in degrees: roll (in the image plane), yaw (turn), pitch (nod). + pub roll: f64, + pub yaw: f64, + pub pitch: f64, + /// Eye openness (0 closed … 1 open), left and right. + pub eye_open: [f64; 2], + /// Mouth openness (0 … 1) and width (1 = neutral). + pub mouth_open: f64, + pub mouth_width: f64, + /// Eyebrow raise (−1 … 1), left and right. + pub brow_raise: [f64; 2], +} + +impl Default for FaceParams { + fn default() -> Self { + FaceParams { + center: [0.0; 2], + height: 60.0, + roll: 0.0, + yaw: 0.0, + pitch: 0.0, + eye_open: [1.0; 2], + mouth_open: 0.2, + mouth_width: 1.0, + brow_raise: [0.0; 2], + } + } +} + +fn eye_half_height(open: f64) -> f64 { + 0.012 + 0.06 * open.clamp(0.0, 1.0) +} +fn mouth_half_height(open: f64) -> f64 { + 0.015 + 0.11 * open.clamp(0.0, 1.0) +} + +/// The landmarks of a synthetic face in the face frame (before roll, translation and scale). +pub fn synth_shape(p: &FaceParams) -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + let (sy, sp) = (p.yaw.to_radians().sin(), p.pitch.to_radians().sin()); + // Inner features slide with the head's turn; the nose (closer to the camera) slides more. + let shift = |depth: f64| [0.35 * sy * depth, 0.3 * sp * depth]; + for (i, l) in LANDMARKS.iter().enumerate() { + s[i] = l.2; + } + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + let raise = -0.06 * p.brow_raise[side].clamp(-1.0, 1.0); + for &i in &BROW[side] { + s[i][1] += raise; + } + let h = eye_half_height(p.eye_open[side]); + let e = EYE[side]; + s[e[2]] = [sg * 0.31, -0.22 - h]; + s[e[3]] = [sg * 0.31, -0.22 + h]; + } + let mw = 0.24 * p.mouth_width.clamp(0.5, 1.6); + let mh = mouth_half_height(p.mouth_open); + s[MOUTH[0]] = [-mw, 0.5]; + s[MOUTH[1]] = [mw, 0.5]; + s[MOUTH[2]] = [0.0, 0.5 - mh]; + s[MOUTH[3]] = [0.0, 0.5 + mh]; + for (i, q) in s.iter_mut().enumerate() { + if i == CHIN || JAW.contains(&i) { + continue; + } + let d = if i == NOSE_TIP { + 1.3 + } else if NOSTRIL.contains(&i) { + 1.15 + } else { + 1.0 + }; + let o = shift(d); + q[0] += o[0]; + q[1] += o[1]; + } + s +} + +/// Face frame → layer pixels. +fn to_layer(c: [f64; 2], b: f64, roll_deg: f64, q: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + [c[0] + b * (cs * q[0] - sn * q[1]), c[1] + b * (sn * q[0] + cs * q[1])] +} + +/// Layer pixels → face frame. +fn to_face(c: [f64; 2], b: f64, roll_deg: f64, p: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + let d = [(p[0] - c[0]) / b, (p[1] - c[1]) / b]; + [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]] +} + +/// A synthetic face's landmarks in layer pixels. +pub fn synth_landmarks(p: &FaceParams) -> [[f64; 2]; N] { + synth_shape(p).map(|q| to_layer(p.center, p.height, p.roll, q)) +} + +fn hash(i: i64, j: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 +} + +fn seg_dist(p: [f64; 2], a: [f64; 2], b: [f64; 2]) -> f64 { + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + (p[0] - a[0] - t * d[0]).hypot(p[1] - a[1] - t * d[1]) +} + +/// Colour of a synthetic face at face-frame point `q` (`None` outside the face). +fn face_color(p: &FaceParams, s: &[[f64; 2]; N], q: [f64; 2]) -> Option<[f32; 3]> { + if (q[0] / ASPECT).powi(2) + q[1] * q[1] > 1.0 { + return None; + } + let shade = (0.95 - 0.08 * q[1]) as f32; + let mut col = [0.86 * shade, 0.64 * shade, 0.52 * shade]; + for side in 0..2 { + let b = BROW[side]; + let d = seg_dist(q, s[b[0]], s[b[1]]).min(seg_dist(q, s[b[1]], s[b[2]])); + if d < 0.035 { + col = [0.28, 0.18, 0.12]; + } + let e = EYE[side]; + let ec = [(s[e[0]][0] + s[e[1]][0]) / 2.0, (s[e[2]][1] + s[e[3]][1]) / 2.0]; + let (hw, hh) = ((s[e[1]][0] - s[e[0]][0]).abs() / 2.0, (s[e[3]][1] - s[e[2]][1]).abs() / 2.0); + if ((q[0] - ec[0]) / hw).powi(2) + ((q[1] - ec[1]) / hh).powi(2) <= 1.0 { + let pupil = s[e[4]]; + col = if (q[0] - pupil[0]).hypot(q[1] - pupil[1]) < 0.055 { [0.12, 0.08, 0.06] } else { [0.95, 0.95, 0.93] }; + } + } + let _ = p; + for &n in &NOSTRIL { + if ((q[0] - s[n][0]) / 0.035).powi(2) + ((q[1] - s[n][1]) / 0.022).powi(2) <= 1.0 { + col = [0.3, 0.13, 0.1]; + } + } + let m = MOUTH; + let mc = [(s[m[0]][0] + s[m[1]][0]) / 2.0, (s[m[2]][1] + s[m[3]][1]) / 2.0]; + let (mw, mh) = ((s[m[1]][0] - s[m[0]][0]) / 2.0, (s[m[3]][1] - s[m[2]][1]) / 2.0); + if ((q[0] - mc[0]) / mw).powi(2) + ((q[1] - mc[1]) / mh).powi(2) <= 1.0 { + col = [0.42, 0.08, 0.1]; + } + Some(col) +} + +/// Render a synthetic face over a textured blue-green background (2 × 2 supersampled): the +/// test footage of the face tracker. +pub fn render_synthetic(w: u32, h: u32, p: &FaceParams, background_seed: i64) -> Image { + use rayon::prelude::*; + let s = synth_shape(p); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 3]; + for sy in 0..2 { + for sx in 0..2 { + let l = [x as f64 + 0.25 + 0.5 * sx as f64, y as f64 + 0.25 + 0.5 * sy as f64]; + let q = to_face(p.center, p.height, p.roll, l); + let c = face_color(p, &s, q).unwrap_or_else(|| { + let n = hash((l[0] / 9.0) as i64 + background_seed, (l[1] / 9.0) as i64) as f32; + [0.15 + 0.1 * n, 0.35 + 0.2 * n, 0.45 + 0.25 * n] + }); + for k in 0..3 { + acc[k] += c[k] * 0.25; + } + } + } + *px = [acc[0], acc[1], acc[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- shape model + +/// Point distribution model: mean shape and principal modes (face frame). +pub struct Pdm { + pub mean: Vec, + /// Modes as rows of length `2N`. + pub modes: Vec>, + /// Variance of each mode. + pub var: Vec, +} + +/// The shape model, trained once on synthetic shapes. +pub fn pdm() -> &'static Pdm { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| { + let samples = 600; + let dim = 2 * N; + let mut rng = 0x2545_f491_4f6c_dd1du64; + let mut u = || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + (rng % 1_000_000) as f64 / 1_000_000.0 + }; + let mut data = Vec::with_capacity(samples); + for _ in 0..samples { + let p = FaceParams { + yaw: (u() - 0.5) * 60.0, + pitch: (u() - 0.5) * 40.0, + eye_open: [u(), u()], + mouth_open: u(), + mouth_width: 0.8 + 0.45 * u(), + brow_raise: [u() * 2.0 - 1.0, u() * 2.0 - 1.0], + ..Default::default() + }; + data.push(synth_shape(&p).iter().flat_map(|q| [q[0], q[1]]).collect::>()); + } + let mut mean = vec![0.0; dim]; + for d in &data { + for (m, v) in mean.iter_mut().zip(d) { + *m += v / samples as f64; + } + } + let mut cov = vec![0.0; dim * dim]; + for d in &data { + for a in 0..dim { + let da = d[a] - mean[a]; + for b in 0..dim { + cov[a * dim + b] += da * (d[b] - mean[b]) / samples as f64; + } + } + } + let (vals, vecs) = sym_eigen(&cov, dim); + let total: f64 = vals.iter().map(|v| v.max(0.0)).sum(); + let mut modes = vec![]; + let mut var = vec![]; + let mut acc = 0.0; + for k in (0..dim).rev() { + if acc >= 0.995 * total || vals[k] <= 1e-12 { + break; + } + acc += vals[k]; + modes.push((0..dim).map(|r| vecs[r * dim + k]).collect()); + var.push(vals[k]); + } + Pdm { mean, modes, var } + }) +} + +/// A similarity `p ↦ s R q + t` as `[a, b, tx, ty]` (`x = a u − b v + tx`, `y = b u + a v + ty`). +fn fit_similarity(src: &[[f64; 2]], dst: &[[f64; 2]], w: &[f64]) -> Option<[f64; 4]> { + let ws: f64 = w.iter().sum(); + if ws < 2.0 - 1e-9 { + return None; + } + let mut cs = [0.0; 2]; + let mut cd = [0.0; 2]; + for i in 0..src.len() { + for k in 0..2 { + cs[k] += w[i] * src[i][k] / ws; + cd[k] += w[i] * dst[i][k] / ws; + } + } + let (mut ss, mut dt, mut cr) = (0.0, 0.0, 0.0); + for i in 0..src.len() { + let a = [src[i][0] - cs[0], src[i][1] - cs[1]]; + let b = [dst[i][0] - cd[0], dst[i][1] - cd[1]]; + ss += w[i] * (a[0] * a[0] + a[1] * a[1]); + dt += w[i] * (a[0] * b[0] + a[1] * b[1]); + cr += w[i] * (a[0] * b[1] - a[1] * b[0]); + } + if ss < 1e-12 { + return None; + } + let (a, b) = (dt / ss, cr / ss); + Some([a, b, cd[0] - (a * cs[0] - b * cs[1]), cd[1] - (b * cs[0] + a * cs[1])]) +} + +fn sim_apply(t: &[f64; 4], q: [f64; 2]) -> [f64; 2] { + [t[0] * q[0] - t[1] * q[1] + t[2], t[1] * q[0] + t[0] * q[1] + t[3]] +} + +fn sim_inverse(t: &[f64; 4], p: [f64; 2]) -> [f64; 2] { + let d = [p[0] - t[2], p[1] - t[3]]; + let s2 = t[0] * t[0] + t[1] * t[1]; + [(t[0] * d[0] + t[1] * d[1]) / s2, (-t[1] * d[0] + t[0] * d[1]) / s2] +} + +/// Fit the shape model to detected landmarks (`w` = 0 for missing ones), starting from the +/// face-frame similarity `init`. Returns (model landmarks in layer pixels, similarity). +pub fn fit_shape(detected: &[[f64; 2]; N], w: &[f64; N], init: [f64; 4]) -> ([[f64; 2]; N], [f64; 4]) { + let m = pdm(); + let k = m.modes.len(); + let mut b = vec![0.0; k]; + let mut t = init; + let shape = |b: &[f64]| -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + for i in 0..N { + for d in 0..2 { + let j = 2 * i + d; + s[i][d] = m.mean[j] + m.modes.iter().zip(b).map(|(row, bk)| row[j] * bk).sum::(); + } + } + s + }; + for _ in 0..10 { + let x = shape(&b); + if let Some(nt) = fit_similarity(&x, detected, w) { + t = nt; + } + // Detections in the model frame. + let y: Vec<[f64; 2]> = detected.iter().map(|p| sim_inverse(&t, *p)).collect(); + let mut ata = vec![0.0; k * k]; + let mut atb = vec![0.0; k]; + for i in 0..N { + if w[i] <= 0.0 { + continue; + } + for d in 0..2 { + let j = 2 * i + d; + let r = y[i][d] - m.mean[j]; + for a in 0..k { + atb[a] += w[i] * m.modes[a][j] * r; + for c in 0..k { + ata[a * k + c] += w[i] * m.modes[a][j] * m.modes[c][j]; + } + } + } + } + for a in 0..k { + // A weak prior towards the mean (Tikhonov, scaled by the mode's variance). + ata[a * k + a] += 1e-3 / m.var[a].max(1e-9) * 1e-3 + 1e-9; + } + if cholesky_solve(&mut ata, k, &mut atb) { + for a in 0..k { + let lim = 3.0 * m.var[a].sqrt(); + b[a] = atb[a].clamp(-lim, lim); + } + } + } + let x = shape(&b); + (x.map(|q| sim_apply(&t, q)), t) +} + +// ---------------------------------------------------------------- tracking + +/// The skin colour model (luma, r and g chromaticity: median and spread). +#[derive(Clone, Copy, Debug, PartialEq)] +struct Skin { + med: [f32; 3], + tol: [f32; 3], +} + +fn features(px: [f32; 4]) -> [f32; 3] { + let s = (px[0] + px[1] + px[2]).max(1e-4); + [0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2], px[0] / s, px[1] / s] +} + +impl Skin { + fn is_skin(&self, px: [f32; 4]) -> bool { + let f = features(px); + (0..3).all(|k| (f[k] - self.med[k]).abs() <= self.tol[k]) + } +} + +/// One frame's face fit. +#[derive(Clone, Debug, PartialEq)] +pub struct FaceFit { + /// Landmarks ([`LANDMARKS`] order), layer pixels. + pub landmarks: [[f64; 2]; N], + /// Which landmarks were measured on this frame (the others come from the shape model). + pub detected: [bool; N], + /// The face outline: a closed polygon (layer pixels). + pub outline: Vec<[f64; 2]>, + /// Face ellipse centre, half-height (pixels) and roll (degrees). + pub center: [f64; 2], + pub height: f64, + pub roll: f64, +} + +/// Follows one face through a clip. +#[derive(Clone, Debug)] +pub struct FaceTracker { + skin: Skin, + prev: FaceFit, + /// Previous landmarks in the face frame (the expected feature positions). + expect: [[f64; 2]; N], +} + +/// A pixel window of a frame in layer coordinates. +struct Window { + x0: i64, + y0: i64, + w: usize, + h: usize, +} + +impl Window { + fn around(frame: &Frame, c: [f64; 2], rx: f64, ry: f64) -> Option { + let (ox, oy) = (frame.offset[0], frame.offset[1]); + let x0 = ((c[0] - rx + ox).floor() as i64).max(0); + let y0 = ((c[1] - ry + oy).floor() as i64).max(0); + let x1 = ((c[0] + rx + ox).ceil() as i64).min(frame.img.width as i64); + let y1 = ((c[1] + ry + oy).ceil() as i64).min(frame.img.height as i64); + (x1 > x0 + 4 && y1 > y0 + 4).then(|| Window { x0, y0, w: (x1 - x0) as usize, h: (y1 - y0) as usize }) + } + fn px(&self, frame: &Frame, i: usize, j: usize) -> [f32; 4] { + frame.img.get(self.x0 + i as i64, self.y0 + j as i64) + } + /// Layer position of the centre of window pixel `(i, j)`. + fn layer(&self, frame: &Frame, i: usize, j: usize) -> [f64; 2] { + [(self.x0 + i as i64) as f64 + 0.5 - frame.offset[0], (self.y0 + j as i64) as f64 + 0.5 - frame.offset[1]] + } + fn cell(&self, frame: &Frame, p: [f64; 2]) -> Option<(usize, usize)> { + let x = (p[0] + frame.offset[0]).floor() as i64 - self.x0; + let y = (p[1] + frame.offset[1]).floor() as i64 - self.y0; + (x >= 0 && y >= 0 && (x as usize) < self.w && (y as usize) < self.h).then_some((x as usize, y as usize)) + } +} + +fn median(v: &mut [f32]) -> f32 { + v.sort_by(f32::total_cmp); + v[v.len() / 2] +} + +/// Area-moment ellipse of a closed polygon: (centroid, half-height along the major axis, +/// half-width, major axis angle from vertical in degrees). +fn polygon_ellipse(poly: &[[f64; 2]]) -> Option<([f64; 2], f64, f64, f64)> { + let n = poly.len(); + let (mut a, mut cx, mut cy) = (0.0, 0.0, 0.0); + let (mut ixx, mut iyy, mut ixy) = (0.0, 0.0, 0.0); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let cr = p[0] * q[1] - q[0] * p[1]; + a += cr; + cx += (p[0] + q[0]) * cr; + cy += (p[1] + q[1]) * cr; + ixx += (p[1] * p[1] + p[1] * q[1] + q[1] * q[1]) * cr; + iyy += (p[0] * p[0] + p[0] * q[0] + q[0] * q[0]) * cr; + ixy += (p[0] * q[1] + 2.0 * p[0] * p[1] + 2.0 * q[0] * q[1] + q[0] * p[1]) * cr; + } + a *= 0.5; + if a.abs() < 1e-9 { + return None; + } + cx /= 6.0 * a; + cy /= 6.0 * a; + // Central second moments (per unit area): var x, var y, cov xy. + let vxx = iyy / (12.0 * a) - cx * cx; + let vyy = ixx / (12.0 * a) - cy * cy; + let vxy = ixy / (24.0 * a) - cx * cy; + let tr = vxx + vyy; + let det = vxx * vyy - vxy * vxy; + let disc = (tr * tr / 4.0 - det).max(0.0).sqrt(); + let (l1, l2) = (tr / 2.0 + disc, (tr / 2.0 - disc).max(0.0)); + // Major axis direction. + let ang = 0.5 * (2.0 * vxy).atan2(vxx - vyy); // from +x + let mut from_vertical = ang.to_degrees() - 90.0; + while from_vertical > 90.0 { + from_vertical -= 180.0; + } + while from_vertical < -90.0 { + from_vertical += 180.0; + } + Some(([cx, cy], 2.0 * l1.sqrt(), 2.0 * l2.sqrt(), from_vertical)) +} + +/// A non-skin component inside the face. +struct Blob { + /// Face-frame positions of its pixels and their darkness (skin luma − luma). + pts: Vec<([f64; 2], f64)>, + centroid: [f64; 2], +} + +impl Blob { + fn extreme(&self, dir: [f64; 2]) -> [f64; 2] { + self.pts.iter().map(|p| p.0).max_by(|a, b| (a[0] * dir[0] + a[1] * dir[1]).total_cmp(&(b[0] * dir[0] + b[1] * dir[1]))).unwrap_or(self.centroid) + } + /// Extreme along `v` (`sign` −1 = top) among the pixels within `band` of the column `u`. + fn lid(&self, u: f64, band: f64, sign: f64) -> [f64; 2] { + let c = self.pts.iter().filter(|p| (p.0[0] - u).abs() <= band).map(|p| p.0); + let best = c.max_by(|a, b| (sign * a[1]).total_cmp(&(sign * b[1]))); + best.map(|p| [u, p[1]]).unwrap_or(self.centroid) + } + fn dark_centroid(&self) -> [f64; 2] { + let (mut s, mut w) = ([0.0; 2], 0.0); + for (q, d) in &self.pts { + let k = d.max(0.0).powi(2); + s[0] += k * q[0]; + s[1] += k * q[1]; + w += k; + } + if w > 1e-12 { [s[0] / w, s[1] / w] } else { self.centroid } + } +} + +impl FaceTracker { + /// Start on `frame` with the face inside `region` (`[x0, y0, x1, y1]`, layer pixels: the + /// mask's bounds). `None` when no face-like skin region is found. + pub fn new(frame: &Frame, region: [f64; 4]) -> Option<(FaceTracker, FaceFit)> { + let c = [(region[0] + region[2]) / 2.0, (region[1] + region[3]) / 2.0]; + let (hw, hh) = ((region[2] - region[0]) / 2.0, (region[3] - region[1]) / 2.0); + if !(hw > 4.0 && hh > 4.0) { + return None; + } + // Skin colour: the middle of the region (cheeks and nose). + let win = Window::around(frame, c, hw * 0.35, hh * 0.3)?; + let mut f: [Vec; 3] = [vec![], vec![], vec![]]; + for j in 0..win.h { + for i in 0..win.w { + let v = features(win.px(frame, i, j)); + for k in 0..3 { + f[k].push(v[k]); + } + } + } + let mut med = [0.0f32; 3]; + let mut tol = [0.0f32; 3]; + let floors = [0.12f32, 0.025, 0.025]; + for k in 0..3 { + med[k] = median(&mut f[k]); + let mut dev: Vec = f[k].iter().map(|v| (v - med[k]).abs()).collect(); + tol[k] = (4.0 * median(&mut dev)).max(floors[k]); + } + let skin = Skin { med, tol }; + let canon: [[f64; 2]; N] = LANDMARKS.map(|l| l.2); + let guess = FaceFit { + landmarks: canon.map(|q| to_layer(c, hh, 0.0, q)), + detected: [false; N], + outline: vec![], + center: c, + height: hh.max(hw / ASPECT), + roll: 0.0, + }; + let mut t = FaceTracker { skin, prev: guess, expect: canon }; + let fit = t.fit(frame)?; + t.accept(&fit); + Some((t, fit)) + } + + /// The fit of the last tracked frame. + pub fn last(&self) -> &FaceFit { + &self.prev + } + + fn accept(&mut self, fit: &FaceFit) { + self.expect = fit.landmarks.map(|p| to_face(fit.center, fit.height, fit.roll, p)); + self.prev = fit.clone(); + } + + /// Track into the next frame (either direction). `None` when the face is lost. + pub fn step(&mut self, frame: &Frame) -> Option { + let fit = self.fit(frame)?; + self.accept(&fit); + Some(fit) + } + + fn fit(&self, frame: &Frame) -> Option { + let pv = &self.prev; + let b0 = pv.height; + let reach = 1.7 * b0; + let win = Window::around(frame, pv.center, reach, reach)?; + let (w, h) = (win.w, win.h); + let mut skin = vec![false; w * h]; + let mut luma = vec![0.0f32; w * h]; + for j in 0..h { + for i in 0..w { + let px = win.px(frame, i, j); + skin[j * w + i] = self.skin.is_skin(px); + luma[j * w + i] = features(px)[0]; + } + } + // Seed: the skin pixel nearest the predicted centre. + let (ci, cj) = win.cell(frame, pv.center)?; + let mut seed = None; + 'search: for r in 0..(0.6 * b0).ceil() as i64 { + for dj in -r..=r { + for di in -r..=r { + if di.abs() != r && dj.abs() != r { + continue; + } + let (i, j) = (ci as i64 + di, cj as i64 + dj); + if i >= 0 && j >= 0 && (i as usize) < w && (j as usize) < h && skin[j as usize * w + i as usize] { + seed = Some((i as usize, j as usize)); + break 'search; + } + } + } + } + let (si, sj) = seed?; + // Flood fill (4-connected). + let mut comp = vec![false; w * h]; + let mut stack = vec![(si, sj)]; + comp[sj * w + si] = true; + let mut count = 0usize; + let (mut mx, mut my) = (0.0, 0.0); + while let Some((i, j)) = stack.pop() { + count += 1; + let p = win.layer(frame, i, j); + mx += p[0]; + my += p[1]; + let nb = [(i.wrapping_sub(1), j), (i + 1, j), (i, j.wrapping_sub(1)), (i, j + 1)]; + for (a, b) in nb { + if a < w && b < h && !comp[b * w + a] && skin[b * w + a] { + comp[b * w + a] = true; + stack.push((a, b)); + } + } + } + if (count as f64) < 0.15 * std::f64::consts::PI * ASPECT * b0 * b0 { + return None; + } + let c0 = [mx / count as f64, my / count as f64]; + // Outline along rays: the farthest component pixel. + let rays = 48; + let mut radii = vec![0.0f64; rays]; + for (k, r) in radii.iter_mut().enumerate() { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + let d = [a.cos(), a.sin()]; + let mut t = 0.0; + while t < reach * 1.4 { + let p = [c0[0] + d[0] * t, c0[1] + d[1] * t]; + match win.cell(frame, p) { + Some((i, j)) => { + if comp[j * w + i] { + *r = t + 0.5; + } + } + None => break, + } + t += 0.5; + } + } + // Median of 5, circularly. + let sm: Vec = (0..rays) + .map(|k| { + let mut v: Vec = (0..5).map(|d| radii[(k + rays + d - 2) % rays]).collect(); + v.sort_by(f64::total_cmp); + v[2] + }) + .collect(); + let outline: Vec<[f64; 2]> = (0..rays) + .map(|k| { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + [c0[0] + a.cos() * sm[k], c0[1] + a.sin() * sm[k]] + }) + .collect(); + let (center, major, _minor, ang) = polygon_ellipse(&outline)?; + if !(major > 0.3 * b0 && major < 2.5 * b0) { + return None; + } + // Roll: the ellipse axis nearest the previous roll (it is ambiguous by 180°). + let mut roll = ang; + while roll - pv.roll > 90.0 { + roll -= 180.0; + } + while roll - pv.roll < -90.0 { + roll += 180.0; + } + let b = major; + // Non-skin components inside the face outline (shrunk a little: anti-aliased edges). + let inner: Vec<[f64; 2]> = outline.iter().map(|p| [center[0] + 0.9 * (p[0] - center[0]), center[1] + 0.9 * (p[1] - center[1])]).collect(); + let mut feat = vec![false; w * h]; + for j in 0..h { + for i in 0..w { + if !skin[j * w + i] && crate::mask::inside_polygon(&inner, win.layer(frame, i, j)) { + feat[j * w + i] = true; + } + } + } + let min_area = (0.0006 * b * b).max(2.0) as usize; + let mut seen = vec![false; w * h]; + let mut blobs: Vec = vec![]; + let skin_luma = self.skin.med[0] as f64; + for j0 in 0..h { + for i0 in 0..w { + if !feat[j0 * w + i0] || seen[j0 * w + i0] { + continue; + } + let mut st = vec![(i0, j0)]; + seen[j0 * w + i0] = true; + let mut pts = vec![]; + while let Some((i, j)) = st.pop() { + let q = to_face(center, b, roll, win.layer(frame, i, j)); + pts.push((q, skin_luma - luma[j * w + i] as f64)); + for dj in -1i64..=1 { + for di in -1i64..=1 { + let (a, c) = (i as i64 + di, j as i64 + dj); + if a < 0 || c < 0 || a as usize >= w || c as usize >= h { + continue; + } + let (a, c) = (a as usize, c as usize); + if feat[c * w + a] && !seen[c * w + a] { + seen[c * w + a] = true; + st.push((a, c)); + } + } + } + } + if pts.len() >= min_area { + let n = pts.len() as f64; + let centroid = [pts.iter().map(|p| p.0[0]).sum::() / n, pts.iter().map(|p| p.0[1]).sum::() / n]; + blobs.push(Blob { pts, centroid }); + } + } + } + // Match components to features near their expected positions. + let ex = &self.expect; + let mean_of = |ids: &[usize]| { + let n = ids.len() as f64; + [ids.iter().map(|i| ex[*i][0]).sum::() / n, ids.iter().map(|i| ex[*i][1]).sum::() / n] + }; + let mut used = vec![false; blobs.len()]; + let take = |target: [f64; 2], radius: f64, prefer_large: bool, used: &mut Vec| -> Option { + let mut best: Option<(usize, f64)> = None; + for (k, bl) in blobs.iter().enumerate() { + if used[k] { + continue; + } + let d = (bl.centroid[0] - target[0]).hypot(bl.centroid[1] - target[1]); + if d > radius { + continue; + } + let score = if prefer_large { -(bl.pts.len() as f64) } else { d }; + if best.is_none_or(|b| score < b.1) { + best = Some((k, score)); + } + } + if let Some((k, _)) = best { + used[k] = true; + } + best.map(|b| b.0) + }; + let mut det = [[0.0; 2]; N]; + let mut wgt = [0.0; N]; + let set = |i: usize, q: [f64; 2], det: &mut [[f64; 2]; N], wgt: &mut [f64; N]| { + det[i] = to_layer(center, b, roll, q); + wgt[i] = 1.0; + }; + let eye_ids = [EYE[0].to_vec(), EYE[1].to_vec()]; + let mouth = take(mean_of(&MOUTH), 0.25, true, &mut used); + let eyes = [take(mean_of(&eye_ids[0]), 0.15, false, &mut used), take(mean_of(&eye_ids[1]), 0.15, false, &mut used)]; + let brows = [take(mean_of(&BROW[0]), 0.15, false, &mut used), take(mean_of(&BROW[1]), 0.15, false, &mut used)]; + let nostrils = [take(ex[NOSTRIL[0]], 0.07, false, &mut used), take(ex[NOSTRIL[1]], 0.07, false, &mut used)]; + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + if let Some(k) = eyes[side] { + let bl = &blobs[k]; + let e = EYE[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.15 * (outer[0] - inner[0]).abs(); + let px = 0.5 / b; + set(e[0], [inner[0] - sg * px, inner[1]], &mut det, &mut wgt); + set(e[1], [outer[0] + sg * px, outer[1]], &mut det, &mut wgt); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(e[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(e[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + set(e[4], bl.dark_centroid(), &mut det, &mut wgt); + } + if let Some(k) = brows[side] { + let bl = &blobs[k]; + let br = BROW[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + set(br[0], inner, &mut det, &mut wgt); + set(br[2], outer, &mut det, &mut wgt); + // The middle: the brow's centre line at its mid column. + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.1 * (outer[0] - inner[0]).abs().max(0.05); + let col: Vec = bl.pts.iter().filter(|p| (p.0[0] - mid).abs() <= band).map(|p| p.0[1]).collect(); + if !col.is_empty() { + set(br[1], [mid, col.iter().sum::() / col.len() as f64], &mut det, &mut wgt); + } + } + if let Some(k) = nostrils[side] { + set(NOSTRIL[side], blobs[k].centroid, &mut det, &mut wgt); + } + } + if let (Some(a), Some(c)) = (nostrils[0], nostrils[1]) { + let (p, q) = (blobs[a].centroid, blobs[c].centroid); + set(NOSE_TIP, [0.5 * (p[0] + q[0]), 0.5 * (p[1] + q[1]) - 0.06], &mut det, &mut wgt); + } + if let Some(k) = mouth { + let bl = &blobs[k]; + let l = bl.extreme([-1.0, 0.0]); + let r = bl.extreme([1.0, 0.0]); + let px = 0.5 / b; + set(MOUTH[0], [l[0] - px, l[1]], &mut det, &mut wgt); + set(MOUTH[1], [r[0] + px, r[1]], &mut det, &mut wgt); + let mid = 0.5 * (l[0] + r[0]); + let band = 0.12 * (r[0] - l[0]).abs(); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(MOUTH[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(MOUTH[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + } + // Chin and jaw from the outline, along their canonical directions. + for i in [CHIN, JAW[0], JAW[1]] { + let q = LANDMARKS[i].2; + let dir = to_layer([0.0; 2], 1.0, roll, q); + let ang = dir[1].atan2(dir[0]); + let rad = ray_radius(&outline, center, ang); + let len = dir[0].hypot(dir[1]); + let p = [center[0] + dir[0] / len * rad, center[1] + dir[1] / len * rad]; + det[i] = p; + wgt[i] = 1.0; + } + // Shape model: fill the missing landmarks and replace inconsistent detections. + let (sn, cs) = roll.to_radians().sin_cos(); + let init = [b * cs, b * sn, center[0], center[1]]; + let (model, _) = fit_shape(&det, &wgt, init); + let mut w2 = wgt; + for i in 0..N { + if wgt[i] > 0.0 && (det[i][0] - model[i][0]).hypot(det[i][1] - model[i][1]) > 0.12 * b { + w2[i] = 0.0; + } + } + let (model, sim) = if w2 != wgt { fit_shape(&det, &w2, init) } else { (model, init) }; + let _ = sim; + let mut landmarks = model; + let mut detected = [false; N]; + for i in 0..N { + if w2[i] > 0.0 { + landmarks[i] = det[i]; + detected[i] = true; + } + } + // Roll from the jaw line (it does not move with yaw); fall back to the ellipse. + let jl = landmarks[JAW[0]]; + let jr = landmarks[JAW[1]]; + let roll_jaw = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + let roll = if (roll_jaw - roll).abs() < 20.0 { roll_jaw } else { roll }; + Some(FaceFit { landmarks, detected, outline, center, height: b, roll }) + } +} + +/// Distance from `c` to the polygon `poly` (star-shaped around `c`) along angle `ang`. +fn ray_radius(poly: &[[f64; 2]], c: [f64; 2], ang: f64) -> f64 { + let d = [ang.cos(), ang.sin()]; + let mut best = 0.0f64; + let n = poly.len(); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let e = [q[0] - p[0], q[1] - p[1]]; + let den = d[0] * e[1] - d[1] * e[0]; + if den.abs() < 1e-12 { + continue; + } + let w = [p[0] - c[0], p[1] - c[1]]; + let t = (w[0] * e[1] - w[1] * e[0]) / den; + let s = (w[0] * d[1] - w[1] * d[0]) / den; + if t > 0.0 && (-1e-9..=1.0 + 1e-9).contains(&s) { + best = best.max(t); + } + } + best +} + +// ---------------------------------------------------------------- measurements + +/// Face measurements (Extract & Copy Face Measurements): (id, display name). +pub const MEASUREMENTS: [(&str, &str); 14] = [ + ("headPositionX", "Head Position X"), + ("headPositionY", "Head Position Y"), + ("headScale", "Head Scale"), + ("headOrientationX", "Head Orientation X"), + ("headOrientationY", "Head Orientation Y"), + ("headOrientationZ", "Head Orientation Z"), + ("leftEyeOpenness", "Left Eye Openness"), + ("rightEyeOpenness", "Right Eye Openness"), + ("leftEyebrowRaise", "Left Eyebrow Distance from Eye"), + ("rightEyebrowRaise", "Right Eyebrow Distance from Eye"), + ("mouthOpenness", "Mouth Openness"), + ("mouthWidth", "Mouth Width"), + ("mouthOffsetX", "Mouth Offset X"), + ("mouthOffsetY", "Mouth Offset Y"), +]; + +/// Face centre, half-height and roll from the chin and jaw landmarks. +pub fn face_frame(l: &[[f64; 2]; N]) -> ([f64; 2], f64, f64) { + let (ch, jl, jr) = (l[CHIN], l[JAW[0]], l[JAW[1]]); + let jm = [0.5 * (jl[0] + jr[0]), 0.5 * (jl[1] + jr[1])]; + let k = 1.0 - LANDMARKS[JAW[0]].2[1]; + let c = [ch[0] + (jm[0] - ch[0]) / k, ch[1] + (jm[1] - ch[1]) / k]; + // Half-height from the chin and from the jaw width (averaged: less noise). + let jw = (jr[0] - jl[0]).hypot(jr[1] - jl[1]) / (2.0 * LANDMARKS[JAW[1]].2[0]); + let b = (0.5 * ((ch[0] - c[0]).hypot(ch[1] - c[1]) + jw)).max(1e-9); + let roll = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + (c, b, roll) +} + +/// The measurements of a frame's landmarks ([`MEASUREMENTS`] order); `reference_height`: the +/// face half-height that is Head Scale 100 %. +pub fn measure(l: &[[f64; 2]; N], reference_height: f64) -> [f64; 14] { + let (c, b, roll) = face_frame(l); + let q = l.map(|p| to_face(c, b, roll, p)); + let canon = LANDMARKS.map(|x| x.2); + // Inner features slide with yaw and pitch (see `synth_shape`). + let inner = [EYE[0][4], EYE[1][4], MOUTH[0], MOUTH[1], EYE[0][0], EYE[1][0], EYE[0][1], EYE[1][1]]; + let n = inner.len() as f64; + let du = inner.iter().map(|i| q[*i][0] - canon[*i][0]).sum::() / n; + let dv = inner.iter().map(|i| q[*i][1] - canon[*i][1]).sum::() / n; + let yaw = (du / 0.35).clamp(-1.0, 1.0).asin().to_degrees(); + let pitch = (dv / 0.3).clamp(-1.0, 1.0).asin().to_degrees(); + let open = |e: [usize; 5]| { + let wdt = (q[e[1]][0] - q[e[0]][0]).abs().max(1e-9); + (q[e[3]][1] - q[e[2]][1]).max(0.0) / wdt * 100.0 + }; + let brow = |side: usize| { + let e = EYE[side]; + let eye_v = 0.5 * (q[e[2]][1] + q[e[3]][1]); + (eye_v - q[BROW[side][1]][1]) * 100.0 + }; + let mw = (q[MOUTH[1]][0] - q[MOUTH[0]][0]).abs(); + let mc = [0.5 * (q[MOUTH[0]][0] + q[MOUTH[1]][0]), 0.5 * (q[MOUTH[2]][1] + q[MOUTH[3]][1])]; + [ + c[0], + c[1], + b / reference_height.max(1e-9) * 100.0, + pitch, + yaw, + roll, + open(EYE[0]), + open(EYE[1]), + brow(0), + brow(1), + (q[MOUTH[3]][1] - q[MOUTH[2]][1]).max(0.0) / mw.max(1e-9) * 100.0, + mw / (2.0 * ASPECT) * 100.0, + (mc[0] - q[NOSE_TIP][0]) * 100.0, + (mc[1] - q[NOSE_TIP][1]) * 100.0, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame_params(t: usize) -> FaceParams { + let tf = t as f64; + FaceParams { + center: [160.0 + 30.0 * (tf * 0.15).sin(), 120.0 + 12.0 * (tf * 0.1).cos()], + height: 62.0 * (1.0 + 0.08 * (tf * 0.07).sin()), + roll: 14.0 * (tf * 0.12).sin(), + yaw: 12.0 * (tf * 0.09).sin(), + pitch: 6.0 * (tf * 0.11).cos(), + eye_open: [0.8 + 0.2 * (tf * 0.3).sin(), 0.8 + 0.2 * (tf * 0.3).sin()], + mouth_open: 0.5 + 0.45 * (tf * 0.2).sin(), + mouth_width: 1.0 + 0.1 * (tf * 0.13).cos(), + brow_raise: [0.4 * (tf * 0.17).sin(), 0.4 * (tf * 0.17).sin()], + } + } + + #[test] + fn shape_model_spans_the_synthetic_shapes() { + let m = pdm(); + assert!(m.modes.len() >= 5 && m.modes.len() <= 14, "{} modes", m.modes.len()); + // A synthetic shape is reproduced from half its landmarks. + let p = FaceParams { yaw: 15.0, mouth_open: 0.8, eye_open: [0.3, 0.9], center: [100.0, 80.0], height: 50.0, roll: 10.0, ..Default::default() }; + let truth = synth_landmarks(&p); + let mut w = [0.0; N]; + for (i, wi) in w.iter_mut().enumerate() { + *wi = if i % 2 == 0 || i >= 19 { 1.0 } else { 0.0 }; + } + let (sn, cs) = 10f64.to_radians().sin_cos(); + let (fit, _) = fit_shape(&truth, &w, [50.0 * cs, 50.0 * sn, 100.0, 80.0]); + let err = (0..N).map(|i| (fit[i][0] - truth[i][0]).hypot(fit[i][1] - truth[i][1])).fold(0.0, f64::max); + assert!(err < 0.06 * 50.0, "max error {err}"); + } + + #[test] + fn tracks_a_moving_rotating_synthetic_face() { + let (w, h) = (320, 240); + let p0 = frame_params(0); + let img0 = render_synthetic(w, h, &p0, 3); + // The user's mask: a loose box around the face. + let region = [p0.center[0] - 60.0, p0.center[1] - 75.0, p0.center[0] + 60.0, p0.center[1] + 75.0]; + let (mut tr, fit0) = FaceTracker::new(&Frame { img: &img0, offset: [0.0; 2] }, region).expect("face found"); + let mut fits = vec![fit0]; + let n = 40; + for t in 1..n { + let img = render_synthetic(w, h, &frame_params(t), 3); + fits.push(tr.step(&Frame { img: &img, offset: [0.0; 2] }).unwrap_or_else(|| panic!("lost at {t}"))); + } + let mut per = [0.0f64; N]; + let mut worst = 0.0f64; + let mut outline_err = 0.0; + let mut meas = vec![]; + for (t, f) in fits.iter().enumerate() { + let p = frame_params(t); + let truth = synth_landmarks(&p); + for i in 0..N { + let e = (f.landmarks[i][0] - truth[i][0]).hypot(f.landmarks[i][1] - truth[i][1]) / p.height; + per[i] += e / n as f64; + worst = worst.max(e); + } + // Outline: on the face ellipse. + let mut oe = 0.0; + for q in &f.outline { + let u = to_face(p.center, p.height, p.roll, *q); + oe += ((u[0] / ASPECT).powi(2) + u[1] * u[1]).sqrt() - 1.0; + } + outline_err += (oe / f.outline.len() as f64).abs() / n as f64; + meas.push((p, measure(&f.landmarks, fits[0].height))); + assert!((f.roll - p.roll).abs() < 3.0, "frame {t}: roll {} vs {}", f.roll, p.roll); + } + let mean = per.iter().sum::() / N as f64; + eprintln!("face landmarks: mean error {mean:.4}, worst {worst:.4} face heights; outline {outline_err:.4}"); + for (i, e) in per.iter().enumerate() { + assert!(*e < 0.04, "{}: mean error {e:.3} (face heights)", LANDMARKS[i].0); + } + assert!(mean < 0.02, "mean landmark error {mean:.4}"); + assert!(worst < 0.08, "worst landmark error {worst:.3}"); + assert!(outline_err < 0.04, "outline error {outline_err:.3}"); + // Measurements follow the generator: mouth openness and yaw correlate strongly. + let corr = |a: &[f64], b: &[f64]| { + let n = a.len() as f64; + let (ma, mb) = (a.iter().sum::() / n, b.iter().sum::() / n); + let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0); + for (x, y) in a.iter().zip(b) { + sab += (x - ma) * (y - mb); + saa += (x - ma).powi(2); + sbb += (y - mb).powi(2); + } + sab / (saa * sbb).sqrt() + }; + let mo: Vec = meas.iter().map(|m| m.1[10]).collect(); + let mt: Vec = meas.iter().map(|m| m.0.mouth_open).collect(); + assert!(corr(&mo, &mt) > 0.95, "mouth openness r = {}", corr(&mo, &mt)); + let yw: Vec = meas.iter().map(|m| m.1[4]).collect(); + let yt: Vec = meas.iter().map(|m| m.0.yaw).collect(); + assert!(corr(&yw, &yt) > 0.9, "yaw r = {}", corr(&yw, &yt)); + let rl: Vec = meas.iter().map(|m| m.1[5]).collect(); + let rt: Vec = meas.iter().map(|m| m.0.roll).collect(); + assert!(corr(&rl, &rt) > 0.98, "roll r = {}", corr(&rl, &rt)); + let sc: Vec = meas.iter().map(|m| m.1[2]).collect(); + let st: Vec = meas.iter().map(|m| m.0.height).collect(); + assert!(corr(&sc, &st) > 0.95, "scale r = {}", corr(&sc, &st)); + } + + #[test] + fn measurements_of_the_synthetic_model_are_exact() { + let p = FaceParams { center: [50.0, 60.0], height: 40.0, roll: 20.0, yaw: 10.0, pitch: -5.0, ..Default::default() }; + let m = measure(&synth_landmarks(&p), 40.0); + assert!((m[0] - 50.0).abs() < 1e-6 && (m[1] - 60.0).abs() < 1e-6, "{m:?}"); + assert!((m[2] - 100.0).abs() < 1e-6); + assert!((m[5] - 20.0).abs() < 1e-6); + assert!((m[4] - 10.0).abs() < 0.5 && (m[3] + 5.0).abs() < 0.5, "{m:?}"); + } +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index 1c6e6a15..f4a7b88f 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -19,6 +19,7 @@ //! after subtracting the frame's `offset` (see [`Frame`]). pub mod camtrack; +pub mod face; pub mod fit; pub mod klt; pub mod mask; @@ -26,6 +27,7 @@ pub mod plane; pub mod roto; pub mod solve; pub mod stabilize; +pub mod subspace; use effectcraft_raster::Image; use rayon::prelude::*; diff --git a/crates/track/src/solve.rs b/crates/track/src/solve.rs index 6c267721..693c52f7 100644 --- a/crates/track/src/solve.rs +++ b/crates/track/src/solve.rs @@ -37,6 +37,12 @@ pub fn solve_linear(mut a: [[f64; N]; N], mut b: [f64; N]) -> Op #[derive(Clone, Copy, Debug, PartialEq)] pub struct Homography(pub [[f64; 3]; 3]); +impl Default for Homography { + fn default() -> Self { + Homography::IDENTITY + } +} + impl Homography { pub const IDENTITY: Homography = Homography([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); diff --git a/crates/track/src/stabilize.rs b/crates/track/src/stabilize.rs index 1abe79f9..9769fcbd 100644 --- a/crates/track/src/stabilize.rs +++ b/crates/track/src/stabilize.rs @@ -18,14 +18,23 @@ //! Video Stabilization with Robust L1 Optimal Camera Paths" (CVPR 2011), applied here //! per frame rather than through a linear program. //! +//! +//! *Subspace Warp* additionally follows long feature trajectories through the clip +//! ([`crate::camtrack::TrackAnalyzer`]) and stabilises each frame with a content-preserving mesh +//! warp fitted to the subspace-smoothed trajectories ([`crate::subspace`]): scenes with depth +//! (parallax) that one homography per frame cannot hold still are stabilised locally. The +//! framing (crop, auto-scale, Maximum Scale relaxation) works the same on meshes. +//! //! Coordinates are layer pixels; transforms map *source* frame pixels to *stabilized* pixels. use serde::{Deserialize, Serialize}; use crate::Frame; -use crate::fit::{Model, ransac}; +use crate::camtrack::{Track2D, TrackAnalyzer}; +use crate::fit::{Model, least_squares, ransac}; use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; use crate::solve::Homography; +use crate::subspace::{Mesh, MeshOpts, smooth_trajectories, solve_mesh}; /// Inter-frame motion of one frame (from the previous frame to this one) for each model. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -82,6 +91,10 @@ pub struct WarpAnalysis { pub detailed: bool, /// Per frame; frame 0's motion is the identity. pub frames: Vec, + /// Long feature trajectories (layer pixels, 0.1 px) for Subspace Warp; empty in analyses + /// made before it (which then stabilise like Perspective). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tracks: Vec, } impl WarpAnalysis { @@ -120,6 +133,7 @@ pub struct Analyzer { prev: Option, frames: Vec, size: [f64; 2], + trajectories: TrackAnalyzer, } fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { @@ -129,7 +143,8 @@ fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { impl Analyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> Analyzer { - Analyzer { opts, prev: None, frames: vec![], size } + let (side, want) = if opts.detailed { (640, 260) } else { (480, 160) }; + Analyzer { opts, prev: None, frames: vec![], size, trajectories: TrackAnalyzer::with_budget(size, side, want) } } pub fn len(&self) -> usize { @@ -144,6 +159,7 @@ impl Analyzer { let (max_side, max_feat) = if self.opts.detailed { (960, 700) } else { (480, 350) }; let factor = analysis_factor(frame.img.width, frame.img.height, max_side); let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 3); + self.trajectories.push(frame); let mut fm = FrameMotion::default(); if let Some(prev) = &self.prev && prev.width() == pyr.width() @@ -184,7 +200,17 @@ impl Analyzer { } pub fn finish(self, start: f64, frame_duration: f64) -> WarpAnalysis { - WarpAnalysis { version: 1, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames } + let mut tracks = self.trajectories.finish(start, frame_duration).tracks; + // Short trajectories hardly constrain the subspace: keep those of at least 6 frames, at + // 0.1 px (smaller JSON). + tracks.retain(|t| t.pts.len() >= 6); + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 10.0).round() / 10.0, (p[1] * 10.0).round() / 10.0]; + } + } + WarpAnalysis { version: 2, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames, tracks } } } @@ -205,11 +231,31 @@ pub enum Method { /// Position, Scale, Rotation. Similarity, Perspective, - /// Approximated by Perspective (see the module docs of the effect). + /// Content-preserving mesh warps fitted to subspace-smoothed trajectories. #[default] SubspaceWarp, } +/// Advanced ▸ Rolling Shutter Ripple: how freely Subspace Warp's mesh may bend row by row (a +/// rolling-shutter camera exposes rows at different times, so shake wobbles the frame). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Ripple { + /// A 12 × 8 mesh. + #[default] + Automatic, + /// Twice the rows and a softer shape term: follows row-wise wobble more closely. + Enhanced, +} + +impl Ripple { + pub fn mesh(self) -> MeshOpts { + match self { + Ripple::Automatic => MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 }, + Ripple::Enhanced => MeshOpts { cols: 12, rows: 16, alpha: 0.5, prior: 0.02 }, + } + } +} + /// Borders ▸ Framing. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub enum Framing { @@ -239,6 +285,7 @@ pub struct StabSettings { pub crop_less_smooth_more: f64, /// Frames per second (for Smoothness). pub fps: f64, + pub ripple: Ripple, } impl Default for StabSettings { @@ -254,6 +301,7 @@ impl Default for StabSettings { additional_scale: 100.0, crop_less_smooth_more: 0.0, fps: 30.0, + ripple: Ripple::Automatic, } } } @@ -269,8 +317,15 @@ impl StabSettings { /// The per-frame stabilization of a clip. #[derive(Clone, Debug, Default)] pub struct Plan { - /// Final source → output transform per frame (layer pixels), framing scale included. + /// Final source → output transform per frame (layer pixels), framing scale included. For + /// mesh warps, the closest homography (for overlays and edge synthesis). pub warps: Vec, + /// Subspace Warp: per frame, the mesh (stabilised pixels before framing → source pixels); + /// empty for the other methods. + pub meshes: Vec, + /// The framing transform (auto-scale and Additional Scale about the centre) after the + /// stabilisation. + pub framing: Homography, /// The visible region in output pixels `[x0, y0, x1, y1]` (Stabilize, Crop / Auto-scale); /// `None` = everything the warp produces. pub crop: Option<[f64; 4]>, @@ -280,6 +335,24 @@ pub struct Plan { pub valid_fraction: f64, } +impl Plan { + /// The source position (frame `k`'s layer pixels) shown at output point `p`. + pub fn source_of(&self, k: usize, p: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + let inv = self.framing.inverse()?; + return Some(m.sample(inv.apply(p))); + } + Some(self.warps.get(k)?.inverse()?.apply(p)) + } + /// Where source point `q` of frame `k` lands in the output. + pub fn output_of(&self, k: usize, q: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + return m.forward(q).map(|p| self.framing.apply(p)); + } + Some(self.warps.get(k)?.apply(q)) + } +} + fn convex_contains(q: &[[f64; 2]; 4], p: [f64; 2]) -> bool { let mut sign = 0.0; for i in 0..4 { @@ -418,14 +491,100 @@ pub fn corrections(a: &WarpAnalysis, s: &StabSettings) -> Vec { out } +/// One frame's stabilisation before framing. +#[derive(Clone, Debug)] +enum Warp { + H(Homography), + M(Mesh), +} + +impl Warp { + fn valid(&self, size: [f64; 2]) -> f64 { + match self { + Warp::H(h) => valid_fraction(h, size), + Warp::M(m) => m.valid_fraction(), + } + } + /// Blend towards the identity (`l` = 1 keeps the warp). + fn relax(&self, l: f64) -> Warp { + match self { + Warp::H(h) => Warp::H(Homography::IDENTITY.lerp(h, l)), + Warp::M(m) => Warp::M(m.lerp_identity(l)), + } + } + fn homography(&self) -> Homography { + match self { + Warp::H(h) => *h, + Warp::M(m) => m.homography(), + } + } +} + +/// Remove the uniform scale of the correspondences' best similarity (Preserve Scale): the +/// wanted positions are scaled back about their centroid. +fn unscale(pairs: &mut [([f64; 2], [f64; 2])]) { + let (src, dst): (Vec<[f64; 2]>, Vec<[f64; 2]>) = pairs.iter().copied().unzip(); + let Some(h) = least_squares(Model::Similarity, &src, &dst) else { return }; + let m = h.0; + let k = (m[0][0] * m[1][1] - m[0][1] * m[1][0]).abs().sqrt(); + if k < 1e-6 { + return; + } + let n = dst.len() as f64; + let c = [dst.iter().map(|q| q[0]).sum::() / n, dst.iter().map(|q| q[1]).sum::() / n]; + for (_, d) in pairs.iter_mut() { + *d = [c[0] + (d[0] - c[0]) / k, c[1] + (d[1] - c[1]) / k]; + } +} + +/// Subspace Warp's meshes (stabilised → source, before framing) for every frame; `None` for the +/// other methods and for analyses without trajectories. `base`: the perspective corrections +/// ([`corrections`]), the meshes' prior. +pub fn meshes(a: &WarpAnalysis, s: &StabSettings, base: &[Homography]) -> Option> { + if s.method != Method::SubspaceWarp || a.tracks.is_empty() || base.len() != a.frames.len() { + return None; + } + let n = a.frames.len(); + let o = s.ripple.mesh(); + let size = a.size; + let pairs: Vec> = match s.result { + StabResult::NoMotion => (0..n).map(|t| a.tracks.iter().filter_map(|tr| tr.at(t as u32)).map(|p| (p, base[t].apply(p))).collect()).collect(), + StabResult::SmoothMotion => { + let sigma = s.sigma(); + if sigma < 0.3 { + return Some((0..n).map(|_| Mesh::identity(o.cols, o.rows, size)).collect()); + } + smooth_trajectories(&a.tracks, n, sigma, 9) + } + }; + use rayon::prelude::*; + Some( + pairs + .into_par_iter() + .enumerate() + .map(|(t, mut pr)| { + if s.preserve_scale && pr.len() >= 2 { + unscale(&mut pr); + } + if pr.len() < 6 { Mesh::from_homography(o.cols, o.rows, size, &base[t]) } else { solve_mesh(size, &o, &pr, &base[t]) } + }) + .collect(), + ) +} + /// Stabilization plan: corrections plus framing. pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { - let mut b = corrections(a, s); + use rayon::prelude::*; + let base = corrections(a, s); + let mut b: Vec = match meshes(a, s, &base) { + Some(m) => m.into_iter().map(Warp::M).collect(), + None => base.into_iter().map(Warp::H).collect(), + }; let size = a.size; let c = [size[0] / 2.0, size[1] / 2.0]; let add = (s.additional_scale / 100.0).max(0.01); let z_add = Homography::scale_about(add, c); - let fracs = |b: &[Homography]| b.iter().map(|h| valid_fraction(h, size)).collect::>(); + let fracs = |b: &[Warp]| b.par_iter().map(|w| w.valid(size)).collect::>(); let mut fr = fracs(&b); let mut vmin = fr.iter().copied().fold(1.0, f64::min); let margin = (s.action_safe / 100.0).clamp(0.0, 0.45); @@ -435,22 +594,24 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { // Relax the correction on frames that need more than Maximum Scale. let target = need_cover / max_scale; let n = b.len(); - let mut lam = vec![1.0f64; n]; - for t in 0..n { - if fr[t] >= target { - continue; - } - let (mut lo, mut hi) = (0.0, 1.0); - for _ in 0..24 { - let mid = 0.5 * (lo + hi); - if valid_fraction(&Homography::IDENTITY.lerp(&b[t], mid), size) >= target { - lo = mid; - } else { - hi = mid; + let lam: Vec = (0..n) + .into_par_iter() + .map(|t| { + if fr[t] >= target { + return 1.0; } - } - lam[t] = lo; - } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if b[t].relax(mid).valid(size) >= target { + lo = mid; + } else { + hi = mid; + } + } + lo + }) + .collect(); // Spread the relaxation smoothly (min filter, then a Gaussian of the same radius) so // the camera doesn't jump. let r = ((s.sigma() * 0.5).ceil() as usize).max(1); @@ -468,7 +629,7 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { (acc / ws).min(lam[t]) }) .collect(); - b = b.iter().zip(&smooth).map(|(h, l)| if *l >= 1.0 { *h } else { Homography::IDENTITY.lerp(h, *l) }).collect(); + b = b.iter().zip(&smooth).map(|(w, l)| if *l >= 1.0 { w.clone() } else { w.relax(*l) }).collect(); fr = fracs(&b); vmin = fr.iter().copied().fold(1.0, f64::min); } @@ -483,13 +644,23 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { } }; let z = Homography::scale_about(auto, c); - let warps = b.iter().map(|h| z_add.then_after(&z.then_after(h))).collect(); - Plan { warps, crop, auto_scale: auto, valid_fraction: vmin } + let framing = z_add.then_after(&z); + let warps = b.par_iter().map(|w| framing.then_after(&w.homography())).collect(); + let meshes = b + .into_iter() + .filter_map(|w| match w { + Warp::M(m) => Some(m), + Warp::H(_) => None, + }) + .collect(); + Plan { warps, meshes, framing, crop, auto_scale: auto, valid_fraction: vmin } } #[cfg(test)] mod tests { use super::*; + use effectcraft_raster::Image; + use rayon::prelude::*; /// A synthetic analysis: a slow pan plus per-frame jitter. fn shaky(n: usize) -> (WarpAnalysis, Vec<[f64; 2]>) { @@ -497,7 +668,7 @@ mod tests { let j = [((k * 7919) % 13) as f64 - 6.0, ((k * 104729) % 11) as f64 - 5.0]; [2.0 * k as f64 + j[0], 0.5 * k as f64 + j[1]] }; - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![], tracks: vec![] }; let path: Vec<[f64; 2]> = (0..n).map(pos).collect(); for k in 0..n { let mut f = FrameMotion::default(); @@ -573,4 +744,139 @@ mod tests { let j = WarpAnalysis::from_json(&a.to_json()).unwrap(); assert_eq!(j, a); } + + // ------------------------------------------------------------ synthetic shaky footage + + fn hash(i: i64, j: i64, k: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663) ^ k.wrapping_mul(83_492_791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + + fn value_noise(x: f64, y: f64, cell: f64, seed: i64) -> f64 { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (u, v) = (fx - i as f64, fy - j as f64); + let (u, v) = (u * u * (3.0 - 2.0 * u), v * v * (3.0 - 2.0 * v)); + let a = hash(i, j, seed) * (1.0 - u) + hash(i + 1, j, seed) * u; + let b = hash(i, j + 1, seed) * (1.0 - u) + hash(i + 1, j + 1, seed) * u; + a * (1.0 - v) + b * v + } + + fn texture(x: f64, y: f64) -> f32 { + (0.55 * value_noise(x, y, 5.0, 1) + 0.45 * value_noise(x, y, 13.0, 2)) as f32 + } + + /// A scene with depth: a near bump in the middle moves `1 + depth` times as much as the + /// background when the camera shakes (parallax a homography cannot model). + struct Scene { + size: [f64; 2], + cams: Vec<[f64; 2]>, + } + + impl Scene { + fn new(n: usize) -> Scene { + let cams = (0..n) + .map(|t| { + let j = [5.0 * (hash(t as i64, 7, 3) - 0.5), 5.0 * (hash(t as i64, 11, 5) - 0.5)]; + [0.4 * t as f64 + j[0], 0.15 * t as f64 + j[1]] + }) + .collect(); + Scene { size: [320.0, 180.0], cams } + } + fn depth(&self, s: [f64; 2]) -> f64 { + 1.0 + 1.2 * (-((s[0] - 160.0).powi(2) + (s[1] - 90.0).powi(2)) / (2.0 * 38.0f64.powi(2))).exp() + } + /// Where scene point `s` appears on frame `t`. + fn image_of(&self, t: usize, s: [f64; 2]) -> [f64; 2] { + let (c, d) = (self.cams[t], self.depth(s)); + [s[0] + c[0] * d, s[1] + c[1] * d] + } + fn frame(&self, t: usize) -> Image { + let (w, h) = (self.size[0] as u32, self.size[1] as u32); + let mut im = Image::new(w, h); + let c = self.cams[t]; + im.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let mut s = p; + for _ in 0..10 { + let d = self.depth(s); + s = [p[0] - c[0] * d, p[1] - c[1] * d]; + } + let v = texture(s[0], s[1]); + *px = [v, v, v, 1.0]; + } + }); + im + } + } + + fn residual_jitter(scene: &Scene, out: impl Fn(usize, [f64; 2]) -> Option<[f64; 2]>) -> f64 { + let n = scene.cams.len(); + let mut tot = 0.0; + let mut cnt = 0; + for gy in 0..5 { + for gx in 0..7 { + let s = [70.0 + gx as f64 * 30.0, 45.0 + gy as f64 * 22.0]; + let ys: Vec> = (0..n).map(|t| out(t, scene.image_of(t, s))).collect(); + for t in 6..n - 6 { + if let (Some(a), Some(b), Some(c)) = (ys[t - 1], ys[t], ys[t + 1]) { + tot += (c[0] - 2.0 * b[0] + a[0]).hypot(c[1] - 2.0 * b[1] + a[1]); + cnt += 1; + } + } + } + } + tot / cnt.max(1) as f64 + } + + #[test] + fn subspace_warp_beats_perspective_on_parallax() { + let n = 48; + let scene = Scene::new(n); + let mut an = Analyzer::new(scene.size, AnalyzeOpts::default()); + for t in 0..n { + let img = scene.frame(t); + an.push(&Frame { img: &img, offset: [0.0; 2] }); + } + let a = an.finish(0.0, 1.0 / 24.0); + assert!(a.tracks.len() >= 40, "trajectories: {}", a.tracks.len()); + // The analysis round-trips with its trajectories. + assert_eq!(WarpAnalysis::from_json(&a.to_json()).unwrap(), a); + let base = StabSettings { framing: Framing::StabilizeOnly, fps: 24.0, ..Default::default() }; + let before = residual_jitter(&scene, |_, x| Some(x)); + let ppl = plan(&a, &StabSettings { method: Method::Perspective, ..base }); + assert!(ppl.meshes.is_empty()); + let persp = residual_jitter(&scene, |t, x| ppl.output_of(t, x)); + let spl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ..base }); + assert_eq!(spl.meshes.len(), n); + let sub = residual_jitter(&scene, |t, x| spl.output_of(t, x)); + let epl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ripple: Ripple::Enhanced, ..base }); + let enh = residual_jitter(&scene, |t, x| epl.output_of(t, x)); + eprintln!("jitter: before {before:.3}, perspective {persp:.3}, subspace {sub:.3}, enhanced {enh:.3}"); + assert!(persp < 0.8 * before, "perspective {persp} vs {before}"); + assert!(sub < 0.6 * persp, "subspace {sub} vs perspective {persp}"); + assert!(enh < 0.6 * persp, "enhanced {enh} vs perspective {persp}"); + // source_of and output_of agree. + let q = [150.0, 80.0]; + let o = spl.output_of(10, q).unwrap(); + let back = spl.source_of(10, o).unwrap(); + assert!((back[0] - q[0]).abs() < 1e-3 && (back[1] - q[1]).abs() < 1e-3); + // Auto-scale framing covers the frame with mesh warps too. + let fpl = plan(&a, &StabSettings { method: Method::SubspaceWarp, fps: 24.0, ..Default::default() }); + assert!(fpl.auto_scale >= 1.0); + for k in [0, n / 2, n - 1] { + for p in [[1.0, 1.0], [319.0, 1.0], [319.0, 179.0], [1.0, 179.0]] { + let q = fpl.source_of(k, p).unwrap(); + let inside = q[0] >= -0.5 && q[1] >= -0.5 && q[0] <= 320.5 && q[1] <= 180.5; + assert!(inside || fpl.crop.is_some(), "frame {k}: {p:?} → {q:?}"); + } + } + // Preserve Scale keeps working with meshes. + let psp = plan(&a, &StabSettings { method: Method::SubspaceWarp, preserve_scale: true, ..base }); + assert_eq!(psp.meshes.len(), n); + } } diff --git a/crates/track/src/subspace.rs b/crates/track/src/subspace.rs new file mode 100644 index 00000000..339f9708 --- /dev/null +++ b/crates/track/src/subspace.rs @@ -0,0 +1,447 @@ +//! Warp Stabilizer ▸ Method ▸ **Subspace Warp**: content-preserving warps fitted to smoothed +//! feature trajectories, clean-room from the published method: +//! +//! - Liu, Gleicher, Wang, Jin & Agarwala, "Subspace Video Stabilization", ACM TOG 30(1), 2011: +//! the feature trajectories of a short window form a matrix `M` (two rows per trajectory, one +//! column per frame) that is close to low rank (rank 9 for a moving camera over a static +//! scene); its factorisation `M ≈ W C` gives *eigen-trajectories* `C`, which are smoothed (here +//! with a Gaussian) instead of the trajectories themselves. Every trajectory's smoothed +//! position is `W_i Ĉ_t`; trajectories that do not span the whole window get their +//! coefficients `W_i` by least squares over the frames they cover (the paper's handling of +//! incomplete trajectories in a moving factorisation). Smoothing in the subspace keeps the +//! smoothed trajectories consistent with one another, so the warp does not tear the scene. +//! - Liu, Gleicher, Jin & Agarwala, "Content-Preserving Warps for 3D Video Stabilization", ACM +//! TOG 28(3), 2009: each frame is warped by a grid mesh fitted to the feature correspondences +//! (original → smoothed position) in least squares, with a **similarity term** asking every +//! grid triangle to keep its shape (`V1 = V2 + u (V3 − V2) + v R90 (V3 − V2)` with `u, v` from +//! the undeformed grid) so the warp bends where the scene's depth demands it but stays locally +//! rigid elsewhere. +//! +//! The mesh is stored *inversely*: a regular grid over the stabilised frame whose vertices hold +//! the source position they show ([`Mesh`]), so rendering is a bilinear lookup per output pixel. +//! The data term is the same linear constraint read the other way (the smoothed position's grid +//! cell, bilinearly, must land on the original position). A weak prior ties every vertex to the +//! frame's perspective correction, which fills feature-less regions and keeps the system well +//! posed. + +use rayon::prelude::*; + +use crate::camtrack::Track2D; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; +use crate::fit::{Model, least_squares}; +use crate::solve::Homography; + +/// A warp mesh: a regular `cols × rows` grid over the stabilised frame (`size`), each vertex +/// holding the source-frame position it shows. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub cols: usize, + pub rows: usize, + pub size: [f64; 2], + /// `(rows + 1) × (cols + 1)` source positions, row-major. + pub verts: Vec<[f64; 2]>, +} + +impl Mesh { + /// The identity mesh. + pub fn identity(cols: usize, rows: usize, size: [f64; 2]) -> Mesh { + let mut m = Mesh { cols: cols.max(1), rows: rows.max(1), size, verts: vec![] }; + m.verts = (0..=m.rows).flat_map(|j| (0..=m.cols).map(move |i| (i, j))).map(|(i, j)| m.grid(i, j)).collect(); + m + } + + /// The mesh of a homography (`h`: source → stabilised). + pub fn from_homography(cols: usize, rows: usize, size: [f64; 2], h: &Homography) -> Mesh { + let inv = h.inverse().unwrap_or(Homography::IDENTITY); + let mut m = Mesh::identity(cols, rows, size); + for v in m.verts.iter_mut() { + *v = inv.apply(*v); + } + m + } + + fn cell(&self) -> [f64; 2] { + [self.size[0] / self.cols as f64, self.size[1] / self.rows as f64] + } + + /// The undeformed position of vertex `(i, j)` (column, row). + pub fn grid(&self, i: usize, j: usize) -> [f64; 2] { + let c = self.cell(); + [i as f64 * c[0], j as f64 * c[1]] + } + + fn idx(&self, i: usize, j: usize) -> usize { + j * (self.cols + 1) + i + } + + /// Cell and bilinear coordinates of a stabilised point (the outer cells extend beyond the + /// frame, so points outside it extrapolate). + fn locate(&self, p: [f64; 2]) -> (usize, usize, f64, f64) { + let c = self.cell(); + let fx = p[0] / c[0]; + let fy = p[1] / c[1]; + let i = (fx.floor().max(0.0) as usize).min(self.cols - 1); + let j = (fy.floor().max(0.0) as usize).min(self.rows - 1); + (i, j, fx - i as f64, fy - j as f64) + } + + /// The source position shown at stabilised point `p`. + pub fn sample(&self, p: [f64; 2]) -> [f64; 2] { + let (i, j, u, v) = self.locate(p); + let a = self.verts[self.idx(i, j)]; + let b = self.verts[self.idx(i + 1, j)]; + let c = self.verts[self.idx(i, j + 1)]; + let d = self.verts[self.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + [0, 1].map(|k| w[0] * a[k] + w[1] * b[k] + w[2] * c[k] + w[3] * d[k]) + } + + /// Where source point `q` lands in the stabilised frame (Newton iterations on + /// [`Mesh::sample`]); `None` when it does not converge (a folded mesh). + pub fn forward(&self, q: [f64; 2]) -> Option<[f64; 2]> { + // Start from the mesh's own offset at q. + let s = self.sample(q); + let mut p = [2.0 * q[0] - s[0], 2.0 * q[1] - s[1]]; + for _ in 0..30 { + let f = self.sample(p); + let e = [f[0] - q[0], f[1] - q[1]]; + if e[0].hypot(e[1]) < 1e-6 { + return Some(p); + } + let h = 0.5; + let fx = self.sample([p[0] + h, p[1]]); + let fy = self.sample([p[0], p[1] + h]); + let j = [[(fx[0] - f[0]) / h, (fy[0] - f[0]) / h], [(fx[1] - f[1]) / h, (fy[1] - f[1]) / h]]; + let det = j[0][0] * j[1][1] - j[0][1] * j[1][0]; + if det.abs() < 1e-12 { + return None; + } + p[0] -= (j[1][1] * e[0] - j[0][1] * e[1]) / det; + p[1] -= (-j[1][0] * e[0] + j[0][0] * e[1]) / det; + } + let f = self.sample(p); + ((f[0] - q[0]).hypot(f[1] - q[1]) < 1e-3).then_some(p) + } + + /// `(1 − t) · identity + t · self` per vertex. + pub fn lerp_identity(&self, t: f64) -> Mesh { + let mut m = self.clone(); + for j in 0..=self.rows { + for i in 0..=self.cols { + let g = self.grid(i, j); + let k = self.idx(i, j); + m.verts[k] = [g[0] + (self.verts[k][0] - g[0]) * t, g[1] + (self.verts[k][1] - g[1]) * t]; + } + } + m + } + + /// Largest `s ∈ [0, 1]` such that the centred `s·W × s·H` rectangle of the stabilised frame + /// shows only source pixels (its outline, sampled densely, maps inside the source frame). + pub fn valid_fraction(&self) -> f64 { + let [w, h] = self.size; + let c = [w / 2.0, h / 2.0]; + let tol = 1e-6; + let inside = |q: [f64; 2]| q[0] >= -tol && q[1] >= -tol && q[0] <= w + tol && q[1] <= h + tol; + let fits = |s: f64| { + let (hw, hh) = (s * w / 2.0, s * h / 2.0); + let n = 48; + (0..=n).all(|k| { + let t = k as f64 / n as f64; + let x = c[0] - hw + 2.0 * hw * t; + let y = c[1] - hh + 2.0 * hh * t; + inside(self.sample([x, c[1] - hh])) + && inside(self.sample([x, c[1] + hh])) + && inside(self.sample([c[0] - hw, y])) + && inside(self.sample([c[0] + hw, y])) + }) + }; + if fits(1.0) { + return 1.0; + } + if !fits(0.0) { + return 0.0; + } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if fits(mid) { + lo = mid; + } else { + hi = mid; + } + } + lo + } + + /// The homography (source → stabilised) closest to the mesh in least squares. + pub fn homography(&self) -> Homography { + let mut src = vec![]; + let mut dst = vec![]; + for j in 0..=self.rows { + for i in 0..=self.cols { + src.push(self.verts[self.idx(i, j)]); + dst.push(self.grid(i, j)); + } + } + least_squares(Model::Homography, &src, &dst).unwrap_or(Homography::IDENTITY) + } +} + +/// Mesh resolution and rigidity. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MeshOpts { + pub cols: usize, + pub rows: usize, + /// Weight of the similarity (shape-preserving) term relative to one feature. + pub alpha: f64, + /// Weight of the prior towards the frame's perspective correction, per vertex. + pub prior: f64, +} + +impl Default for MeshOpts { + fn default() -> Self { + MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 } + } +} + +/// Normal equations `AᵀA x = Aᵀb`, accumulated row by row. +struct Normal { + n: usize, + a: Vec, + b: Vec, +} + +impl Normal { + fn row(&mut self, coefs: &[(usize, f64)], rhs: f64, w: f64) { + for (ia, ca) in coefs { + self.b[*ia] += w * ca * rhs; + for (ib, cb) in coefs { + self.a[ia * self.n + ib] += w * ca * cb; + } + } + } +} + +/// Fit the content-preserving warp of one frame: `pairs` are (source position, wanted +/// stabilised position) of its features; `base` is the frame's perspective correction (source → +/// stabilised), the prior. +pub fn solve_mesh(size: [f64; 2], o: &MeshOpts, pairs: &[([f64; 2], [f64; 2])], base: &Homography) -> Mesh { + let mut m = Mesh::from_homography(o.cols, o.rows, size, base); + let nv = (m.cols + 1) * (m.rows + 1); + let n = 2 * nv; + let mut ne = Normal { n, a: vec![0.0; n * n], b: vec![0.0; n] }; + // Prior. + for k in 0..nv { + let v = m.verts[k]; + ne.row(&[(2 * k, 1.0)], v[0], o.prior); + ne.row(&[(2 * k + 1, 1.0)], v[1], o.prior); + } + // Data: the bilinear combination of the wanted position's cell lands on the source position. + let mut used = 0; + for (src, dst) in pairs { + if !(dst[0] >= 0.0 && dst[1] >= 0.0 && dst[0] <= size[0] && dst[1] <= size[1] && src[0].is_finite() && src[1].is_finite()) { + continue; + } + let (i, j, u, v) = m.locate(*dst); + let ks = [m.idx(i, j), m.idx(i + 1, j), m.idx(i, j + 1), m.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + for d in 0..2 { + let coefs: Vec<(usize, f64)> = ks.iter().zip(w).map(|(k, w)| (2 * k + d, w)).collect(); + ne.row(&coefs, src[d], 1.0); + } + used += 1; + } + if used == 0 { + return m; + } + // Similarity: four triangles per cell, each corner against its two cell neighbours. + let a2 = o.alpha * o.alpha; + for j in 0..m.rows { + for i in 0..m.cols { + let (a, b, c, d) = ((i, j), (i + 1, j), (i + 1, j + 1), (i, j + 1)); + for (v1, v2, v3) in [(a, b, d), (b, c, a), (c, d, b), (d, a, c)] { + let (g1, g2, g3) = (m.grid(v1.0, v1.1), m.grid(v2.0, v2.1), m.grid(v3.0, v3.1)); + let dv = [g3[0] - g2[0], g3[1] - g2[1]]; + let e = [g1[0] - g2[0], g1[1] - g2[1]]; + let l2 = dv[0] * dv[0] + dv[1] * dv[1]; + // R90(d) = (−d.y, d.x). + let u = (e[0] * dv[0] + e[1] * dv[1]) / l2; + let v = (e[0] * -dv[1] + e[1] * dv[0]) / l2; + let (k1, k2, k3) = (m.idx(v1.0, v1.1), m.idx(v2.0, v2.1), m.idx(v3.0, v3.1)); + // x: V1x − V2x − u (V3x − V2x) + v (V3y − V2y) = 0 + ne.row(&[(2 * k1, 1.0), (2 * k2, u - 1.0), (2 * k3, -u), (2 * k3 + 1, v), (2 * k2 + 1, -v)], 0.0, a2); + // y: V1y − V2y − u (V3y − V2y) − v (V3x − V2x) = 0 + ne.row(&[(2 * k1 + 1, 1.0), (2 * k2 + 1, u - 1.0), (2 * k3 + 1, -u), (2 * k3, -v), (2 * k2, v)], 0.0, a2); + } + } + } + let mut x = ne.b; + if cholesky_solve(&mut ne.a, n, &mut x) { + for k in 0..nv { + m.verts[k] = [x[2 * k], x[2 * k + 1]]; + } + } + m +} + +/// Gaussian weights of the frames `lo..=hi` around `t`. +fn gauss(lo: usize, hi: usize, t: usize, sigma: f64) -> Vec { + let g: Vec = (lo..=hi).map(|j| (-((j as f64 - t as f64).powi(2)) / (2.0 * sigma * sigma)).exp()).collect(); + let s: f64 = g.iter().sum(); + g.into_iter().map(|v| v / s.max(1e-300)).collect() +} + +/// Subspace-smoothed feature positions: for every frame `t < frames`, the (original, smoothed) +/// positions of the trajectories seen on it. `sigma`: Gaussian smoothing in frames; `rank`: the +/// subspace dimension (9 in the paper). Frames without enough trajectories get no pairs. +pub fn smooth_trajectories(tracks: &[Track2D], frames: usize, sigma: f64, rank: usize) -> Vec> { + let max_r = ((3.0 * sigma).ceil() as usize).clamp(1, 30); + (0..frames) + .into_par_iter() + .map(|t| { + let t32 = t as u32; + let mut r = max_r; + // The window around t: shrink it until enough trajectories span it. + let (lo, hi, complete) = loop { + let lo = t.saturating_sub(r); + let hi = (t + r).min(frames.saturating_sub(1)); + let complete: Vec<&Track2D> = tracks.iter().filter(|tr| tr.start as usize <= lo && tr.end() as usize > hi).collect(); + if complete.len() >= 8 || r <= 2 { + break (lo, hi, complete); + } + r = (r * 2 / 3).max(2); + }; + if complete.len() < 4 { + return vec![]; + } + let m = hi - lo + 1; + let rows = 2 * complete.len(); + let k = rank.min(m).min(rows).max(1); + // M: rows × m (row-major). + let mut mat = vec![0.0; rows * m]; + for (i, tr) in complete.iter().enumerate() { + for c in 0..m { + let p = tr.at((lo + c) as u32).unwrap_or([0.0; 2]); + mat[(2 * i) * m + c] = p[0]; + mat[(2 * i + 1) * m + c] = p[1]; + } + } + // Gram MᵀM and its top eigenvectors: the eigen-trajectories (orthonormal rows of C). + let mut gram = vec![0.0; m * m]; + for a in 0..m { + for b in a..m { + let s: f64 = (0..rows).map(|r| mat[r * m + a] * mat[r * m + b]).sum(); + gram[a * m + b] = s; + gram[b * m + a] = s; + } + } + let (_, vecs) = sym_eigen(&gram, m); + // C[q][c] = vecs[c][m − 1 − q] (largest first). + let cmat: Vec> = (0..k).map(|q| (0..m).map(|c| vecs[c * m + (m - 1 - q)]).collect()).collect(); + let g = gauss(lo, hi, t, sigma.max(1e-6)); + let chat: Vec = cmat.iter().map(|row| row.iter().zip(&g).map(|(a, b)| a * b).sum()).collect(); + let tc = t - lo; + let mut out = Vec::new(); + // Complete trajectories: W = M Cᵀ. + for (i, _) in complete.iter().enumerate() { + let src = [mat[(2 * i) * m + tc], mat[(2 * i + 1) * m + tc]]; + let mut dst = [0.0; 2]; + for (d, slot) in dst.iter_mut().enumerate() { + let row = &mat[(2 * i + d) * m..(2 * i + d + 1) * m]; + for q in 0..k { + let w: f64 = row.iter().zip(&cmat[q]).map(|(a, b)| a * b).sum(); + *slot += w * chat[q]; + } + } + out.push((src, dst)); + } + // Incomplete trajectories seen on t: coefficients by least squares over the frames + // they cover in the window. + let need = (k + 3).max(m / 2); + for tr in tracks.iter().filter(|tr| tr.at(t32).is_some() && !(tr.start as usize <= lo && tr.end() as usize > hi)) { + let cols: Vec = (0..m).filter(|c| tr.at((lo + c) as u32).is_some()).collect(); + if cols.len() < need { + continue; + } + let mut ata = vec![0.0; k * k]; + let mut atb = [vec![0.0; k], vec![0.0; k]]; + for &c in &cols { + let p = tr.at((lo + c) as u32).expect("covered"); + for a in 0..k { + for d in 0..2 { + atb[d][a] += cmat[a][c] * p[d]; + } + for b in 0..k { + ata[a * k + b] += cmat[a][c] * cmat[b][c]; + } + } + } + for a in 0..k { + ata[a * k + a] += 1e-9; + } + let mut dst = [0.0; 2]; + let mut ok = true; + for d in 0..2 { + let mut x = atb[d].clone(); + let mut aa = ata.clone(); + if !cholesky_solve(&mut aa, k, &mut x) { + ok = false; + break; + } + dst[d] = x.iter().zip(&chat).map(|(a, b)| a * b).sum(); + } + if ok { + out.push((tr.at(t32).expect("seen"), dst)); + } + } + out + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mesh_identity_sample_and_forward() { + let m = Mesh::identity(4, 3, [100.0, 60.0]); + assert_eq!(m.sample([37.0, 21.0]), [37.0, 21.0]); + assert_eq!(m.valid_fraction(), 1.0); + let h = Homography([[1.0, 0.02, 5.0], [-0.01, 1.0, -3.0], [0.0, 0.0, 1.0]]); + let mh = Mesh::from_homography(4, 3, [100.0, 60.0], &h); + // Affine maps are exact on a bilinear mesh. + let q = [40.0, 30.0]; + let p = mh.forward(q).unwrap(); + let e = h.apply(q); + assert!((p[0] - e[0]).abs() < 1e-6 && (p[1] - e[1]).abs() < 1e-6, "{p:?} {e:?}"); + assert!(mh.homography().max_diff(&h) < 1e-6); + assert!(mh.valid_fraction() < 1.0); + } + + #[test] + fn mesh_fits_a_translation_and_bends_for_local_motion() { + let size = [200.0, 120.0]; + let mut pairs = vec![]; + for y in 0..12 { + for x in 0..20 { + let p = [5.0 + x as f64 * 10.0, 5.0 + y as f64 * 10.0]; + // Everything moves by (3, −2); a blob near the centre by (8, −2). + let bump = (-((p[0] - 100.0).powi(2) + (p[1] - 60.0).powi(2)) / (2.0 * 20.0f64.powi(2))).exp(); + pairs.push((p, [p[0] + 3.0 + 5.0 * bump, p[1] - 2.0])); + } + } + let base = Homography::translation([3.0, -2.0]); + let m = solve_mesh(size, &MeshOpts { cols: 10, rows: 6, alpha: 0.5, prior: 0.01 }, &pairs, &base); + let mut err_mesh = 0.0; + let mut err_h = 0.0; + for (s, d) in &pairs { + let q = m.sample(*d); + err_mesh += (q[0] - s[0]).hypot(q[1] - s[1]); + let qh = base.inverse().unwrap().apply(*d); + err_h += (qh[0] - s[0]).hypot(qh[1] - s[1]); + } + assert!(err_mesh < 0.35 * err_h, "{err_mesh} vs {err_h}"); + } +} From da0ec526f3736c895d4e2cfdf261d314bcf33859 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:12:21 -0400 Subject: [PATCH 23/36] M13.2: PDF / Illustrator (.ai) / EPS vector footage (effectcraft-pdf): continuous rasterisation, layered composition import, Create Shapes from Vector Layer; SVG render tree clip groups Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 12 + Cargo.toml | 1 + crates/engine/Cargo.toml | 1 + crates/engine/src/commands/create.rs | 8 +- crates/engine/src/commands/file.rs | 33 +- crates/engine/src/psd_import.rs | 2 +- crates/engine/src/vector.rs | 73 +- crates/host/Cargo.toml | 1 + crates/host/tests/vector_import.rs | 113 ++ crates/media/Cargo.toml | 1 + crates/media/src/layered.rs | 46 +- crates/media/src/lib.rs | 3 +- crates/media/src/pool.rs | 2 +- crates/pdf/Cargo.toml | 15 + crates/pdf/README.md | 48 + crates/pdf/src/build.rs | 93 ++ crates/pdf/src/color.rs | 288 +++++ crates/pdf/src/lib.rs | 185 ++++ crates/pdf/src/object.rs | 708 +++++++++++++ crates/pdf/src/page.rs | 573 ++++++++++ crates/pdf/src/ps.rs | 1462 ++++++++++++++++++++++++++ crates/pdf/src/tests.rs | 150 +++ crates/pdf/src/write.rs | 32 + crates/project/src/lib.rs | 11 +- crates/render/src/lib.rs | 2 +- crates/svg/src/lib.rs | 15 +- crates/svg/src/raster.rs | 21 + crates/ui-egui/src/menus.rs | 2 +- xtask/src/main.rs | 2 + 29 files changed, 3875 insertions(+), 28 deletions(-) create mode 100644 crates/pdf/Cargo.toml create mode 100644 crates/pdf/README.md create mode 100644 crates/pdf/src/build.rs create mode 100644 crates/pdf/src/color.rs create mode 100644 crates/pdf/src/lib.rs create mode 100644 crates/pdf/src/object.rs create mode 100644 crates/pdf/src/page.rs create mode 100644 crates/pdf/src/ps.rs create mode 100644 crates/pdf/src/tests.rs create mode 100644 crates/pdf/src/write.rs diff --git a/Cargo.lock b/Cargo.lock index c7934100..87cd95d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ dependencies = [ "effectcraft-lottie", "effectcraft-model", "effectcraft-path", + "effectcraft-pdf", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", @@ -1309,6 +1310,7 @@ dependencies = [ "effectcraft-export", "effectcraft-expr", "effectcraft-media", + "effectcraft-pdf", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", @@ -1352,6 +1354,7 @@ name = "effectcraft-media" version = "0.1.0" dependencies = [ "effectcraft-model", + "effectcraft-pdf", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", @@ -1398,6 +1401,15 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "effectcraft-pdf" +version = "0.1.0" +dependencies = [ + "effectcraft-svg", + "kurbo", + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-project" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 22709f60..cda7c302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-pdf = { path = "crates/pdf" } effectcraft-psd = { path = "crates/psd" } effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 45508a71..7a7adb30 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -26,6 +26,7 @@ effectcraft-lottie = { workspace = true } effectcraft-psd = { workspace = true } kurbo = { workspace = true } effectcraft-svg = { workspace = true } +effectcraft-pdf = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/commands/create.rs b/crates/engine/src/commands/create.rs index 72cb2420..a1d22fb5 100644 --- a/crates/engine/src/commands/create.rs +++ b/crates/engine/src/commands/create.rs @@ -28,12 +28,14 @@ fn create(s: &mut Session, p: &Value) -> Result { }); match op { "shapesFromVector" => { - let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else { - skipped.push(format!("{}: not a vector (SVG) footage layer", src.name)); + let Some(f) = footage.filter(crate::vector::is_vector) else { + skipped.push(format!("{}: not a vector (SVG, PDF, AI or EPS) footage layer", src.name)); continue; }; let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; - let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?; + let doc = crate::vector::vector_doc(&f.path, &bytes, f.layer.as_ref()) + .unwrap_or_else(|| Err(format!("{}: not a vector file", f.path))) + .map_err(EngineError::Other)?; let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| { let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc); let id = l.id; diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 5a69a213..661169e0 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -101,6 +101,17 @@ fn import(s: &mut Session, p: &Value) -> Result { for path in paths { let bytes = match s.services.read_file(&path) { Ok(b) if effectcraft_psd::is_psd(&b) => b, + Ok(b) if effectcraft_pdf::sniff(&b).is_some() && !path.to_ascii_lowercase().ends_with(".svg") => { + // PDF / Illustrator / EPS: one layer per file layer. + match import_vector_comp(s, &path, &b) { + Ok((comp, items)) => { + out_comps.push(comp); + ids.extend(items); + } + Err(e) => errors.push(e.to_string()), + } + continue; + } _ => { rest.push(path); continue; @@ -153,7 +164,9 @@ fn import(s: &mut Session, p: &Value) -> Result { && f.codec == "PSD" { match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) { - Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }), + Some((index, name)) => { + f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false, embedded: None }) + } None => { errors.push(format!("{path}: no layer {sel}")); continue; @@ -217,6 +230,22 @@ fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Ok((r.comp.0, items, r.warnings)) } +/// Import a PDF / Illustrator / EPS file as a composition (one undo step). Returns (comp, items). +fn import_vector_comp(s: &mut Session, path: &str, bytes: &[u8]) -> Result<(u64, Vec)> { + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Vector".into()); + let rate = effectcraft_time::FrameRate::FPS_29_97; + let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 }; + let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs)); + let (comp, folder, items) = s.edit("Import", None, |proj, st| { + let r = crate::vector::import_vector_comp(proj, path, bytes, &name, rate, duration).map_err(EngineError::Other)?; + st.project_selection = vec![r.0]; + Ok(r) + })?; + let mut out = vec![folder.0]; + out.extend(items.iter().map(|i| i.0)); + Ok((comp.0, out)) +} + /// Extensions imported as data footage. pub const DATA_EXTENSIONS: &[&str] = &["json", "csv", "tsv"]; @@ -381,7 +410,7 @@ pub fn specs() -> Vec { "File...", ["File", "Import"], Some("Cmd+I"), - "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop, PDF, Illustrator and EPS files), layer?: name|index (footage of one Photoshop layer)}", always, import ), diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs index f64fd3af..2f012eee 100644 --- a/crates/engine/src/psd_import.rs +++ b/crates/engine/src/psd_import.rs @@ -156,7 +156,7 @@ fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> ( missing: false, sequence: vec![], color_profile: None, - layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }), + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty(), embedded: None }), ..Default::default() }; let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); diff --git a/crates/engine/src/vector.rs b/crates/engine/src/vector.rs index a3f3e040..3465d385 100644 --- a/crates/engine/src/vector.rs +++ b/crates/engine/src/vector.rs @@ -1,5 +1,6 @@ -//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage → -//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines). +//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG, PDF, +//! Illustrator and EPS footage → shape layer), Create Shapes from Text and Create Masks from +//! Text (glyph outlines), and vector files imported as compositions (one layer per file layer). use effectcraft_keyframe::{Gradient, ShapePath, Value}; use effectcraft_project::build::{self, Ids}; @@ -149,6 +150,74 @@ pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec { if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items } } +/// Whether footage is a vector file (SVG / PDF / AI / EPS). +pub fn is_vector(f: &effectcraft_project::Footage) -> bool { + effectcraft_render::is_vector_footage(f) || f.path.to_ascii_lowercase().ends_with(".svg") +} + +/// The vector document of an SVG / PDF / AI / EPS file (`None` for other formats), restricted +/// to one layer when `layer` names it. +pub fn vector_doc(path: &str, bytes: &[u8], layer: Option<&effectcraft_project::SourceLayer>) -> Option> { + if path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) { + return Some(effectcraft_svg::parse(bytes).map_err(|e| format!("{path}: {e}"))); + } + effectcraft_pdf::sniff(bytes)?; + Some( + effectcraft_pdf::parse(bytes) + .map(|d| match layer { + Some(l) => effectcraft_pdf::layer_doc(&d, l.index as usize), + None => d, + }) + .map_err(|e| format!("{path}: {e}")), + ) +} + +/// Import a PDF / Illustrator / EPS document as a composition the size of its page: one footage +/// item per file layer (each showing only that layer, continuously rasterisable), top layer +/// first. Returns (comp, folder, items). +pub fn import_vector_comp( + proj: &mut Project, + path: &str, + bytes: &[u8], + name: &str, + rate: effectcraft_time::FrameRate, + duration: effectcraft_time::Tick, +) -> Result<(ItemId, ItemId, Vec), String> { + use effectcraft_color::Label; + use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemKind, SourceLayer}; + let doc = effectcraft_pdf::parse(bytes).map_err(|e| format!("{path}: {e}"))?; + let codec = effectcraft_pdf::codec(path, bytes).unwrap_or("PDF"); + let (w, h) = doc.pixel_size(); + let file = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder); + let mut comp = Comp::new(w, h, rate, duration); + let mut items = vec![]; + let names = effectcraft_pdf::layer_names(&doc); + for (i, lname) in names.iter().enumerate() { + let f = Footage { + path: path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: rate, + has_video: true, + alpha: AlphaMode::Straight, + loop_count: 1, + codec: codec.into(), + layer: Some(SourceLayer { index: i as u32, name: lname.clone(), layer_size: false, ..Default::default() }), + ..Default::default() + }; + let id = proj.add_item(&format!("{lname}/{file}"), Label::Lavender, Some(folder), ItemKind::Footage(f)); + items.push(id); + let mut l = build::layer(proj, &comp, lname, LayerSource::Footage { item: id }, (w, h), None); + l.name = comp.unique_layer_name(lname); + comp.layers.insert(0, l); + } + let cid = proj.add_item(name, Label::Sandstone, None, ItemKind::Comp(std::sync::Arc::new(comp))); + Ok((cid, folder, items)) +} + /// A copy of `src`'s transform group with fresh uids. fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) { let Some(tr) = src.props.sub("transform") else { return }; diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index 635e3260..564efb3b 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -21,5 +21,6 @@ effectcraft-export = { workspace = true } serde_json = { workspace = true } effectcraft-psd = { workspace = true } effectcraft-svg = { workspace = true } +effectcraft-pdf = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-time = { workspace = true } diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index 590112c9..7b516a2a 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -184,3 +184,116 @@ fn svg_footage_continuously_rasterizes() { assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); } + +/// A two-layer Illustrator-style PDF (optional-content layers "Sky" and "Art"): a gradient sky, +/// a red curved shape with a dark outline, and a clipped green bar. +fn layered_pdf() -> Vec { + let content = "/OC /L0 BDC\n/Pattern cs /P0 scn 0 0 120 80 re f\nEMC\n\ +/OC /L1 BDC\n1 0 0 rg 0.1 0.1 0.1 RG 3 w 20 20 m 20 60 l 60 70 100 60 v 100 20 l h B\n\ +q 30 0 40 80 re W n 0 0.8 0 rg 0 35 120 10 re f Q\nEMC\n"; + let page = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 120 80] /Resources << /Properties << /L0 5 0 R /L1 6 0 R >> /Pattern << /P0 7 0 R >> >> /Contents 4 0 R >>"; + let pattern = "<< /PatternType 2 /Shading << /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 0 80] /Function << /FunctionType 2 /Domain [0 1] /C0 [0.9 0.9 1] /C1 [0.2 0.4 0.9] /N 1 >> /Extend [true true] >> >>"; + effectcraft_pdf::write::pdf( + &[ + (1, "<< /Type /Catalog /Pages 2 0 R >>".into(), None), + (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), None), + (3, page.into(), None), + (4, "<< >>".into(), Some(content.as_bytes().to_vec())), + (5, "<< /Type /OCG /Name (Sky) >>".into(), None), + (6, "<< /Type /OCG /Name (Art) >>".into(), None), + (7, pattern.into(), None), + ], + 1, + ) +} + +#[test] +fn pdf_and_ai_footage_layers_and_shapes_from_vector_layer() { + let bytes = layered_pdf(); + let doc = effectcraft_pdf::parse(&bytes).unwrap(); + let (w, h) = doc.pixel_size(); + assert_eq!((w, h), (120, 80)); + let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0); + for name in ["art.pdf", "art.ai"] { + let path = tmp(name); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + let effectcraft_project::ItemKind::Footage(f) = &s.project.item(ItemId(item)).unwrap().kind else { panic!() }; + assert_eq!(f.codec, if name.ends_with(".ai") { "AI" } else { "PDF" }); + assert_eq!((f.width, f.height), (120, 80)); + s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let foot = s.render(cid, t, RenderOpts::default()); + assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised page"); + // Continuously Rasterize at 300 %. + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004 && sharp < soft, "{name}: continuously rasterised {sharp} vs {soft}"); + s.execute("prop.set", json!({"layer": lid, "path": "transform/scale", "value": [100, 100]})).unwrap(); + // Create Shapes from Vector Layer. + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + let shapes = s.render(cid, t, RenderOpts::default()); + // Clipping groups are not shape-layer features: the clipped bar (rows 33–47) draws in + // full; everything else matches. + let rows = |img: &Image| Image { + width: w, + height: h - 15, + data: img.data.iter().enumerate().filter(|(i, _)| !(33..48).contains(&(*i as u32 / w))).map(|(_, p)| *p).collect(), + }; + let d = mean_diff(&rows(&shapes), &rows(&direct)); + assert!(d < 0.01, "{name}: shapes vs page mean diff {d}"); + let fill = shapes.get(60, 30); + assert!(fill[0] > 0.9 && fill[1] < 0.1, "red shape {fill:?}"); + } + // Import As: Composition — one layer per file layer, top first, together like the page. + let path = tmp("layers.ai"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + let names: Vec<&str> = comp.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, vec!["Art", "Sky"]); + assert_eq!((comp.width, comp.height), (120, 80)); + let art = comp.layers[0].id.0; + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let d = mean_diff(&img, &direct); + assert!(d < 1.0 / 255.0, "layered comp vs page {d}"); + // Each layer's footage shows only its layer. + s.execute("layer.setSwitch", json!({"layers": [art], "switch": "video", "value": false})).unwrap(); + let sky = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + assert_eq!(sky.get(5, 5), img.get(5, 5)); + let p = sky.get(60, 30); + assert!(p[2] > p[0], "only sky under the art {p:?}"); +} + +#[test] +fn eps_footage_imports() { + let eps = + b"%!PS-Adobe-3.0 EPSF-3.0\n%%BoundingBox: 0 0 40 20\n1 0 0 setrgbcolor 0 0 20 20 rectfill 0 0 1 setrgbcolor newpath 30 10 8 0 360 arc fill\n%%EOF\n"; + let path = tmp("shape.eps"); + std::fs::write(&path, eps).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "E", "width": 40, "height": 20, "frameRate": 30, "duration": 1})).unwrap(); + s.execute_checked("layer.addItem", json!({"item": item})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + assert!(img.get(10, 10)[0] > 0.99 && img.get(30, 10)[2] > 0.99, "{:?} {:?}", img.get(10, 10), img.get(30, 10)); +} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 10998031..3db4b1a9 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -16,6 +16,7 @@ effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } effectcraft-psd = { workspace = true } effectcraft-svg = { workspace = true } +effectcraft-pdf = { workspace = true } effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs index 204312a5..111140a4 100644 --- a/crates/media/src/layered.rs +++ b/crates/media/src/layered.rs @@ -1,5 +1,7 @@ //! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one -//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised). +//! layer for footage imported as a composition) and vector files rasterised at any scale: SVG +//! (`effectcraft-svg`), PDF / Illustrator / EPS (`effectcraft-pdf`; one layer of the file for +//! footage imported as a composition). use effectcraft_project::{AlphaMode, Footage}; use effectcraft_raster::Image; @@ -11,7 +13,29 @@ fn is_svg(path: &str, bytes: &[u8]) -> bool { path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) } -/// Footage for a Photoshop or SVG file (`None` for other formats). +/// The vector document of an SVG / PDF / AI / EPS file (`None` for other formats), restricted +/// to the footage's layer when it names one. +pub fn vector_doc(path: &str, bytes: &[u8], layer: Option<&effectcraft_project::SourceLayer>) -> Option> { + if is_svg(path, bytes) { + return Some(effectcraft_svg::parse(bytes).map_err(|e| format!("{path}: {e}"))); + } + effectcraft_pdf::sniff(bytes)?; + Some( + effectcraft_pdf::parse(bytes) + .map(|d| match layer { + Some(l) => effectcraft_pdf::layer_doc(&d, l.index as usize), + None => d, + }) + .map_err(|e| format!("{path}: {e}")), + ) +} + +fn is_pdf_like(path: &str, bytes: &[u8]) -> bool { + let ext = std::path::Path::new(path).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + effectcraft_pdf::sniff(bytes).is_some() && (effectcraft_pdf::EXTENSIONS.contains(&ext.as_str()) || bytes.starts_with(b"%PDF")) +} + +/// Footage for a Photoshop or vector file (`None` for other formats). pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { if effectcraft_psd::is_psd(bytes) { let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; @@ -28,6 +52,14 @@ pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { f.alpha = AlphaMode::Straight; return Ok(Some(f)); } + if is_pdf_like(path, bytes) { + let doc = effectcraft_pdf::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true); + f.codec = effectcraft_pdf::codec(path, bytes).unwrap_or("PDF").into(); + f.alpha = AlphaMode::Straight; + return Ok(Some(f)); + } Ok(None) } @@ -42,8 +74,8 @@ pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) - .map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); } - if is_svg(path, bytes) { - let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + if is_svg(path, bytes) || is_pdf_like(path, bytes) { + let doc = vector_doc(path, bytes, footage.layer.as_ref()).unwrap_or(Err(String::new())).map_err(MediaError::Decode)?; let (w, h) = doc.pixel_size(); let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); // The rasteriser produces premultiplied pixels already. @@ -53,9 +85,9 @@ pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) - Ok(None) } -/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize). -pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option { - let doc = effectcraft_svg::parse(bytes).ok()?; +/// A vector file rasterised at `scale` × its pixel size (Continuously Rasterize). +pub(crate) fn rasterize_vector(path: &str, bytes: &[u8], layer: Option<&effectcraft_project::SourceLayer>, scale: f64) -> Option { + let doc = vector_doc(path, bytes, layer)?.ok()?; let (w, h) = doc.pixel_size(); let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384)); Some(effectcraft_svg::rasterize(&doc, sw, sh, scale)) diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 6c67f50f..5de2eb8c 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -16,6 +16,7 @@ mod convert; pub mod exr_channels; mod layered; +pub use layered::vector_doc; mod pool; mod probe; @@ -47,7 +48,7 @@ impl From for MediaError { } /// File extensions recognised as stills. -pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"]; +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg", "pdf", "ai", "eps"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 9bdefcb6..ed6521c9 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -516,7 +516,7 @@ impl FootageSource for MediaPool { return None; } let bytes = self.inner.read(&footage.path).ok()?; - crate::layered::rasterize_svg(&bytes, scale).map(Arc::new) + crate::layered::rasterize_vector(&footage.path, &bytes, footage.layer.as_ref(), scale).map(Arc::new) } fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml new file mode 100644 index 00000000..93d082b5 --- /dev/null +++ b/crates/pdf/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "effectcraft-pdf" +description = "Vector footage import: PDF and PDF-compatible Illustrator (.ai) pages and EPS (a PostScript subset) parsed into the SVG render tree (paths, fills, strokes, axial / radial shadings, clipping, optional-content layers)" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-svg = { workspace = true } +kurbo = { workspace = true } +miniz_oxide = "0.8" diff --git a/crates/pdf/README.md b/crates/pdf/README.md new file mode 100644 index 00000000..a1a9d0b3 --- /dev/null +++ b/crates/pdf/README.md @@ -0,0 +1,48 @@ +# effectcraft-pdf + +Vector footage from PDF files, PDF-compatible Adobe Illustrator files (`.ai`, which are PDF +documents) and Encapsulated PostScript (`.eps`). Files parse into the same render tree as SVG +(`effectcraft_svg::Doc`), so they rasterise at any scale (Continuously Rasterize), convert to shape +layers (Layer ▸ Create ▸ Create Shapes from Vector Layer) and import as compositions with one +layer per file layer (File ▸ Import ▸ Composition). + +Specifications used: **ISO 32000-1:2008** (PDF 1.7; the public Adobe edition) for the file +structure, filters, content streams, graphics state, colour spaces, functions, shadings and +optional content; the **PostScript Language Reference, third edition** and the **Encapsulated +PostScript File Format Specification 3.0** for EPS. No third-party PDF/PostScript code was read; +test files are generated in code ([`write`](src/write.rs) and the tests). + +## What is read + +- **Structure:** indirect objects found by scanning (damaged or incremental cross-reference + sections do not matter), object streams, the page tree with inherited resources, MediaBox / + CropBox and `/Rotate`. Filters: Flate (with PNG predictors), LZW, ASCIIHex, ASCII85, + RunLength. Encrypted files are refused. +- **Content:** `q`/`Q`, `cm`, line width / cap / join / miter / dash, ExtGState (`CA`, `ca`, `LW`, + `LC`, `LJ`, `ML`, `D`), all path construction and painting operators (non-zero and even-odd), + clipping (`W`, `W*`) as clipping groups, colour in DeviceGray / RGB / CMYK, ICCBased (by + component count), CalGray / CalRGB, Lab, Indexed, Separation and DeviceN (tint transforms of + function types 0, 2 and 3), axial and radial shadings as pattern fills and through `sh`, form + XObjects (matrix, bounding-box clip, nested resources). +- **Layers:** top-level optional-content marked sequences (`/OC … BDC`, Illustrator's layers) + become the document's layers, named after their OCG; content outside them is gathered into + layers "Layer N". +- **EPS:** a PostScript interpreter for the subset exported vector EPS uses: operand and + dictionary stacks, procedures and `def`/`bind`/`load`/`where`, arithmetic, comparisons, + `if`/`ifelse`/`for`/`repeat`/`loop`/`forall`, arrays and strings, `gsave`/`grestore`, + `save`/`restore`, matrices (`concat`, `translate`, `scale`, `rotate`, `transform` …), path + construction including `arc`/`arcn` and relative operators, `fill`/`eofill`/`stroke`, + `clip`/`eoclip`, `rectfill`/`rectstroke`/`rectclip`, gray / RGB / CMYK / HSB colour. DOS EPS + binary headers and `%%HiResBoundingBox` / `%%BoundingBox` are honoured; `%%BeginData` / + `%%BeginBinary` / preview sections are skipped. + +## Limitations + +- Text is not drawn (embedded fonts are not read); images, tiling patterns, mesh shadings + (types 4–7), PostScript calculator functions, soft masks and blend modes are skipped. What a + document skipped is listed in `Doc::skipped`. +- Illustrator EPS files whose drawing depends on Adobe's procedure-set resources (defined in + the file through constructs beyond this subset) and Level 3 `shfill` gradients in EPS draw + only what the subset understands; save such artwork as PDF-compatible `.ai` or PDF instead. +- Shape-layer conversion has no clipping: clipped artwork converts unclipped. +- Only the first page of a multi-page PDF is imported. diff --git a/crates/pdf/src/build.rs b/crates/pdf/src/build.rs new file mode 100644 index 00000000..ca063dee --- /dev/null +++ b/crates/pdf/src/build.rs @@ -0,0 +1,93 @@ +//! The output side shared by the PDF and PostScript interpreters: an open-group stack (clipping +//! groups, optional-content layers) that collects shapes into the SVG render tree. + +use effectcraft_svg::{Affine, BezPath, FillRule, Geom, Group, Node, Paint, Shape, Stroke}; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Plain, + Clip, + Layer, +} + +pub(crate) struct Builder { + stack: Vec<(Group, Kind)>, + /// Which of the finished top-level nodes are optional-content layers. + pub top_layers: Vec, + pub skipped: Vec, + pub shapes: usize, +} + +pub(crate) fn group(name: &str) -> Group { + Group { name: name.to_string(), transform: Affine::IDENTITY, opacity: 1.0, children: vec![], clip: vec![] } +} + +impl Builder { + pub fn new() -> Builder { + Builder { stack: vec![(group("page"), Kind::Plain)], top_layers: vec![], skipped: vec![], shapes: 0 } + } + + pub fn depth(&self) -> usize { + self.stack.len() + } + + pub fn skip(&mut self, what: &str) { + if !self.skipped.iter().any(|s| s == what) { + self.skipped.push(what.to_string()); + } + } + + fn add(&mut self, n: Node, layer: bool) { + if self.stack.len() == 1 { + self.top_layers.push(layer); + } + if let Some((g, _)) = self.stack.last_mut() { + g.children.push(n); + } + } + + /// Close every group opened above `depth`. + pub fn close_to(&mut self, depth: usize) { + while self.stack.len() > depth.max(1) { + let Some((g, k)) = self.stack.pop() else { break }; + // Empty clipping groups draw nothing. + if g.children.is_empty() { + continue; + } + self.add(Node::Group(g), k == Kind::Layer); + } + } + + /// Children drawn from now on are clipped by `path` (page space) until the group closes. + pub fn clip(&mut self, path: BezPath, rule: FillRule) { + let mut g = group("Clip Group"); + g.clip.push((path, rule)); + self.stack.push((g, Kind::Clip)); + } + + /// Open an optional-content layer (or a named group). + pub fn open(&mut self, name: &str, layer: bool) { + self.stack.push((group(name), if layer { Kind::Layer } else { Kind::Plain })); + } + + pub fn fill_stroke(&mut self, path: BezPath, transform: Affine, fill: Option<(Paint, f64, FillRule)>, stroke: Option) { + if path.elements().is_empty() || (fill.is_none() && stroke.is_none()) { + return; + } + self.shapes += 1; + let (f, fo, rule) = match fill { + Some((p, o, r)) => (Some(p), o, r), + None => (None, 1.0, FillRule::NonZero), + }; + let s = Shape { name: "Path".into(), geom: Geom::Path(path), transform, fill: f, fill_opacity: fo, fill_rule: rule, stroke, opacity: 1.0 }; + self.add(Node::Shape(s), false); + } + + /// The finished top-level nodes and their layer flags. + pub fn finish(mut self) -> (Vec, Vec, Vec) { + self.close_to(1); + let root = self.stack.pop().map(|(g, _)| g.children).unwrap_or_default(); + let flags = self.top_layers; + (root, flags, self.skipped) + } +} diff --git a/crates/pdf/src/color.rs b/crates/pdf/src/color.rs new file mode 100644 index 00000000..b35e9f39 --- /dev/null +++ b/crates/pdf/src/color.rs @@ -0,0 +1,288 @@ +//! Colour spaces (ISO 32000-1 §8.6) reduced to sRGB, and functions (§7.10: sampled, +//! exponential and stitching; PostScript calculator functions are not evaluated). + +use crate::object::{Dict, File, Obj, decode_stream}; + +#[derive(Clone, Debug, PartialEq)] +pub enum Cs { + Gray, + Rgb, + Cmyk, + /// CIE L*a*b* (approximated). + Lab, + Indexed { + base: Box, + hival: usize, + lookup: Vec, + }, + /// Separation / DeviceN: components → alternate space through a tint transform. + Tint { + n: usize, + alt: Box, + func: Func, + }, + Pattern, +} + +impl Cs { + pub fn components(&self) -> usize { + match self { + Cs::Gray | Cs::Indexed { .. } => 1, + Cs::Rgb | Cs::Lab => 3, + Cs::Cmyk => 4, + Cs::Tint { n, .. } => *n, + Cs::Pattern => 0, + } + } + + /// The initial colour of the space (black, or tint 1). + pub fn initial(&self) -> Vec { + match self { + Cs::Cmyk => vec![0.0, 0.0, 0.0, 1.0], + Cs::Tint { n, .. } => vec![1.0; *n], + Cs::Lab => vec![0.0, 0.0, 0.0], + c => vec![0.0; c.components()], + } + } + + pub fn to_rgb(&self, c: &[f64]) -> [f64; 3] { + let g = |i: usize| c.get(i).copied().unwrap_or(0.0).clamp(0.0, 1.0); + match self { + Cs::Gray => [g(0); 3], + Cs::Rgb | Cs::Pattern => [g(0), g(1), g(2)], + Cs::Cmyk => { + let k = g(3); + [(1.0 - g(0)) * (1.0 - k), (1.0 - g(1)) * (1.0 - k), (1.0 - g(2)) * (1.0 - k)] + } + Cs::Lab => lab_to_rgb(c.first().copied().unwrap_or(0.0), c.get(1).copied().unwrap_or(0.0), c.get(2).copied().unwrap_or(0.0)), + Cs::Indexed { base, hival, lookup } => { + let i = (c.first().copied().unwrap_or(0.0).round().max(0.0) as usize).min(*hival); + let n = base.components(); + let v: Vec = (0..n).map(|k| lookup.get(i * n + k).copied().unwrap_or(0) as f64 / 255.0).collect(); + base.to_rgb(&v) + } + Cs::Tint { alt, func, .. } => alt.to_rgb(&func.eval(c)), + } + } +} + +fn lab_to_rgb(l: f64, a: f64, b: f64) -> [f64; 3] { + let fy = (l + 16.0) / 116.0; + let fx = fy + a / 500.0; + let fz = fy - b / 200.0; + let f = |t: f64| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f64 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + let (x, y, z) = (0.9505 * f(fx), f(fy), 1.089 * f(fz)); + let lin = [3.2406 * x - 1.5372 * y - 0.4986 * z, -0.9689 * x + 1.8758 * y + 0.0415 * z, 0.0557 * x - 0.204 * y + 1.057 * z]; + lin.map(|v| { + let v = v.clamp(0.0, 1.0); + if v <= 0.003_130_8 { 12.92 * v } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } + }) +} + +/// Read a colour space (a name, or an array such as `[/ICCBased 5 0 R]`); named resources are +/// looked up in `res` (`/ColorSpace`). +pub fn color_space(file: &File, o: &Obj, res: Option<&Dict>) -> Cs { + let o = file.resolve(o); + match o { + Obj::Name(n) => match n.as_str() { + "DeviceGray" | "G" | "CalGray" => Cs::Gray, + "DeviceRGB" | "RGB" | "CalRGB" => Cs::Rgb, + "DeviceCMYK" | "CMYK" => Cs::Cmyk, + "Pattern" => Cs::Pattern, + other => match res.and_then(|r| file.get_dict(r, "ColorSpace")).and_then(|cs| file.get(cs, other)) { + Some(x) if !matches!(x, Obj::Name(m) if m == other) => color_space(file, x, None), + _ => Cs::Gray, + }, + }, + Obj::Array(a) => { + let head = a.first().map(|h| file.resolve(h)).and_then(Obj::name).unwrap_or(""); + match head { + "ICCBased" => { + let n = a.get(1).map(|s| file.resolve(s)).and_then(Obj::dict).and_then(|d| file.get_num(d, "N")).unwrap_or(3.0) as usize; + match n { + 1 => Cs::Gray, + 4 => Cs::Cmyk, + _ => Cs::Rgb, + } + } + "CalRGB" => Cs::Rgb, + "CalGray" => Cs::Gray, + "Lab" => Cs::Lab, + "Pattern" => Cs::Pattern, + "Indexed" | "I" => { + let base = a.get(1).map(|b| color_space(file, b, res)).unwrap_or(Cs::Rgb); + let hival = a.get(2).and_then(|h| file.resolve(h).num()).unwrap_or(0.0) as usize; + let lookup = match a.get(3).map(|l| file.resolve(l)) { + Some(Obj::Str(s)) => s.clone(), + Some(Obj::Stream(d, raw)) => decode_stream(file, d, raw).unwrap_or_default(), + _ => vec![], + }; + Cs::Indexed { base: Box::new(base), hival, lookup } + } + "Separation" | "DeviceN" => { + let n = if head == "Separation" { 1 } else { a.get(1).map(|x| file.resolve(x)).and_then(Obj::array).map_or(1, <[Obj]>::len) }; + let alt = a.get(2).map(|b| color_space(file, b, res)).unwrap_or(Cs::Gray); + let func = a.get(3).map(|f| Func::read(file, f)).unwrap_or(Func::Unsupported); + // An unreadable tint transform: tint 1 = black, 0 = white. + if func == Func::Unsupported { + return Cs::Tint { n, alt: Box::new(Cs::Gray), func: Func::Exp { domain: [0.0, 1.0], c0: vec![1.0], c1: vec![0.0], n: 1.0 } }; + } + Cs::Tint { n, alt: Box::new(alt), func } + } + _ => Cs::Rgb, + } + } + _ => Cs::Gray, + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Func { + /// Type 2. + Exp { + domain: [f64; 2], + c0: Vec, + c1: Vec, + n: f64, + }, + /// Type 3. + Stitch { + domain: [f64; 2], + funcs: Vec, + bounds: Vec, + encode: Vec, + }, + /// Type 0 with one input. + Sampled { + domain: [f64; 2], + size: usize, + outputs: usize, + encode: [f64; 2], + decode: Vec, + samples: Vec, + }, + /// An array of one-output functions. + Array(Vec), + Unsupported, +} + +impl Func { + pub fn read(file: &File, o: &Obj) -> Func { + let o = file.resolve(o); + if let Obj::Array(a) = o { + return Func::Array(a.iter().map(|f| Func::read(file, f)).collect()); + } + let Some(d) = o.dict() else { return Func::Unsupported }; + let dom = file.get(d, "Domain").map(|x| file.nums(x)).unwrap_or_default(); + let domain = [dom.first().copied().unwrap_or(0.0), dom.get(1).copied().unwrap_or(1.0)]; + match file.get_num(d, "FunctionType").unwrap_or(-1.0) as i32 { + 2 => Func::Exp { + domain, + c0: file.get(d, "C0").map(|x| file.nums(x)).unwrap_or_else(|| vec![0.0]), + c1: file.get(d, "C1").map(|x| file.nums(x)).unwrap_or_else(|| vec![1.0]), + n: file.get_num(d, "N").unwrap_or(1.0), + }, + 3 => Func::Stitch { + domain, + funcs: file.get(d, "Functions").and_then(Obj::array).map(|a| a.iter().map(|f| Func::read(file, f)).collect()).unwrap_or_default(), + bounds: file.get(d, "Bounds").map(|x| file.nums(x)).unwrap_or_default(), + encode: file.get(d, "Encode").map(|x| file.nums(x)).unwrap_or_default(), + }, + 0 => { + let Obj::Stream(sd, raw) = o else { return Func::Unsupported }; + let Some(data) = decode_stream(file, sd, raw) else { return Func::Unsupported }; + let size = file.get(d, "Size").map(|x| file.nums(x)).and_then(|v| v.first().copied()).unwrap_or(0.0) as usize; + let bps = file.get_num(d, "BitsPerSample").unwrap_or(8.0) as usize; + let range = file.get(d, "Range").map(|x| file.nums(x)).unwrap_or_default(); + let outputs = range.len() / 2; + if size == 0 || outputs == 0 || !matches!(bps, 1 | 2 | 4 | 8 | 12 | 16 | 24 | 32) { + return Func::Unsupported; + } + let enc = file.get(d, "Encode").map(|x| file.nums(x)).unwrap_or_default(); + let encode = [enc.first().copied().unwrap_or(0.0), enc.get(1).copied().unwrap_or(size as f64 - 1.0)]; + let decode = file.get(d, "Decode").map(|x| file.nums(x)).filter(|v| v.len() >= outputs * 2).unwrap_or_else(|| range.clone()); + let max = ((1u64 << bps) - 1) as f64; + let mut samples = Vec::with_capacity(size * outputs); + let mut bit = 0usize; + for _ in 0..size * outputs { + let mut v = 0u64; + for _ in 0..bps { + let byte = data.get(bit / 8).copied().unwrap_or(0); + v = (v << 1) | ((byte >> (7 - bit % 8)) & 1) as u64; + bit += 1; + } + samples.push(v as f64 / max); + } + Func::Sampled { domain, size, outputs, encode, decode, samples } + } + _ => Func::Unsupported, + } + } + + /// Evaluate at the first input (the functions used by shadings and tints take one input + /// in practice; extra inputs of DeviceN tints are averaged in). + pub fn eval(&self, input: &[f64]) -> Vec { + let t = input.first().copied().unwrap_or(0.0); + self.eval1(t) + } + + pub fn eval1(&self, t: f64) -> Vec { + match self { + Func::Exp { domain, c0, c1, n } => { + let t = t.clamp(domain[0].min(domain[1]), domain[0].max(domain[1])); + let x = if *n == 1.0 { t } else { t.max(0.0).powf(*n) }; + (0..c0.len().max(c1.len())) + .map(|i| c0.get(i).copied().unwrap_or(0.0) + x * (c1.get(i).copied().unwrap_or(1.0) - c0.get(i).copied().unwrap_or(0.0))) + .collect() + } + Func::Stitch { domain, funcs, bounds, encode } => { + if funcs.is_empty() { + return vec![0.0]; + } + let t = t.clamp(domain[0], domain[1]); + let k = bounds.iter().take_while(|b| t >= **b).count().min(funcs.len() - 1); + let lo = if k == 0 { domain[0] } else { bounds[k - 1] }; + let hi = bounds.get(k).copied().unwrap_or(domain[1]); + let (e0, e1) = (encode.get(2 * k).copied().unwrap_or(0.0), encode.get(2 * k + 1).copied().unwrap_or(1.0)); + let u = if hi > lo { e0 + (t - lo) / (hi - lo) * (e1 - e0) } else { e0 }; + funcs[k].eval1(u) + } + Func::Sampled { domain, size, outputs, encode, decode, samples } => { + let t = t.clamp(domain[0], domain[1]); + let e = if domain[1] > domain[0] { encode[0] + (t - domain[0]) / (domain[1] - domain[0]) * (encode[1] - encode[0]) } else { encode[0] }; + let e = e.clamp(0.0, *size as f64 - 1.0); + let (i0, f) = (e.floor() as usize, e.fract()); + let i1 = (i0 + 1).min(size - 1); + (0..*outputs) + .map(|o| { + let s = samples[i0 * outputs + o] * (1.0 - f) + samples[i1 * outputs + o] * f; + let (d0, d1) = (decode.get(2 * o).copied().unwrap_or(0.0), decode.get(2 * o + 1).copied().unwrap_or(1.0)); + d0 + s * (d1 - d0) + }) + .collect() + } + Func::Array(fs) => fs.iter().map(|f| f.eval1(t).first().copied().unwrap_or(0.0)).collect(), + Func::Unsupported => vec![0.5], + } + } + + /// Input values worth sampling exactly (stitching bounds). + pub fn breakpoints(&self) -> Vec { + match self { + Func::Stitch { bounds, .. } => bounds.clone(), + _ => vec![], + } + } +} + +/// PDF text string → Rust string (UTF-16BE with a byte-order mark, else PDFDocEncoding ≈ Latin-1). +pub fn text_string(b: &[u8]) -> String { + if b.starts_with(&[0xFE, 0xFF]) { + let u: Vec = b[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + return String::from_utf16_lossy(&u); + } + if let Some(rest) = b.strip_prefix(&[0xEF, 0xBB, 0xBF]) { + return String::from_utf8_lossy(rest).into_owned(); + } + b.iter().map(|&c| c as char).collect() +} diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs new file mode 100644 index 00000000..cd835113 --- /dev/null +++ b/crates/pdf/src/lib.rs @@ -0,0 +1,185 @@ +//! Vector footage: PDF, PDF-compatible Illustrator files (`.ai`) and Encapsulated PostScript +//! (see README.md for what is read and the specifications used). +//! +//! Files parse into the same render tree as SVG ([`effectcraft_svg::Doc`]): document pixels are +//! PostScript points (1/72 in), y down, so footage rasterises at any scale (Continuously +//! Rasterize) and converts to shape layers (Create Shapes from Vector Layer). The root group's +//! children are the document's **layers**: one group per top-level optional-content group +//! (Illustrator layers), with content outside them gathered into groups of its own. + +mod build; +mod color; +mod object; +mod page; +mod ps; +pub mod write; + +pub use effectcraft_svg::Doc; +use effectcraft_svg::{Affine, Group, Node}; + +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + NotVector, + Encrypted, + NoPages, + PageOutOfRange(usize), + NoBoundingBox, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotVector => write!(f, "not a PDF, Illustrator or EPS file"), + Error::Encrypted => write!(f, "encrypted PDF files are not supported"), + Error::NoPages => write!(f, "the PDF has no pages"), + Error::PageOutOfRange(n) => write!(f, "the PDF has no page {}", n + 1), + Error::NoBoundingBox => write!(f, "the EPS file has no %%BoundingBox"), + } + } +} + +impl std::error::Error for Error {} + +/// File extensions read as vector footage by this crate. +pub const EXTENSIONS: &[&str] = &["pdf", "ai", "eps", "epsf", "epsi"]; + +/// The format of a file, by its first bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Format { + Pdf, + Eps, +} + +/// Sniff a PDF (`%PDF-` near the start, as PDF-compatible `.ai` files are) or an EPS file. +pub fn sniff(bytes: &[u8]) -> Option { + let head = &bytes[..bytes.len().min(1024)]; + if head.windows(5).any(|w| w == b"%PDF-") { + return Some(Format::Pdf); + } + if bytes.starts_with(&[0xC5, 0xD0, 0xD3, 0xC6]) || bytes.starts_with(b"%!PS") { + return Some(Format::Eps); + } + None +} + +/// A footage codec name for a sniffed file ("PDF", "AI", "EPS"). +pub fn codec(path: &str, bytes: &[u8]) -> Option<&'static str> { + let f = sniff(bytes)?; + let ai = path.to_ascii_lowercase().ends_with(".ai"); + Some(match (f, ai) { + (_, true) => "AI", + (Format::Pdf, false) => "PDF", + (Format::Eps, false) => "EPS", + }) +} + +/// Parse the first page of a PDF / `.ai`, or an EPS file. +pub fn parse(bytes: &[u8]) -> Result { + parse_page(bytes, 0) +} + +/// Parse page `index` of a PDF (EPS files have one page). +pub fn parse_page(bytes: &[u8], index: usize) -> Result { + match sniff(bytes) { + Some(Format::Pdf) => parse_pdf(bytes, index), + Some(Format::Eps) => parse_eps(bytes), + None => Err(Error::NotVector), + } +} + +/// Number of pages (1 for EPS). +pub fn page_count(bytes: &[u8]) -> usize { + match sniff(bytes) { + Some(Format::Pdf) => page::pages(&object::File::parse(bytes)).len(), + Some(Format::Eps) => 1, + None => 0, + } +} + +/// Build the document: `root` children become layer groups (see the crate docs). +fn finish(nodes: Vec, flags: Vec, skipped: Vec, width: f64, height: f64, to_doc: Affine) -> Doc { + let mut layers: Vec = vec![]; + let mut loose: Vec = vec![]; + let flush = |loose: &mut Vec, layers: &mut Vec| { + if !loose.is_empty() { + let mut g = build::group(""); + g.children = std::mem::take(loose); + layers.push(Node::Group(g)); + } + }; + for (n, is_layer) in nodes.into_iter().zip(flags.into_iter().chain(std::iter::repeat(false))) { + if is_layer { + flush(&mut loose, &mut layers); + layers.push(n); + } else { + loose.push(n); + } + } + flush(&mut loose, &mut layers); + // Name unnamed layers "Layer N" (bottom first, as Illustrator numbers them). + for (i, l) in layers.iter_mut().enumerate() { + if let Node::Group(g) = l + && g.name.is_empty() + { + g.name = format!("Layer {}", i + 1); + } + } + Doc { width, height, root: Group { name: "page".into(), transform: to_doc, opacity: 1.0, children: layers, clip: vec![] }, skipped } +} + +fn parse_pdf(bytes: &[u8], index: usize) -> Result { + let file = object::File::parse(bytes); + if file.encrypted() { + return Err(Error::Encrypted); + } + let pages = page::pages(&file); + if pages.is_empty() { + return Err(Error::NoPages); + } + let pg = pages.get(index).ok_or(Error::PageOutOfRange(index))?; + let [x0, y0, x1, y1] = pg.bbox; + let (w, h) = (x1 - x0, y1 - y0); + // Default user space (y up) → document pixels (y down), with the page's /Rotate. + let (to_doc, dw, dh) = match pg.rotate { + 90 => (Affine::new([0.0, 1.0, 1.0, 0.0, -y0, -x0]), h, w), + 180 => (Affine::new([-1.0, 0.0, 0.0, 1.0, x1, -y0]), w, h), + 270 => (Affine::new([0.0, -1.0, -1.0, 0.0, y1, x1]), h, w), + _ => (Affine::new([1.0, 0.0, 0.0, -1.0, -x0, y1]), w, h), + }; + let content = page::page_content(&file, pg); + let mut it = page::Interp::new(&file, pg.bbox); + it.run(&content, &pg.resources, Affine::IDENTITY, 0); + let (nodes, flags, skipped) = it.b.finish(); + Ok(finish(nodes, flags, skipped, dw.max(1.0), dh.max(1.0), to_doc)) +} + +fn parse_eps(bytes: &[u8]) -> Result { + let (ps, bb) = ps::eps_parts(bytes).ok_or(Error::NoBoundingBox)?; + let mut it = ps::Interp::new(bb); + it.run(ps); + let (nodes, flags, skipped) = it.b.finish(); + let to_doc = Affine::new([1.0, 0.0, 0.0, -1.0, -bb[0], bb[3]]); + Ok(finish(nodes, flags, skipped, (bb[2] - bb[0]).max(1.0), (bb[3] - bb[1]).max(1.0), to_doc)) +} + +/// The document's layer names (the root's children), bottom first. +pub fn layer_names(doc: &Doc) -> Vec { + doc.root + .children + .iter() + .map(|n| match n { + Node::Group(g) => g.name.clone(), + Node::Shape(s) => s.name.clone(), + }) + .collect() +} + +/// The document with only layer `index` (for one layer of a file imported as a composition). +pub fn layer_doc(doc: &Doc, index: usize) -> Doc { + let mut d = doc.clone(); + d.root.children = doc.root.children.get(index).cloned().into_iter().collect(); + d +} + +#[cfg(test)] +mod tests; diff --git a/crates/pdf/src/object.rs b/crates/pdf/src/object.rs new file mode 100644 index 00000000..e24f6ec8 --- /dev/null +++ b/crates/pdf/src/object.rs @@ -0,0 +1,708 @@ +//! PDF objects (ISO 32000-1 §7.2–7.5): the lexer, the object model, stream filters and the +//! indirect-object table. +//! +//! The table is built by scanning the file for `n g obj` headers (later definitions win, as +//! incremental updates do) and by unpacking object streams, so damaged or rewritten +//! cross-reference sections do not matter. + +use std::collections::HashMap; + +#[derive(Clone, Debug, PartialEq)] +pub enum Obj { + Null, + Bool(bool), + Num(f64), + Name(String), + Str(Vec), + Array(Vec), + Dict(Dict), + Ref(u32, u16), + /// A stream: its dictionary and raw (still encoded) data. + Stream(Dict, Vec), + /// A content-stream operator (or any bare keyword). + Op(String), +} + +pub type Dict = HashMap; + +impl Obj { + pub fn num(&self) -> Option { + match self { + Obj::Num(n) => Some(*n), + _ => None, + } + } + pub fn name(&self) -> Option<&str> { + match self { + Obj::Name(n) => Some(n), + _ => None, + } + } + pub fn array(&self) -> Option<&[Obj]> { + match self { + Obj::Array(a) => Some(a), + _ => None, + } + } + pub fn dict(&self) -> Option<&Dict> { + match self { + Obj::Dict(d) | Obj::Stream(d, _) => Some(d), + _ => None, + } + } +} + +pub fn is_white(c: u8) -> bool { + matches!(c, 0 | 9 | 10 | 12 | 13 | 32) +} +pub fn is_delim(c: u8) -> bool { + matches!(c, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') +} + +/// A byte lexer producing PDF objects (arrays and dictionaries nested). +pub struct Lexer<'a> { + pub data: &'a [u8], + pub pos: usize, +} + +impl<'a> Lexer<'a> { + pub fn new(data: &'a [u8], pos: usize) -> Self { + Lexer { data, pos } + } + + pub fn skip_ws(&mut self) { + while self.pos < self.data.len() { + let c = self.data[self.pos]; + if is_white(c) { + self.pos += 1; + } else if c == b'%' { + while self.pos < self.data.len() && !matches!(self.data[self.pos], b'\n' | b'\r') { + self.pos += 1; + } + } else { + break; + } + } + } + + fn word(&mut self) -> &'a [u8] { + let s = self.pos; + while self.pos < self.data.len() && !is_white(self.data[self.pos]) && !is_delim(self.data[self.pos]) { + self.pos += 1; + } + &self.data[s..self.pos] + } + + /// The next object, `None` at the end. `]` and `>>` come back as `Op`s. + pub fn next(&mut self) -> Option { + self.next_depth(0) + } + + fn next_depth(&mut self, depth: usize) -> Option { + self.skip_ws(); + let c = *self.data.get(self.pos)?; + match c { + b'/' => { + self.pos += 1; + let w = self.word(); + Some(Obj::Name(decode_name(w))) + } + b'(' => Some(Obj::Str(self.literal_string())), + b'<' if self.data.get(self.pos + 1) == Some(&b'<') => { + self.pos += 2; + let mut d = Dict::new(); + loop { + if depth > 64 { + return Some(Obj::Null); + } + let k = self.next_depth(depth + 1); + match k { + None => break, + Some(Obj::Op(o)) if o == ">>" => break, + Some(Obj::Name(k)) => { + let v = self.next_depth(depth + 1).unwrap_or(Obj::Null); + let v = self.maybe_ref(v); + d.insert(k, v); + } + Some(_) => {} + } + } + Some(Obj::Dict(d)) + } + b'<' => { + self.pos += 1; + let mut hex = vec![]; + while self.pos < self.data.len() && self.data[self.pos] != b'>' { + let h = self.data[self.pos]; + if h.is_ascii_hexdigit() { + hex.push(h); + } + self.pos += 1; + } + self.pos += 1; + if hex.len() % 2 == 1 { + hex.push(b'0'); + } + Some(Obj::Str(hex.chunks(2).map(|p| (hexv(p[0]) << 4) | hexv(p[1])).collect())) + } + b'>' if self.data.get(self.pos + 1) == Some(&b'>') => { + self.pos += 2; + Some(Obj::Op(">>".into())) + } + b'[' => { + self.pos += 1; + let mut a = vec![]; + loop { + if depth > 64 { + return Some(Obj::Null); + } + match self.next_depth(depth + 1) { + None => break, + Some(Obj::Op(o)) if o == "]" => break, + Some(v) => { + let v = self.maybe_ref(v); + a.push(v); + } + } + } + Some(Obj::Array(a)) + } + b']' | b'{' | b'}' | b')' | b'>' => { + self.pos += 1; + Some(Obj::Op((c as char).to_string())) + } + _ => { + let w = self.word(); + if w.is_empty() { + self.pos += 1; + return Some(Obj::Op(String::new())); + } + Some(match w { + b"true" => Obj::Bool(true), + b"false" => Obj::Bool(false), + b"null" => Obj::Null, + _ => match parse_num(w) { + Some(n) => Obj::Num(n), + None => Obj::Op(String::from_utf8_lossy(w).into_owned()), + }, + }) + } + } + } + + /// `n g R` after a number inside arrays and dictionaries. + fn maybe_ref(&mut self, v: Obj) -> Obj { + let Obj::Num(n) = v else { return v }; + if n.fract() != 0.0 || n < 0.0 { + return v; + } + let save = self.pos; + self.skip_ws(); + let gp = self.pos; + let g = self.word(); + if let Some(gn) = parse_num(g).filter(|g| g.fract() == 0.0 && *g >= 0.0) { + self.skip_ws(); + if self.data.get(self.pos) == Some(&b'R') && self.data.get(self.pos + 1).is_none_or(|c| is_white(*c) || is_delim(*c)) { + self.pos += 1; + return Obj::Ref(n as u32, gn as u16); + } + } + let _ = gp; + self.pos = save; + v + } + + fn literal_string(&mut self) -> Vec { + self.pos += 1; + let mut out = vec![]; + let mut depth = 1; + while self.pos < self.data.len() { + let c = self.data[self.pos]; + self.pos += 1; + match c { + b'\\' => { + let Some(&e) = self.data.get(self.pos) else { break }; + self.pos += 1; + match e { + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'b' => out.push(8), + b'f' => out.push(12), + b'\r' => { + if self.data.get(self.pos) == Some(&b'\n') { + self.pos += 1; + } + } + b'\n' => {} + b'0'..=b'7' => { + let mut v = (e - b'0') as u32; + for _ in 0..2 { + match self.data.get(self.pos) { + Some(d @ b'0'..=b'7') => { + v = v * 8 + (d - b'0') as u32; + self.pos += 1; + } + _ => break, + } + } + out.push(v as u8); + } + other => out.push(other), + } + } + b'(' => { + depth += 1; + out.push(c); + } + b')' => { + depth -= 1; + if depth == 0 { + break; + } + out.push(c); + } + _ => out.push(c), + } + } + out + } +} + +fn hexv(c: u8) -> u8 { + match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + b'A'..=b'F' => c - b'A' + 10, + _ => 0, + } +} + +fn decode_name(w: &[u8]) -> String { + let mut out = vec![]; + let mut i = 0; + while i < w.len() { + if w[i] == b'#' && w.get(i + 1).is_some_and(u8::is_ascii_hexdigit) && w.get(i + 2).is_some_and(u8::is_ascii_hexdigit) { + out.push((hexv(w[i + 1]) << 4) | hexv(w[i + 2])); + i += 3; + } else { + out.push(w[i]); + i += 1; + } + } + String::from_utf8_lossy(&out).into_owned() +} + +pub fn parse_num(w: &[u8]) -> Option { + if w.is_empty() || !w.iter().all(|c| c.is_ascii_digit() || matches!(c, b'+' | b'-' | b'.')) || !w.iter().any(u8::is_ascii_digit) { + return None; + } + let s = std::str::from_utf8(w).ok()?; + // "--5" and similar malformed numbers: take what parses. + s.parse::().ok().or_else(|| s.trim_start_matches(['+', '-']).parse::().ok().map(|v| if s.starts_with('-') { -v } else { v })) +} + +// ------------------------------------------------------------------ filters + +/// Decode a stream's data through its `/Filter` chain. Unknown filters (images: DCT, JPX, CCITT, +/// JBIG2) give `None`. +pub fn decode_stream(file: &File, d: &Dict, raw: &[u8]) -> Option> { + let filters: Vec = match d.get("Filter").map(|f| file.resolve(f)) { + None => vec![], + Some(Obj::Name(n)) => vec![n.clone()], + Some(Obj::Array(a)) => a.iter().filter_map(|f| file.resolve(f).name().map(str::to_string)).collect(), + _ => vec![], + }; + let parms: Vec> = match d.get("DecodeParms").or_else(|| d.get("DP")).map(|p| file.resolve(p)) { + Some(Obj::Dict(p)) => vec![Some(p.clone())], + Some(Obj::Array(a)) => a.iter().map(|p| file.resolve(p).dict().cloned()).collect(), + _ => vec![], + }; + let mut data = raw.to_vec(); + for (i, f) in filters.iter().enumerate() { + data = match f.as_str() { + "FlateDecode" | "Fl" => inflate(&data)?, + "ASCIIHexDecode" | "AHx" => ascii_hex(&data), + "ASCII85Decode" | "A85" => ascii85(&data), + "LZWDecode" | "LZW" => { + lzw(&data, parms.get(i).and_then(|p| p.as_ref()).and_then(|p| p.get("EarlyChange")).and_then(Obj::num).unwrap_or(1.0) != 0.0) + } + "RunLengthDecode" | "RL" => run_length(&data), + _ => return None, + }; + if let Some(Some(p)) = parms.get(i) + && matches!(f.as_str(), "FlateDecode" | "Fl" | "LZWDecode" | "LZW") + { + data = predict(&data, p); + } + } + Some(data) +} + +pub fn inflate(data: &[u8]) -> Option> { + miniz_oxide::inflate::decompress_to_vec_zlib(data).ok().or_else(|| miniz_oxide::inflate::decompress_to_vec(data).ok()).or_else(|| { + // Truncated streams: keep what inflates. + let mut d = miniz_oxide::inflate::stream::InflateState::new_boxed(miniz_oxide::DataFormat::Zlib); + let mut out = vec![0u8; data.len() * 8 + 1024]; + let r = miniz_oxide::inflate::stream::inflate(&mut d, data, &mut out, miniz_oxide::MZFlush::Finish); + (r.bytes_written > 0).then(|| out[..r.bytes_written].to_vec()) + }) +} + +fn ascii_hex(data: &[u8]) -> Vec { + let mut hex: Vec = data.iter().copied().take_while(|c| *c != b'>').filter(u8::is_ascii_hexdigit).collect(); + if hex.len() % 2 == 1 { + hex.push(b'0'); + } + hex.chunks(2).map(|p| (hexv(p[0]) << 4) | hexv(p[1])).collect() +} + +pub fn ascii85(data: &[u8]) -> Vec { + let mut out = vec![]; + let mut group = [0u32; 5]; + let mut n = 0; + let mut i = 0; + if data.starts_with(b"<~") { + i = 2; + } + while i < data.len() { + let c = data[i]; + i += 1; + match c { + b'~' => break, + b'z' if n == 0 => out.extend_from_slice(&[0; 4]), + b'!'..=b'u' => { + group[n] = (c - b'!') as u32; + n += 1; + if n == 5 { + let v = group.iter().fold(0u32, |a, &d| a.wrapping_mul(85).wrapping_add(d)); + out.extend_from_slice(&v.to_be_bytes()); + n = 0; + } + } + _ => {} + } + } + if n > 1 { + for g in group.iter_mut().skip(n) { + *g = 84; + } + let v = group.iter().fold(0u32, |a, &d| a.wrapping_mul(85).wrapping_add(d)); + out.extend_from_slice(&v.to_be_bytes()[..n - 1]); + } + out +} + +fn lzw(data: &[u8], early: bool) -> Vec { + let mut out = vec![]; + let mut table: Vec> = (0..=255u16).map(|b| vec![b as u8]).collect(); + table.push(vec![]); + table.push(vec![]); + let (mut bits, mut acc, mut nbits) = (9u32, 0u32, 0u32); + let mut prev: Option> = None; + for &b in data { + acc = (acc << 8) | b as u32; + nbits += 8; + while nbits >= bits { + let code = ((acc >> (nbits - bits)) & ((1 << bits) - 1)) as usize; + nbits -= bits; + match code { + 256 => { + table.truncate(258); + bits = 9; + prev = None; + continue; + } + 257 => return out, + _ => {} + } + let entry = if code < table.len() { + table[code].clone() + } else if let Some(p) = &prev { + let mut e = p.clone(); + e.push(p[0]); + e + } else { + return out; + }; + out.extend_from_slice(&entry); + if let Some(p) = prev { + let mut e = p; + e.push(entry[0]); + table.push(e); + } + prev = Some(entry); + let limit = table.len() + usize::from(early); + if limit >= (1 << bits) && bits < 12 { + bits += 1; + } + } + } + out +} + +fn run_length(data: &[u8]) -> Vec { + let mut out = vec![]; + let mut i = 0; + while i < data.len() { + let l = data[i] as usize; + i += 1; + if l < 128 { + out.extend_from_slice(&data[i.min(data.len())..(i + l + 1).min(data.len())]); + i += l + 1; + } else if l > 128 { + if let Some(&b) = data.get(i) { + out.extend(std::iter::repeat_n(b, 257 - l)); + } + i += 1; + } else { + break; + } + } + out +} + +/// PNG / TIFF predictors (`/Predictor`, `/Colors`, `/BitsPerComponent`, `/Columns`). +fn predict(data: &[u8], p: &Dict) -> Vec { + let g = |k: &str, d: f64| p.get(k).and_then(Obj::num).unwrap_or(d) as usize; + let pred = g("Predictor", 1.0); + if pred < 10 { + return data.to_vec(); + } + let bpp = (g("Colors", 1.0) * g("BitsPerComponent", 8.0)).div_ceil(8).max(1); + let row = (g("Columns", 1.0) * g("Colors", 1.0) * g("BitsPerComponent", 8.0)).div_ceil(8); + let mut out = Vec::with_capacity(data.len()); + let mut prev = vec![0u8; row]; + for chunk in data.chunks(row + 1) { + if chunk.len() < row + 1 { + break; + } + let ft = chunk[0]; + let mut cur = chunk[1..].to_vec(); + for i in 0..row { + let a = if i >= bpp { cur[i - bpp] as i32 } else { 0 }; + let b = prev[i] as i32; + let c = if i >= bpp { prev[i - bpp] as i32 } else { 0 }; + let add = match ft { + 1 => a, + 2 => b, + 3 => (a + b) / 2, + 4 => { + let pp = a + b - c; + let (pa, pb, pc) = ((pp - a).abs(), (pp - b).abs(), (pp - c).abs()); + if pa <= pb && pa <= pc { + a + } else if pb <= pc { + b + } else { + c + } + } + _ => 0, + }; + cur[i] = (cur[i] as i32 + add) as u8; + } + out.extend_from_slice(&cur); + prev = cur; + } + out +} + +// ------------------------------------------------------------------ the file + +/// Every indirect object of a PDF file, plus the trailer dictionaries found. +pub struct File { + pub objects: HashMap, + pub trailers: Vec, +} + +fn find(hay: &[u8], needle: &[u8], from: usize) -> Option { + if from >= hay.len() || needle.is_empty() { + return None; + } + hay[from..].windows(needle.len()).position(|w| w == needle).map(|p| p + from) +} + +impl File { + pub fn parse(data: &[u8]) -> File { + let mut f = File { objects: HashMap::new(), trailers: vec![] }; + let mut i = 0; + while let Some(p) = find(data, b"obj", i) { + i = p + 3; + if p >= 3 && &data[p - 3..p] == b"end" { + continue; + } + if data.get(p + 3).is_some_and(|c| !is_white(*c) && !is_delim(*c)) { + continue; + } + // Walk back over "num gen ". + let mut q = p; + let back_ws = |q: &mut usize| { + let s = *q; + while *q > 0 && is_white(data[*q - 1]) { + *q -= 1; + } + *q < s + }; + let back_digits = |q: &mut usize| { + let e = *q; + while *q > 0 && data[*q - 1].is_ascii_digit() { + *q -= 1; + } + (*q < e).then(|| std::str::from_utf8(&data[*q..e]).ok()?.parse::().ok()).flatten() + }; + if !back_ws(&mut q) { + continue; + } + let Some(_gen) = back_digits(&mut q) else { continue }; + if !back_ws(&mut q) { + continue; + } + let Some(num) = back_digits(&mut q) else { continue }; + if q > 0 && !is_white(data[q - 1]) && !is_delim(data[q - 1]) { + continue; + } + let mut lx = Lexer::new(data, p + 3); + let Some(obj) = lx.next() else { continue }; + let obj = lx.maybe_ref(obj); + let obj = match obj { + Obj::Dict(d) => { + let save = lx.pos; + lx.skip_ws(); + if data[lx.pos..].starts_with(b"stream") { + let mut s = lx.pos + 6; + if data.get(s) == Some(&b'\r') { + s += 1; + } + if data.get(s) == Some(&b'\n') { + s += 1; + } + let declared = d.get("Length").and_then(Obj::num).map(|l| l as usize); + let end = match declared.filter(|l| { + s + l <= data.len() && { + let mut k = s + l; + while k < data.len() && is_white(data[k]) { + k += 1; + } + data[k..].starts_with(b"endstream") + } + }) { + Some(l) => s + l, + None => { + let mut e = find(data, b"endstream", s).unwrap_or(data.len()); + if e > s && data[e - 1] == b'\n' { + e -= 1; + } + if e > s && data[e - 1] == b'\r' { + e -= 1; + } + e + } + }; + i = end; + Obj::Stream(d, data[s..end].to_vec()) + } else { + lx.pos = save; + Obj::Dict(d) + } + } + o => o, + }; + if let Obj::Stream(d, _) = &obj + && d.get("Type").and_then(Obj::name) == Some("XRef") + { + f.trailers.push(d.clone()); + } + f.objects.insert(num, obj); + } + // Trailer dictionaries of classic cross-reference sections. + let mut i = 0; + while let Some(p) = find(data, b"trailer", i) { + i = p + 7; + let mut lx = Lexer::new(data, i); + if let Some(Obj::Dict(d)) = lx.next() { + f.trailers.push(d); + } + } + // Object streams. + let streams: Vec<(Dict, Vec)> = f + .objects + .values() + .filter_map(|o| match o { + Obj::Stream(d, raw) if d.get("Type").and_then(Obj::name) == Some("ObjStm") => Some((d.clone(), raw.clone())), + _ => None, + }) + .collect(); + for (d, raw) in streams { + let Some(dec) = decode_stream(&f, &d, &raw) else { continue }; + let n = d.get("N").and_then(Obj::num).unwrap_or(0.0) as usize; + let first = d.get("First").and_then(Obj::num).unwrap_or(0.0) as usize; + let mut lx = Lexer::new(&dec, 0); + let mut heads = vec![]; + for _ in 0..n.min(100_000) { + let (Some(Obj::Num(num)), Some(Obj::Num(off))) = (lx.next(), lx.next()) else { break }; + heads.push((num as u32, off as usize)); + } + for (num, off) in heads { + if f.objects.contains_key(&num) || first + off >= dec.len() { + continue; + } + let mut lx = Lexer::new(&dec, first + off); + if let Some(o) = lx.next() { + let o = lx.maybe_ref(o); + f.objects.insert(num, o); + } + } + } + f + } + + /// Follow references (bounded). + pub fn resolve<'a>(&'a self, o: &'a Obj) -> &'a Obj { + let mut o = o; + for _ in 0..32 { + match o { + Obj::Ref(n, _) => match self.objects.get(n) { + Some(x) => o = x, + None => return &Obj::Null, + }, + _ => return o, + } + } + &Obj::Null + } + + pub fn get<'a>(&'a self, d: &'a Dict, key: &str) -> Option<&'a Obj> { + d.get(key).map(|o| self.resolve(o)).filter(|o| !matches!(o, Obj::Null)) + } + + pub fn get_dict<'a>(&'a self, d: &'a Dict, key: &str) -> Option<&'a Dict> { + self.get(d, key)?.dict() + } + + pub fn get_num(&self, d: &Dict, key: &str) -> Option { + self.get(d, key)?.num() + } + + /// Numbers of an array (references resolved). + pub fn nums(&self, o: &Obj) -> Vec { + self.resolve(o).array().map(|a| a.iter().filter_map(|x| self.resolve(x).num()).collect()).unwrap_or_default() + } + + /// The document catalog. + pub fn catalog(&self) -> Option<&Dict> { + for t in self.trailers.iter().rev() { + if let Some(Obj::Dict(d)) = t.get("Root").map(|r| self.resolve(r)) { + return Some(d); + } + } + let mut cats: Vec<(&u32, &Obj)> = + self.objects.iter().filter(|(_, o)| matches!(o, Obj::Dict(d) if d.get("Type").and_then(Obj::name) == Some("Catalog"))).collect(); + cats.sort_by_key(|(n, _)| **n); + cats.last().and_then(|(_, o)| o.dict()) + } + + pub fn encrypted(&self) -> bool { + self.trailers.iter().any(|t| t.contains_key("Encrypt")) + } +} diff --git a/crates/pdf/src/page.rs b/crates/pdf/src/page.rs new file mode 100644 index 00000000..9a67f85c --- /dev/null +++ b/crates/pdf/src/page.rs @@ -0,0 +1,573 @@ +//! Pages and content streams (ISO 32000-1 §7.7.3, §8): the page tree, the graphics state, +//! path construction and painting, clipping, colour, shading patterns and `sh`, form XObjects +//! and optional-content marked sequences. Text, images and tiling patterns are skipped (listed +//! in [`effectcraft_svg::Doc::skipped`]). + +use effectcraft_svg::{Affine, BezPath, Cap, FillRule, Gradient, GradientKind, Join, Paint, Spread, Stroke}; +use kurbo::Point; + +use crate::build::Builder; +use crate::color::{Cs, Func, color_space, text_string}; +use crate::object::{Dict, File, Lexer, Obj, decode_stream}; + +/// A page: its dictionary with inherited attributes resolved. +pub struct Page { + pub dict: Dict, + pub resources: Dict, + /// CropBox (else MediaBox) `[x0, y0, x1, y1]`. + pub bbox: [f64; 4], + pub rotate: i32, +} + +/// The pages in order. +pub fn pages(file: &File) -> Vec { + let mut out = vec![]; + let Some(cat) = file.catalog() else { return out }; + let Some(root) = file.get_dict(cat, "Pages") else { return out }; + fn walk(file: &File, node: &Dict, inherited: (Option, Option>, Option>, i32), out: &mut Vec, depth: usize) { + if depth > 32 || out.len() > 10_000 { + return; + } + let res = file.get_dict(node, "Resources").cloned().or(inherited.0); + let media = file.get(node, "MediaBox").map(|m| file.nums(m)).filter(|v| v.len() == 4).or(inherited.1); + let crop = file.get(node, "CropBox").map(|m| file.nums(m)).filter(|v| v.len() == 4).or(inherited.2); + let rot = file.get_num(node, "Rotate").map(|r| r as i32).unwrap_or(inherited.3); + match file.get(node, "Kids").and_then(Obj::array) { + Some(kids) if file.get(node, "Type").and_then(Obj::name) != Some("Page") => { + for k in kids { + if let Some(d) = file.resolve(k).dict() { + walk(file, d, (res.clone(), media.clone(), crop.clone(), rot), out, depth + 1); + } + } + } + _ => { + let b = crop.or(media).unwrap_or_else(|| vec![0.0, 0.0, 612.0, 792.0]); + let bbox = [b[0].min(b[2]), b[1].min(b[3]), b[0].max(b[2]), b[1].max(b[3])]; + out.push(Page { dict: node.clone(), resources: res.unwrap_or_default(), bbox, rotate: rot.rem_euclid(360) }); + } + } + } + walk(file, root, (None, None, None, 0), &mut out, 0); + out +} + +/// The page's content streams, decoded and joined. +pub fn page_content(file: &File, page: &Page) -> Vec { + let mut out = vec![]; + let parts: Vec<&Obj> = match page.dict.get("Contents").map(|c| file.resolve(c)) { + Some(Obj::Array(a)) => a.iter().map(|x| file.resolve(x)).collect(), + Some(o) => vec![o], + None => vec![], + }; + for p in parts { + if let Obj::Stream(d, raw) = p + && let Some(data) = decode_stream(file, d, raw) + { + out.extend_from_slice(&data); + out.push(b'\n'); + } + } + out +} + +#[derive(Clone)] +enum Fill { + Color([f64; 3]), + Paint(Paint), + None, +} + +#[derive(Clone)] +struct GState { + ctm: Affine, + fill_cs: Cs, + stroke_cs: Cs, + fill: Fill, + stroke: Fill, + lw: f64, + cap: Cap, + join: Join, + miter: f64, + dash: Option<(Vec, f64)>, + fill_alpha: f64, + stroke_alpha: f64, +} + +impl Default for GState { + fn default() -> Self { + GState { + ctm: Affine::IDENTITY, + fill_cs: Cs::Gray, + stroke_cs: Cs::Gray, + fill: Fill::Color([0.0; 3]), + stroke: Fill::Color([0.0; 3]), + lw: 1.0, + cap: Cap::Butt, + join: Join::Miter, + miter: 10.0, + dash: None, + fill_alpha: 1.0, + stroke_alpha: 1.0, + } + } +} + +pub(crate) struct Interp<'a> { + file: &'a File, + pub b: Builder, + gs: GState, + saved: Vec<(GState, usize)>, + path: BezPath, + cur: Option, + start: Option, + pending_clip: Option, + marked: Vec>, + /// The page box in default user space (for `sh`). + page_box: [f64; 4], + budget: usize, +} + +fn affine(v: &[f64]) -> Affine { + if v.len() < 6 { + return Affine::IDENTITY; + } + Affine::new([v[0], v[1], v[2], v[3], v[4], v[5]]) +} + +impl<'a> Interp<'a> { + pub fn new(file: &'a File, page_box: [f64; 4]) -> Self { + Interp { + file, + b: Builder::new(), + gs: GState::default(), + saved: vec![], + path: BezPath::new(), + cur: None, + start: None, + pending_clip: None, + marked: vec![], + page_box, + budget: 20_000_000, + } + } + + /// Run a content stream with resources `res`; `pattern_base` maps pattern space to the page + /// (the CTM where the stream's default coordinate system starts). + pub fn run(&mut self, content: &[u8], res: &Dict, pattern_base: Affine, depth: usize) { + let mut lx = Lexer::new(content, 0); + let mut args: Vec = Vec::with_capacity(8); + let mut in_text = false; + while let Some(o) = lx.next() { + let Obj::Op(op) = o else { + if args.len() < 64 { + args.push(o); + } + continue; + }; + if self.budget == 0 { + self.b.skip("content (operation limit)"); + return; + } + self.budget -= 1; + if in_text { + if op == "ET" { + in_text = false; + } + args.clear(); + continue; + } + let n = |i: usize| args.get(i).and_then(Obj::num).unwrap_or(0.0); + let nums = || args.iter().filter_map(Obj::num).collect::>(); + match op.as_str() { + "q" => self.saved.push((self.gs.clone(), self.b.depth())), + "Q" => { + if let Some((g, d)) = self.saved.pop() { + self.gs = g; + self.b.close_to(d); + } + } + "cm" => self.gs.ctm *= affine(&nums()), + "w" => self.gs.lw = n(0), + "J" => self.gs.cap = [Cap::Butt, Cap::Round, Cap::Square][(n(0) as usize).min(2)], + "j" => self.gs.join = [Join::Miter, Join::Round, Join::Bevel][(n(0) as usize).min(2)], + "M" => self.gs.miter = n(0), + "d" => { + let arr = args.first().map(|a| self.file.nums(a)).unwrap_or_default(); + self.gs.dash = (!arr.is_empty() && arr.iter().any(|v| *v > 0.0)).then(|| (arr, n(1))); + } + "gs" => { + if let Some(name) = args.first().and_then(Obj::name) + && let Some(g) = self.file.get_dict(res, "ExtGState").and_then(|e| self.file.get_dict(e, name)) + { + self.ext_gstate(g); + } + } + // Path construction. + "m" => { + let p = Point::new(n(0), n(1)); + self.path.move_to(p); + self.cur = Some(p); + self.start = Some(p); + } + "l" => { + let p = Point::new(n(0), n(1)); + self.ensure_start(); + self.path.line_to(p); + self.cur = Some(p); + } + "c" => { + self.ensure_start(); + let p = Point::new(n(4), n(5)); + self.path.curve_to(Point::new(n(0), n(1)), Point::new(n(2), n(3)), p); + self.cur = Some(p); + } + "v" => { + self.ensure_start(); + let c0 = self.cur.unwrap_or_default(); + let p = Point::new(n(2), n(3)); + self.path.curve_to(c0, Point::new(n(0), n(1)), p); + self.cur = Some(p); + } + "y" => { + self.ensure_start(); + let p = Point::new(n(2), n(3)); + self.path.curve_to(Point::new(n(0), n(1)), p, p); + self.cur = Some(p); + } + "h" => { + if self.cur.is_some() { + self.path.close_path(); + self.cur = self.start; + } + } + "re" => { + let (x, y, w, h) = (n(0), n(1), n(2), n(3)); + self.path.move_to((x, y)); + self.path.line_to((x + w, y)); + self.path.line_to((x + w, y + h)); + self.path.line_to((x, y + h)); + self.path.close_path(); + self.cur = Some(Point::new(x, y)); + self.start = self.cur; + } + // Painting. + "S" => self.paint(false, None, true), + "s" => { + self.path.close_path(); + self.paint(false, None, true) + } + "f" | "F" => self.paint(true, Some(FillRule::NonZero), false), + "f*" => self.paint(true, Some(FillRule::EvenOdd), false), + "B" => self.paint(true, Some(FillRule::NonZero), true), + "B*" => self.paint(true, Some(FillRule::EvenOdd), true), + "b" => { + self.path.close_path(); + self.paint(true, Some(FillRule::NonZero), true) + } + "b*" => { + self.path.close_path(); + self.paint(true, Some(FillRule::EvenOdd), true) + } + "n" => self.paint(false, None, false), + "W" => self.pending_clip = Some(FillRule::NonZero), + "W*" => self.pending_clip = Some(FillRule::EvenOdd), + // Colour. + "g" => self.gs.fill_cs = Cs::Gray, + "G" => self.gs.stroke_cs = Cs::Gray, + "rg" => self.gs.fill_cs = Cs::Rgb, + "RG" => self.gs.stroke_cs = Cs::Rgb, + "k" => self.gs.fill_cs = Cs::Cmyk, + "K" => self.gs.stroke_cs = Cs::Cmyk, + "cs" | "CS" => { + let cs = args.first().map(|a| color_space(self.file, a, Some(res))).unwrap_or(Cs::Gray); + let init = Fill::Color(cs.to_rgb(&cs.initial())); + if op == "cs" { + self.gs.fill_cs = cs; + self.gs.fill = init; + } else { + self.gs.stroke_cs = cs; + self.gs.stroke = init; + } + } + "sc" | "scn" | "SC" | "SCN" => { + let fill = op.starts_with('s'); + let cs = if fill { self.gs.fill_cs.clone() } else { self.gs.stroke_cs.clone() }; + let v = if let (Cs::Pattern, Some(name)) = (&cs, args.last().and_then(Obj::name)) { + self.pattern(res, name, pattern_base) + } else { + Fill::Color(cs.to_rgb(&nums())) + }; + if fill { + self.gs.fill = v; + } else { + self.gs.stroke = v; + } + } + "sh" => { + if let Some(name) = args.first().and_then(Obj::name) { + self.shade(res, name); + } + } + "Do" => { + if let Some(name) = args.first().and_then(Obj::name) { + self.xobject(res, name, depth); + } + } + "BT" => { + in_text = true; + self.b.skip("text"); + } + "BI" => { + // Inline image: skip to EI. + let d = lx.data; + let mut p = lx.pos; + while p + 2 < d.len() && !(d[p] == b'I' && d[p - 1] == b'D' && crate::object::is_white(d[p + 1])) { + p += 1; + } + while p + 2 < d.len() + && !(crate::object::is_white(d[p - 1]) && d[p] == b'E' && d[p + 1] == b'I' && (p + 2 == d.len() || crate::object::is_white(d[p + 2]))) + { + p += 1; + } + lx.pos = (p + 2).min(d.len()); + self.b.skip("image"); + } + "BMC" => self.marked.push(None), + "BDC" => { + let layer = match (args.first().and_then(Obj::name), args.get(1)) { + (Some("OC"), Some(Obj::Name(prop))) => self + .file + .get_dict(res, "Properties") + .and_then(|p| self.file.get_dict(p, prop)) + .map(|ocg| self.ocg_name(ocg).unwrap_or_else(|| prop.clone())), + (Some("OC"), Some(Obj::Dict(ocg))) => self.ocg_name(ocg), + _ => None, + }; + match layer { + Some(name) => { + self.marked.push(Some(self.b.depth())); + self.b.open(&name, true); + } + None => self.marked.push(None), + } + } + "EMC" => { + if let Some(Some(d)) = self.marked.pop() { + self.b.close_to(d); + } + } + _ => {} + } + // Colour operands set by the device-space shorthands. + match op.as_str() { + "g" | "rg" | "k" => self.gs.fill = Fill::Color(self.gs.fill_cs.to_rgb(&nums())), + "G" | "RG" | "K" => self.gs.stroke = Fill::Color(self.gs.stroke_cs.to_rgb(&nums())), + _ => {} + } + args.clear(); + } + } + + fn ocg_name(&self, ocg: &Dict) -> Option { + // An optional-content membership dictionary names its groups in /OCGs. + let d = match self.file.get(ocg, "OCGs") { + Some(Obj::Dict(g)) => g, + Some(Obj::Array(a)) => a.first().map(|x| self.file.resolve(x)).and_then(Obj::dict).unwrap_or(ocg), + _ => ocg, + }; + match self.file.get(d, "Name") { + Some(Obj::Str(s)) => Some(text_string(s)), + _ => None, + } + } + + fn ensure_start(&mut self) { + if self.cur.is_none() { + self.path.move_to((0.0, 0.0)); + self.cur = Some(Point::ZERO); + self.start = self.cur; + } + } + + fn ext_gstate(&mut self, g: &Dict) { + let f = self.file; + if let Some(v) = f.get_num(g, "LW") { + self.gs.lw = v; + } + if let Some(v) = f.get_num(g, "LC") { + self.gs.cap = [Cap::Butt, Cap::Round, Cap::Square][(v as usize).min(2)]; + } + if let Some(v) = f.get_num(g, "LJ") { + self.gs.join = [Join::Miter, Join::Round, Join::Bevel][(v as usize).min(2)]; + } + if let Some(v) = f.get_num(g, "ML") { + self.gs.miter = v; + } + if let Some(v) = f.get_num(g, "CA") { + self.gs.stroke_alpha = v.clamp(0.0, 1.0); + } + if let Some(v) = f.get_num(g, "ca") { + self.gs.fill_alpha = v.clamp(0.0, 1.0); + } + if let Some(Obj::Array(d)) = f.get(g, "D") { + let arr = d.first().map(|a| f.nums(a)).unwrap_or_default(); + let ph = d.get(1).and_then(|p| f.resolve(p).num()).unwrap_or(0.0); + self.gs.dash = (!arr.is_empty() && arr.iter().any(|v| *v > 0.0)).then_some((arr, ph)); + } + if f.get(g, "SMask").is_some_and(|s| !matches!(s, Obj::Name(n) if n == "None")) { + self.b.skip("soft mask"); + } + } + + fn paint_of(f: &Fill) -> Option { + match f { + Fill::Color(c) => Some(Paint::Color(*c)), + Fill::Paint(p) => Some(p.clone()), + Fill::None => None, + } + } + + fn paint(&mut self, fill: bool, rule: Option, stroke: bool) { + let path = std::mem::take(&mut self.path); + self.cur = None; + self.start = None; + let ctm = self.gs.ctm; + let fill = if fill { Self::paint_of(&self.gs.fill).map(|p| (p, self.gs.fill_alpha, rule.unwrap_or(FillRule::NonZero))) } else { None }; + let stroke = if stroke { + Self::paint_of(&self.gs.stroke).map(|p| { + // Width 0: the thinnest line the device can draw (one pixel). + let s = { (ctm.determinant().abs()).sqrt() }; + let width = if self.gs.lw <= 0.0 { if s > 0.0 { 1.0 / s } else { 1.0 } } else { self.gs.lw }; + Stroke { + paint: p, + opacity: self.gs.stroke_alpha, + width, + cap: self.gs.cap, + join: self.gs.join, + miter: self.gs.miter.max(1.0), + dash: self.gs.dash.clone(), + } + }) + } else { + None + }; + if fill.is_some() || stroke.is_some() { + self.b.fill_stroke(path.clone(), ctm, fill, stroke); + } + if let Some(r) = self.pending_clip.take() { + self.b.clip(ctm * path, r); + } + } + + /// A shading dictionary → gradient paint in shading space. + fn shading_gradient(&mut self, sh: &Dict) -> Option { + let f = self.file; + let ty = f.get_num(sh, "ShadingType").unwrap_or(0.0) as i32; + let cs = f.get(sh, "ColorSpace").map(|c| color_space(f, c, None)).unwrap_or(Cs::Rgb); + let coords = f.get(sh, "Coords").map(|c| f.nums(c)).unwrap_or_default(); + let func = f.get(sh, "Function").map(|x| Func::read(f, x)).unwrap_or(Func::Unsupported); + let dom = f.get(sh, "Domain").map(|d| f.nums(d)).filter(|d| d.len() == 2).unwrap_or_else(|| vec![0.0, 1.0]); + let kind = match (ty, coords.len()) { + (2, 4) => GradientKind::Linear { x1: coords[0], y1: coords[1], x2: coords[2], y2: coords[3] }, + (3, 6) => GradientKind::Radial { cx: coords[3], cy: coords[4], r: coords[5], fx: coords[0], fy: coords[1] }, + _ => { + self.b.skip(&format!("shading type {ty}")); + return None; + } + }; + if func == Func::Unsupported { + self.b.skip("shading function"); + } + // Sample the function: evenly, plus exactly at stitching bounds. + let mut ts: Vec = (0..=24).map(|i| i as f64 / 24.0).collect(); + for b in func.breakpoints() { + if dom[1] > dom[0] { + ts.push(((b - dom[0]) / (dom[1] - dom[0])).clamp(0.0, 1.0)); + } + } + ts.sort_by(f64::total_cmp); + ts.dedup_by(|a, b| (*a - *b).abs() < 1e-9); + let stops = ts + .into_iter() + .map(|t| { + let c = cs.to_rgb(&func.eval1(dom[0] + t * (dom[1] - dom[0]))); + (t, [c[0], c[1], c[2], 1.0]) + }) + .collect(); + Some(Gradient { kind, stops, bbox_units: false, transform: Affine::IDENTITY, spread: Spread::Pad }) + } + + fn pattern(&mut self, res: &Dict, name: &str, base: Affine) -> Fill { + let f = self.file; + let Some(p) = f.get_dict(res, "Pattern").and_then(|pp| f.get_dict(pp, name)) else { return Fill::None }; + match f.get_num(p, "PatternType").unwrap_or(0.0) as i32 { + 2 => { + let m = f.get(p, "Matrix").map(|m| affine(&f.nums(m))).unwrap_or(Affine::IDENTITY); + let Some(sh) = f.get(p, "Shading").and_then(Obj::dict) else { return Fill::None }; + let sh = sh.clone(); + match self.shading_gradient(&sh) { + // Gradient space → shape user space (shapes carry the CTM). + Some(mut g) => { + g.transform = self.gs.ctm.inverse() * base * m; + Fill::Paint(Paint::Gradient(g)) + } + None => Fill::None, + } + } + _ => { + self.b.skip("tiling pattern"); + Fill::Color([0.5; 3]) + } + } + } + + fn shade(&mut self, res: &Dict, name: &str) { + let f = self.file; + let Some(sh) = f.get_dict(res, "Shading").and_then(|s| f.get_dict(s, name)).cloned() else { return }; + let Some(g) = self.shading_gradient(&sh) else { return }; + // Fill everything (the current clip limits it): the page box in user space. + let inv = self.gs.ctm.inverse(); + let b = self.page_box; + let mut path = BezPath::new(); + for (i, (x, y)) in [(b[0], b[1]), (b[2], b[1]), (b[2], b[3]), (b[0], b[3])].into_iter().enumerate() { + let p = inv * Point::new(x, y); + if i == 0 { + path.move_to(p); + } else { + path.line_to(p); + } + } + path.close_path(); + self.b.fill_stroke(path, self.gs.ctm, Some((Paint::Gradient(g), self.gs.fill_alpha, FillRule::NonZero)), None); + } + + fn xobject(&mut self, res: &Dict, name: &str, depth: usize) { + let f = self.file; + let Some(x) = f.get_dict(res, "XObject").and_then(|xs| xs.get(name)).map(|x| f.resolve(x)) else { return }; + let Obj::Stream(d, raw) = x else { return }; + match f.get(d, "Subtype").and_then(Obj::name) { + Some("Form") if depth < 12 => { + let Some(content) = decode_stream(f, d, raw) else { return }; + let m = f.get(d, "Matrix").map(|m| affine(&f.nums(m))).unwrap_or(Affine::IDENTITY); + let form_res = f.get_dict(d, "Resources").cloned().unwrap_or_else(|| res.clone()); + self.saved.push((self.gs.clone(), self.b.depth())); + self.gs.ctm *= m; + let bb = f.get(d, "BBox").map(|b| f.nums(b)).unwrap_or_default(); + if bb.len() == 4 { + let r = kurbo::Rect::new(bb[0], bb[1], bb[2], bb[3]); + self.b.clip(self.gs.ctm * kurbo::Shape::to_path(&r, 0.1), FillRule::NonZero); + } + let base = self.gs.ctm; + let path = std::mem::take(&mut self.path); + self.run(&content, &form_res, base, depth + 1); + self.path = path; + if let Some((g, dd)) = self.saved.pop() { + self.gs = g; + self.b.close_to(dd); + } + } + Some("Image") => self.b.skip("image"), + _ => {} + } + } +} diff --git a/crates/pdf/src/ps.rs b/crates/pdf/src/ps.rs new file mode 100644 index 00000000..b9b0bff4 --- /dev/null +++ b/crates/pdf/src/ps.rs @@ -0,0 +1,1462 @@ +//! Encapsulated PostScript: a small PostScript interpreter (PostScript Language Reference, 3rd +//! edition, and the EPSF 3.0 specification) covering what exported vector EPS files use: the +//! operand and dictionary stacks, procedures and `def`, arithmetic, control flow, arrays and +//! dictionaries, the graphics state, path construction (including arcs) and painting, clipping +//! and device colours. Text is skipped, as are images (their inline data is stepped over) and +//! Level 3 smooth shading (`shfill`). + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use effectcraft_svg::{Affine, BezPath, Cap, FillRule, Join, Paint, Stroke}; +use kurbo::{PathEl, Point}; + +use crate::build::Builder; +use crate::object::{ascii85, is_white, parse_num}; + +type DictRef = Rc>>; + +#[derive(Clone)] +enum V { + Num(f64), + Bool(bool), + /// An executable name. + Name(Rc), + /// A literal name (`/name`). + Lit(Rc), + Str(Rc>>), + Proc(Rc>), + Array(Rc>>), + Dict(DictRef), + Mark, + Null, + /// `currentfile`. + File, +} + +impl V { + fn num(&self) -> Option { + match self { + V::Num(n) => Some(*n), + _ => None, + } + } + fn key(&self) -> String { + match self { + V::Name(n) | V::Lit(n) => n.to_string(), + V::Str(s) => String::from_utf8_lossy(&s.borrow()).into_owned(), + V::Num(n) => format!("{n}"), + V::Bool(b) => format!("{b}"), + _ => String::new(), + } + } + fn items(&self) -> Option> { + match self { + V::Array(a) => Some(a.borrow().clone()), + V::Proc(p) => Some(p.as_ref().clone()), + _ => None, + } + } +} + +enum Flow { + Exit, + Stop, + Quit, +} + +#[derive(Clone)] +struct GState { + ctm: Affine, + color: [f64; 3], + lw: f64, + cap: Cap, + join: Join, + miter: f64, + dash: Option<(Vec, f64)>, + path: BezPath, + cur: Option, + start: Option, + depth: usize, +} + +/// Source tokenizer (the data source for `currentfile`). +struct Src<'a> { + d: &'a [u8], + pos: usize, +} + +impl Src<'_> { + fn skip_ws(&mut self) { + while self.pos < self.d.len() { + let c = self.d[self.pos]; + if is_white(c) { + self.pos += 1; + } else if c == b'%' { + let s = self.pos; + while self.pos < self.d.len() && !matches!(self.d[self.pos], b'\n' | b'\r') { + self.pos += 1; + } + // DSC: skip embedded binary / data sections. + let line = &self.d[s..self.pos]; + if line.starts_with(b"%%BeginData:") || line.starts_with(b"%%BeginBinary:") || line.starts_with(b"%%BeginPreview") { + let end: &[u8] = if line.starts_with(b"%%BeginData:") { + b"%%EndData" + } else if line.starts_with(b"%%BeginBinary:") { + b"%%EndBinary" + } else { + b"%%EndPreview" + }; + match self.d[self.pos..].windows(end.len()).position(|w| w == end) { + Some(p) => self.pos += p, + None => self.pos = self.d.len(), + } + } + } else { + break; + } + } + } + + fn word(&mut self) -> &[u8] { + let s = self.pos; + while self.pos < self.d.len() { + let c = self.d[self.pos]; + if is_white(c) || matches!(c, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') { + break; + } + self.pos += 1; + } + &self.d[s..self.pos] + } + + /// The next token; `{ … }` comes back as a whole procedure. + fn token(&mut self, depth: usize) -> Option { + self.skip_ws(); + let c = *self.d.get(self.pos)?; + Some(match c { + b'{' => { + self.pos += 1; + let mut body = vec![]; + loop { + self.skip_ws(); + match self.d.get(self.pos) { + None => break, + Some(b'}') => { + self.pos += 1; + break; + } + _ => { + if depth > 100 { + return None; + } + match self.token(depth + 1) { + Some(t) => body.push(t), + None => break, + } + } + } + } + V::Proc(Rc::new(body)) + } + b'}' => { + self.pos += 1; + V::Null + } + b'/' => { + self.pos += 1; + let imm = self.d.get(self.pos) == Some(&b'/'); + if imm { + self.pos += 1; + } + let w = String::from_utf8_lossy(self.word()).into_owned(); + // `//name` (immediately evaluated) is treated like a plain name. + if imm { V::Name(w.into()) } else { V::Lit(w.into()) } + } + b'(' => { + self.pos += 1; + let mut out = vec![]; + let mut depth = 1; + while self.pos < self.d.len() { + let c = self.d[self.pos]; + self.pos += 1; + match c { + b'\\' => { + let e = self.d.get(self.pos).copied().unwrap_or(b'\\'); + self.pos += 1; + match e { + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'0'..=b'7' => { + let mut v = (e - b'0') as u32; + for _ in 0..2 { + match self.d.get(self.pos) { + Some(d @ b'0'..=b'7') => { + v = v * 8 + (d - b'0') as u32; + self.pos += 1; + } + _ => break, + } + } + out.push(v as u8); + } + b'\n' | b'\r' => {} + o => out.push(o), + } + } + b'(' => { + depth += 1; + out.push(c); + } + b')' => { + depth -= 1; + if depth == 0 { + break; + } + out.push(c); + } + _ => out.push(c), + } + } + V::Str(Rc::new(RefCell::new(out))) + } + b'<' if self.d.get(self.pos + 1) == Some(&b'<') => { + self.pos += 2; + V::Name("<<".into()) + } + b'>' if self.d.get(self.pos + 1) == Some(&b'>') => { + self.pos += 2; + V::Name(">>".into()) + } + b'<' if self.d.get(self.pos + 1) == Some(&b'~') => { + let s = self.pos; + while self.pos + 1 < self.d.len() && !(self.d[self.pos] == b'~' && self.d[self.pos + 1] == b'>') { + self.pos += 1; + } + self.pos = (self.pos + 2).min(self.d.len()); + V::Str(Rc::new(RefCell::new(ascii85(&self.d[s..self.pos])))) + } + b'<' => { + self.pos += 1; + let mut hex = vec![]; + while self.pos < self.d.len() && self.d[self.pos] != b'>' { + if self.d[self.pos].is_ascii_hexdigit() { + hex.push(self.d[self.pos]); + } + self.pos += 1; + } + self.pos += 1; + if hex.len() % 2 == 1 { + hex.push(b'0'); + } + let h = |c: u8| (c as char).to_digit(16).unwrap_or(0) as u8; + V::Str(Rc::new(RefCell::new(hex.chunks(2).map(|p| (h(p[0]) << 4) | h(p[1])).collect()))) + } + b'[' | b']' => { + self.pos += 1; + V::Name(if c == b'[' { "[" } else { "]" }.into()) + } + b')' | b'>' => { + self.pos += 1; + V::Null + } + _ => { + let w = self.word(); + if w.is_empty() { + self.pos += 1; + return Some(V::Null); + } + match parse_num(w) { + Some(n) => V::Num(n), + None => { + // Radix numbers (16#FF). + let s = String::from_utf8_lossy(w).into_owned(); + match s.split_once('#').and_then(|(r, v)| i64::from_str_radix(v, r.parse().ok()?).ok()) { + Some(n) => V::Num(n as f64), + None => V::Name(s.into()), + } + } + } + } + }) + } + + /// Step over inline image data that follows an `image` operator reading `currentfile`. + fn skip_image_data(&mut self) { + self.skip_ws(); + if self.d[self.pos..].starts_with(b"<~") || self.d.get(self.pos).is_some_and(|c| (b'!'..=b'u').contains(c) && !c.is_ascii_hexdigit()) { + match self.d[self.pos..].windows(2).position(|w| w == b"~>") { + Some(p) => self.pos += p + 2, + None => self.pos = self.d.len(), + } + return; + } + while self.pos < self.d.len() && (self.d[self.pos].is_ascii_hexdigit() || is_white(self.d[self.pos])) { + self.pos += 1; + } + if self.d.get(self.pos) == Some(&b'>') { + self.pos += 1; + } + } +} + +pub(crate) struct Interp { + pub b: Builder, + st: Vec, + dicts: Vec, + gs: GState, + saved: Vec, + budget: usize, + page: [f64; 4], + pub unknown: Vec, +} + +fn new_dict() -> DictRef { + Rc::new(RefCell::new(HashMap::new())) +} + +fn array(v: Vec) -> V { + V::Array(Rc::new(RefCell::new(v))) +} + +fn matrix_of(v: &V) -> Option { + let a: Vec = v.items()?.iter().filter_map(V::num).collect(); + (a.len() == 6).then(|| Affine::new([a[0], a[1], a[2], a[3], a[4], a[5]])) +} + +fn matrix_val(m: Affine) -> V { + array(m.as_coeffs().iter().map(|c| V::Num(*c)).collect()) +} + +impl Interp { + pub fn new(page: [f64; 4]) -> Interp { + Interp { + b: Builder::new(), + st: vec![], + dicts: vec![new_dict()], + gs: GState { + ctm: Affine::IDENTITY, + color: [0.0; 3], + lw: 1.0, + cap: Cap::Butt, + join: Join::Miter, + miter: 10.0, + dash: None, + path: BezPath::new(), + cur: None, + start: None, + depth: 1, + }, + saved: vec![], + budget: 8_000_000, + page, + unknown: vec![], + } + } + + pub fn run(&mut self, data: &[u8]) { + let mut src = Src { d: data, pos: 0 }; + while let Some(t) = src.token(0) { + if let Err(Flow::Quit) = self.exec_token(t, &mut src) { + break; + } + } + } + + fn pop(&mut self) -> V { + self.st.pop().unwrap_or(V::Null) + } + fn popn(&mut self) -> f64 { + self.pop().num().unwrap_or(0.0) + } + fn push(&mut self, v: V) { + if self.st.len() < 100_000 { + self.st.push(v); + } + } + + fn lookup(&self, k: &str) -> Option { + self.dicts.iter().rev().find_map(|d| d.borrow().get(k).cloned()) + } + + /// A token read from the file: procedures are pushed, names executed. + fn exec_token(&mut self, t: V, src: &mut Src) -> Result<(), Flow> { + match t { + V::Name(n) => self.exec_name(&n, src), + other => { + self.push(other); + Ok(()) + } + } + } + + /// Execute a value as an object (procedures run). + fn exec_obj(&mut self, v: V, src: &mut Src) -> Result<(), Flow> { + match v { + V::Proc(p) => { + for x in p.iter() { + match x { + V::Name(n) => self.exec_name(n, src)?, + other => self.push(other.clone()), + } + } + Ok(()) + } + V::Name(n) => self.exec_name(&n, src), + other => { + self.push(other); + Ok(()) + } + } + } + + fn exec_name(&mut self, n: &str, src: &mut Src) -> Result<(), Flow> { + if self.budget == 0 { + self.b.skip("PostScript (operation limit)"); + return Err(Flow::Quit); + } + self.budget -= 1; + if let Some(v) = self.lookup(n) { + return match v { + V::Proc(_) => self.exec_obj(v, src), + other => { + self.push(other); + Ok(()) + } + }; + } + self.builtin(n, src) + } + + fn device_scale(&self) -> f64 { + self.gs.ctm.determinant().abs().sqrt() + } + + fn user_point(&mut self) -> Point { + let y = self.popn(); + let x = self.popn(); + Point::new(x, y) + } + + fn move_to(&mut self, p: Point) { + let q = self.gs.ctm * p; + self.gs.path.move_to(q); + self.gs.cur = Some(q); + self.gs.start = Some(q); + } + fn line_to(&mut self, p: Point) { + let q = self.gs.ctm * p; + if self.gs.cur.is_none() { + self.gs.path.move_to(q); + self.gs.start = Some(q); + } + self.gs.path.line_to(q); + self.gs.cur = Some(q); + } + fn curve_to(&mut self, a: Point, b: Point, c: Point) { + let m = self.gs.ctm; + if self.gs.cur.is_none() { + self.gs.path.move_to(m * a); + } + self.gs.path.curve_to(m * a, m * b, m * c); + self.gs.cur = Some(m * c); + } + fn current_user(&self) -> Point { + self.gs.ctm.inverse() * self.gs.cur.unwrap_or_default() + } + + fn arc(&mut self, neg: bool) { + let a2 = self.popn().to_radians(); + let a1 = self.popn().to_radians(); + let r = self.popn(); + let c = self.user_point(); + let mut sweep = a2 - a1; + if neg { + while sweep > 0.0 { + sweep -= std::f64::consts::TAU; + } + } else { + while sweep < 0.0 { + sweep += std::f64::consts::TAU; + } + } + let start = Point::new(c.x + r * a1.cos(), c.y + r * a1.sin()); + if self.gs.cur.is_some() { + self.line_to(start); + } else { + self.move_to(start); + } + let arc = kurbo::Arc::new(c, (r, r), a1, sweep, 0.0); + let m = self.gs.ctm; + arc.to_cubic_beziers(0.1, |p1, p2, p3| { + self.gs.path.curve_to(m * p1, m * p2, m * p3); + self.gs.cur = Some(m * p3); + }); + } + + fn paint(&mut self, fill: Option, stroke: bool) { + let path = std::mem::take(&mut self.gs.path); + self.gs.cur = None; + self.gs.start = None; + let s = self.device_scale(); + let st = stroke.then(|| Stroke { + paint: Paint::Color(self.gs.color), + opacity: 1.0, + width: if self.gs.lw <= 0.0 { 1.0 } else { self.gs.lw * s }, + cap: self.gs.cap, + join: self.gs.join, + miter: self.gs.miter.max(1.0), + dash: self.gs.dash.as_ref().map(|(a, o)| (a.iter().map(|v| v * s).collect(), o * s)), + }); + self.b.fill_stroke(path, Affine::IDENTITY, fill.map(|r| (Paint::Color(self.gs.color), 1.0, r)), st); + } + + fn rect_path(&mut self) -> Vec { + // x y w h, or an array / string of numbers. + let top = self.pop(); + let nums: Vec = match &top { + V::Array(_) | V::Proc(_) => top.items().unwrap_or_default().iter().filter_map(V::num).collect(), + V::Num(h) => { + let w = self.popn(); + let y = self.popn(); + let x = self.popn(); + vec![x, y, w, *h] + } + _ => vec![], + }; + let m = self.gs.ctm; + nums.chunks_exact(4) + .map(|r| { + let mut p = BezPath::new(); + p.move_to(m * Point::new(r[0], r[1])); + p.line_to(m * Point::new(r[0] + r[2], r[1])); + p.line_to(m * Point::new(r[0] + r[2], r[1] + r[3])); + p.line_to(m * Point::new(r[0], r[1] + r[3])); + p.close_path(); + p + }) + .collect() + } + + fn grestore(&mut self) { + if let Some(g) = self.saved.pop() { + let d = g.depth; + self.gs = g; + self.b.close_to(d); + } + } + + fn builtin(&mut self, n: &str, src: &mut Src) -> Result<(), Flow> { + match n { + // Stack. + "pop" => { + self.pop(); + } + "exch" => { + let b = self.pop(); + let a = self.pop(); + self.push(b); + self.push(a); + } + "dup" => { + let a = self.st.last().cloned().unwrap_or(V::Null); + self.push(a); + } + "copy" => match self.pop() { + V::Num(k) => { + let k = (k.max(0.0) as usize).min(self.st.len()); + let tail: Vec = self.st[self.st.len() - k..].to_vec(); + self.st.extend(tail); + } + V::Array(dst) => { + let s = self.pop().items().unwrap_or_default(); + let mut d = dst.borrow_mut(); + for (i, v) in s.into_iter().enumerate() { + if i < d.len() { + d[i] = v; + } + } + drop(d); + self.push(V::Array(dst)); + } + other => self.push(other), + }, + "index" => { + let k = self.popn().max(0.0) as usize; + let v = if k < self.st.len() { self.st[self.st.len() - 1 - k].clone() } else { V::Null }; + self.push(v); + } + "roll" => { + let j = self.popn() as i64; + let k = (self.popn().max(0.0) as usize).min(self.st.len()); + if k > 0 { + let s = self.st.len() - k; + let r = j.rem_euclid(k as i64) as usize; + self.st[s..].rotate_right(r); + } + } + "clear" => self.st.clear(), + "count" => self.push(V::Num(self.st.len() as f64)), + "mark" | "[" | "<<" => self.push(V::Mark), + "cleartomark" => { + while let Some(v) = self.st.pop() { + if matches!(v, V::Mark) { + break; + } + } + } + "counttomark" => { + let k = self.st.iter().rev().position(|v| matches!(v, V::Mark)).unwrap_or(self.st.len()); + self.push(V::Num(k as f64)); + } + "]" => { + let k = self.st.iter().rev().position(|v| matches!(v, V::Mark)).unwrap_or(self.st.len()); + let items = self.st.split_off(self.st.len() - k); + self.st.pop(); + self.push(array(items)); + } + ">>" => { + let k = self.st.iter().rev().position(|v| matches!(v, V::Mark)).unwrap_or(self.st.len()); + let items = self.st.split_off(self.st.len() - k); + self.st.pop(); + let d = new_dict(); + for kv in items.chunks_exact(2) { + d.borrow_mut().insert(kv[0].key(), kv[1].clone()); + } + self.push(V::Dict(d)); + } + // Arithmetic. + "add" | "sub" | "mul" | "div" | "idiv" | "mod" | "atan" | "exp" | "max" | "min" => { + let b = self.popn(); + let a = self.popn(); + let v = match n { + "add" => a + b, + "sub" => a - b, + "mul" => a * b, + "div" => { + if b == 0.0 { + 0.0 + } else { + a / b + } + } + "idiv" => { + if b as i64 == 0 { + 0.0 + } else { + ((a as i64) / (b as i64)) as f64 + } + } + "mod" => { + if b as i64 == 0 { + 0.0 + } else { + ((a as i64) % (b as i64)) as f64 + } + } + "atan" => a.atan2(b).to_degrees().rem_euclid(360.0), + "exp" => a.powf(b), + "max" => a.max(b), + _ => a.min(b), + }; + self.push(V::Num(if v.is_finite() { v } else { 0.0 })); + } + "neg" | "abs" | "sqrt" | "sin" | "cos" | "ln" | "log" | "cvi" | "cvr" | "round" | "floor" | "ceiling" | "truncate" => { + let a = match self.pop() { + V::Str(s) => parse_num(&s.borrow()).unwrap_or(0.0), + v => v.num().unwrap_or(0.0), + }; + let v = match n { + "neg" => -a, + "abs" => a.abs(), + "sqrt" => a.max(0.0).sqrt(), + "sin" => a.to_radians().sin(), + "cos" => a.to_radians().cos(), + "ln" => a.ln(), + "log" => a.log10(), + "cvi" | "truncate" => a.trunc(), + "round" => (a + 0.5).floor(), + "floor" => a.floor(), + "ceiling" => a.ceil(), + _ => a, + }; + self.push(V::Num(if v.is_finite() { v } else { 0.0 })); + } + // Comparison and logic. + "eq" | "ne" => { + let b = self.pop(); + let a = self.pop(); + let eq = match (&a, &b) { + (V::Num(x), V::Num(y)) => x == y, + (V::Bool(x), V::Bool(y)) => x == y, + (V::Null, V::Null) => true, + (V::Array(x), V::Array(y)) => Rc::ptr_eq(x, y), + (V::Dict(x), V::Dict(y)) => Rc::ptr_eq(x, y), + (x, y) => !x.key().is_empty() && x.key() == y.key(), + }; + self.push(V::Bool(eq == (n == "eq"))); + } + "gt" | "ge" | "lt" | "le" => { + let b = self.popn(); + let a = self.popn(); + self.push(V::Bool(match n { + "gt" => a > b, + "ge" => a >= b, + "lt" => a < b, + _ => a <= b, + })); + } + "not" => match self.pop() { + V::Bool(b) => self.push(V::Bool(!b)), + v => self.push(V::Num(!(v.num().unwrap_or(0.0) as i64) as f64)), + }, + "and" | "or" | "xor" => { + let b = self.pop(); + let a = self.pop(); + match (a, b) { + (V::Bool(x), V::Bool(y)) => self.push(V::Bool(match n { + "and" => x && y, + "or" => x || y, + _ => x ^ y, + })), + (x, y) => { + let (x, y) = (x.num().unwrap_or(0.0) as i64, y.num().unwrap_or(0.0) as i64); + self.push(V::Num(match n { + "and" => x & y, + "or" => x | y, + _ => x ^ y, + } as f64)); + } + } + } + "true" => self.push(V::Bool(true)), + "false" => self.push(V::Bool(false)), + "null" => self.push(V::Null), + // Control. + "exec" => { + let v = self.pop(); + self.exec_obj(v, src)?; + } + "if" => { + let p = self.pop(); + if let V::Bool(true) = self.pop() { + self.exec_obj(p, src)?; + } + } + "ifelse" => { + let f = self.pop(); + let t = self.pop(); + let c = matches!(self.pop(), V::Bool(true)); + self.exec_obj(if c { t } else { f }, src)?; + } + "repeat" => { + let p = self.pop(); + let k = self.popn().clamp(0.0, 1e6) as usize; + for _ in 0..k { + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + } + } + "for" => { + let p = self.pop(); + let lim = self.popn(); + let inc = self.popn(); + let mut i = self.popn(); + let mut guard = 0; + while inc != 0.0 && ((inc > 0.0 && i <= lim) || (inc < 0.0 && i >= lim)) && guard < 1_000_000 { + self.push(V::Num(i)); + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + i += inc; + guard += 1; + } + } + "loop" => { + let p = self.pop(); + for _ in 0..1_000_000 { + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + } + } + "forall" => { + let p = self.pop(); + match self.pop() { + V::Dict(d) => { + let items: Vec<(String, V)> = d.borrow().iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + for (k, v) in items { + self.push(V::Lit(k.into())); + self.push(v); + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + } + } + V::Str(s) => { + let bytes = s.borrow().clone(); + for c in bytes { + self.push(V::Num(c as f64)); + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + } + } + a => { + for v in a.items().unwrap_or_default() { + self.push(v); + match self.exec_obj(p.clone(), src) { + Err(Flow::Exit) => break, + r => r?, + } + } + } + } + } + "exit" => return Err(Flow::Exit), + "stop" => return Err(Flow::Stop), + "stopped" => { + let p = self.pop(); + let stopped = match self.exec_obj(p, src) { + Err(Flow::Quit) => return Err(Flow::Quit), + Err(_) => true, + Ok(()) => false, + }; + self.push(V::Bool(stopped)); + } + "quit" => return Err(Flow::Quit), + "showpage" | "copypage" => {} + // Dictionaries. + "def" => { + let v = self.pop(); + let k = self.pop().key(); + if let Some(d) = self.dicts.last() { + d.borrow_mut().insert(k, v); + } + } + "store" => { + let v = self.pop(); + let k = self.pop().key(); + let d = self.dicts.iter().rev().find(|d| d.borrow().contains_key(&k)).or(self.dicts.last()).cloned(); + if let Some(d) = d { + d.borrow_mut().insert(k, v); + } + } + "load" => { + let k = self.pop().key(); + let v = self.lookup(&k).unwrap_or(V::Name(k.into())); + self.push(v); + } + "undef" => { + let k = self.pop().key(); + if let V::Dict(d) = self.pop() { + d.borrow_mut().remove(&k); + } + } + "dict" => { + self.pop(); + self.push(V::Dict(new_dict())); + } + "begin" => { + if let V::Dict(d) = self.pop() + && self.dicts.len() < 64 + { + self.dicts.push(d); + } + } + "end" => { + if self.dicts.len() > 1 { + self.dicts.pop(); + } + } + "currentdict" => { + let d = self.dicts.last().cloned().unwrap_or_else(new_dict); + self.push(V::Dict(d)); + } + "userdict" | "globaldict" | "systemdict" | "statusdict" | "errordict" | "$error" => { + let d = self.dicts[0].clone(); + self.push(V::Dict(d)); + } + "where" => { + let k = self.pop().key(); + match self.dicts.iter().rev().find(|d| d.borrow().contains_key(&k)).cloned() { + Some(d) => { + self.push(V::Dict(d)); + self.push(V::Bool(true)); + } + None => self.push(V::Bool(false)), + } + } + "known" => { + let k = self.pop().key(); + let v = match self.pop() { + V::Dict(d) => d.borrow().contains_key(&k), + _ => false, + }; + self.push(V::Bool(v)); + } + "countdictstack" => self.push(V::Num(self.dicts.len() as f64)), + "cleardictstack" => self.dicts.truncate(1), + "findresource" | "findfont" | "findencoding" => { + if n == "findresource" { + self.pop(); + } + self.pop(); + self.push(V::Dict(new_dict())); + } + "defineresource" => { + self.pop(); + let v = self.pop(); + self.pop(); + self.push(v); + } + "resourcestatus" => { + self.pop(); + self.pop(); + self.push(V::Bool(false)); + } + // Arrays and strings. + "array" => { + let k = self.popn().clamp(0.0, 1e6) as usize; + self.push(array(vec![V::Null; k])); + } + "string" => { + let k = self.popn().clamp(0.0, 1e7) as usize; + self.push(V::Str(Rc::new(RefCell::new(vec![0; k])))); + } + "length" => { + let l = match self.pop() { + V::Str(s) => s.borrow().len(), + V::Dict(d) => d.borrow().len(), + V::Name(s) | V::Lit(s) => s.len(), + v => v.items().map_or(0, |i| i.len()), + }; + self.push(V::Num(l as f64)); + } + "get" => { + let k = self.pop(); + let v = match self.pop() { + V::Dict(d) => d.borrow().get(&k.key()).cloned().unwrap_or(V::Null), + V::Str(s) => V::Num(s.borrow().get(k.num().unwrap_or(0.0) as usize).copied().unwrap_or(0) as f64), + a => a.items().and_then(|i| i.get(k.num().unwrap_or(0.0) as usize).cloned()).unwrap_or(V::Null), + }; + self.push(v); + } + "put" => { + let v = self.pop(); + let k = self.pop(); + match self.pop() { + V::Dict(d) => { + d.borrow_mut().insert(k.key(), v); + } + V::Array(a) => { + let i = k.num().unwrap_or(0.0) as usize; + if let Some(slot) = a.borrow_mut().get_mut(i) { + *slot = v; + } + } + V::Str(s) => { + let i = k.num().unwrap_or(0.0) as usize; + if let Some(slot) = s.borrow_mut().get_mut(i) { + *slot = v.num().unwrap_or(0.0) as u8; + } + } + _ => {} + } + } + "getinterval" => { + let c = self.popn().max(0.0) as usize; + let i = self.popn().max(0.0) as usize; + match self.pop() { + V::Str(s) => { + let b = s.borrow(); + let part = b.get(i.min(b.len())..(i + c).min(b.len())).unwrap_or(&[]).to_vec(); + drop(b); + self.push(V::Str(Rc::new(RefCell::new(part)))); + } + a => { + let items = a.items().unwrap_or_default(); + let part = items.get(i.min(items.len())..(i + c).min(items.len())).unwrap_or(&[]).to_vec(); + self.push(array(part)); + } + } + } + "putinterval" => { + self.pop(); + self.pop(); + self.pop(); + } + "aload" => { + let a = self.pop(); + for v in a.items().unwrap_or_default() { + self.push(v); + } + self.push(a); + } + "astore" => { + let a = self.pop(); + if let V::Array(arr) = &a { + let k = arr.borrow().len().min(self.st.len()); + let items = self.st.split_off(self.st.len() - k); + *arr.borrow_mut() = items; + } + self.push(a); + } + "cvx" => match self.pop() { + V::Array(a) => self.push(V::Proc(Rc::new(a.borrow().clone()))), + V::Lit(s) => self.push(V::Name(s)), + V::Str(s) => { + // Executable string: tokenize it as a procedure. + let bytes = s.borrow().clone(); + let mut sub = Src { d: &bytes, pos: 0 }; + let mut body = vec![]; + while let Some(t) = sub.token(0) { + body.push(t); + } + self.push(V::Proc(Rc::new(body))); + } + v => self.push(v), + }, + "cvlit" => match self.pop() { + V::Proc(p) => self.push(array(p.as_ref().clone())), + V::Name(s) => self.push(V::Lit(s)), + v => self.push(v), + }, + "cvn" => { + let k = self.pop().key(); + self.push(V::Lit(k.into())); + } + "cvs" => { + self.pop(); + let k = self.pop().key(); + self.push(V::Str(Rc::new(RefCell::new(k.into_bytes())))); + } + "xcheck" => { + let v = self.pop(); + self.push(V::Bool(matches!(v, V::Proc(_) | V::Name(_)))); + } + "type" => { + let t = match self.pop() { + V::Num(_) => "realtype", + V::Bool(_) => "booleantype", + V::Name(_) | V::Lit(_) => "nametype", + V::Str(_) => "stringtype", + V::Proc(_) | V::Array(_) => "arraytype", + V::Dict(_) => "dicttype", + V::Mark => "marktype", + V::Null => "nulltype", + V::File => "filetype", + }; + self.push(V::Name(t.into())); + } + "bind" | "readonly" | "executeonly" | "noaccess" => {} + "languagelevel" => self.push(V::Num(3.0)), + "version" | "product" => self.push(V::Str(Rc::new(RefCell::new(b"3010".to_vec())))), + "save" => { + self.saved.push(self.gs.clone()); + self.gs.depth = self.b.depth(); + self.push(V::Null); + } + "restore" => { + self.pop(); + self.grestore(); + } + "currentfile" => self.push(V::File), + "readstring" | "readhexstring" | "readline" => { + self.pop(); + self.pop(); + self.push(V::Str(Rc::new(RefCell::new(vec![])))); + self.push(V::Bool(false)); + } + "flushfile" | "closefile" | "print" | "==" | "=" | "pstack" | "setpagedevice" | "setglobal" | "setpacking" | "setshared" | "setobjectformat" + | "setsmoothness" => { + if !matches!(n, "pstack") { + self.pop(); + } + } + "currentglobal" | "currentpacking" | "currentshared" => self.push(V::Bool(false)), + "vmstatus" => { + for _ in 0..3 { + self.push(V::Num(0.0)); + } + } + // Graphics state. + "gsave" => { + let mut g = self.gs.clone(); + g.depth = self.b.depth(); + self.saved.push(g); + } + "grestore" => self.grestore(), + "grestoreall" => { + while !self.saved.is_empty() { + self.grestore(); + } + } + "initgraphics" => { + self.gs.ctm = Affine::IDENTITY; + self.gs.lw = 1.0; + self.gs.color = [0.0; 3]; + } + "setlinewidth" => self.gs.lw = self.popn(), + "currentlinewidth" => self.push(V::Num(self.gs.lw)), + "setlinecap" => self.gs.cap = [Cap::Butt, Cap::Round, Cap::Square][(self.popn().max(0.0) as usize).min(2)], + "setlinejoin" => self.gs.join = [Join::Miter, Join::Round, Join::Bevel][(self.popn().max(0.0) as usize).min(2)], + "setmiterlimit" => self.gs.miter = self.popn(), + "setdash" => { + let off = self.popn(); + let a: Vec = self.pop().items().unwrap_or_default().iter().filter_map(V::num).collect(); + self.gs.dash = (!a.is_empty() && a.iter().any(|v| *v > 0.0)).then_some((a, off)); + } + "setflat" | "setstrokeadjust" | "setoverprint" | "setrenderingintent" | "setcolorrendering" | "sethalftone" | "setscreen" | "settransfer" => { + self.pop(); + } + "setgray" => { + let g = self.popn().clamp(0.0, 1.0); + self.gs.color = [g; 3]; + } + "setrgbcolor" => { + let b = self.popn(); + let g = self.popn(); + let r = self.popn(); + self.gs.color = [r, g, b].map(|v| v.clamp(0.0, 1.0)); + } + "sethsbcolor" => { + let v = self.popn().clamp(0.0, 1.0); + let s = self.popn().clamp(0.0, 1.0); + let h = self.popn().rem_euclid(1.0) * 6.0; + let (i, f) = (h.floor() as i32, h.fract()); + let (p, q, t) = (v * (1.0 - s), v * (1.0 - s * f), v * (1.0 - s * (1.0 - f))); + self.gs.color = match i { + 0 => [v, t, p], + 1 => [q, v, p], + 2 => [p, v, t], + 3 => [p, q, v], + 4 => [t, p, v], + _ => [v, p, q], + }; + } + "setcmykcolor" => { + let k = self.popn().clamp(0.0, 1.0); + let y = self.popn().clamp(0.0, 1.0); + let m = self.popn().clamp(0.0, 1.0); + let c = self.popn().clamp(0.0, 1.0); + self.gs.color = [(1.0 - c) * (1.0 - k), (1.0 - m) * (1.0 - k), (1.0 - y) * (1.0 - k)]; + } + "setcolorspace" => { + self.pop(); + } + "setcolor" => { + // Device colour by operand count (patterns are not drawn). + let k = self.st.iter().rev().take_while(|v| matches!(v, V::Num(_))).count().min(4); + let c: Vec = self.st.split_off(self.st.len() - k).iter().filter_map(V::num).collect(); + self.gs.color = match c.len() { + 1 => [c[0]; 3], + 3 => [c[0], c[1], c[2]], + 4 => [(1.0 - c[0]) * (1.0 - c[3]), (1.0 - c[1]) * (1.0 - c[3]), (1.0 - c[2]) * (1.0 - c[3])], + _ => self.gs.color, + }; + } + "currentgray" => self.push(V::Num((self.gs.color[0] + self.gs.color[1] + self.gs.color[2]) / 3.0)), + "currentrgbcolor" => { + for c in self.gs.color { + self.push(V::Num(c)); + } + } + // Coordinates. + "matrix" => self.push(matrix_val(Affine::IDENTITY)), + "identmatrix" | "defaultmatrix" => { + self.pop(); + self.push(matrix_val(Affine::IDENTITY)); + } + "currentmatrix" => { + self.pop(); + self.push(matrix_val(self.gs.ctm)); + } + "setmatrix" => { + let m = self.pop(); + self.gs.ctm = matrix_of(&m).unwrap_or(Affine::IDENTITY); + } + "concat" => { + let m = self.pop(); + if let Some(m) = matrix_of(&m) { + self.gs.ctm *= m; + } + } + "concatmatrix" => { + self.pop(); + let b = self.pop(); + let a = self.pop(); + let (a, b) = (matrix_of(&a).unwrap_or(Affine::IDENTITY), matrix_of(&b).unwrap_or(Affine::IDENTITY)); + self.push(matrix_val(b * a)); + } + "invertmatrix" => { + self.pop(); + let a = self.pop(); + self.push(matrix_val(matrix_of(&a).unwrap_or(Affine::IDENTITY).inverse())); + } + "translate" | "scale" => { + let top = self.pop(); + let (m_out, y) = match top { + V::Array(_) => (Some(top), self.popn()), + v => (None, v.num().unwrap_or(0.0)), + }; + let x = self.popn(); + let t = if n == "translate" { Affine::translate((x, y)) } else { Affine::scale_non_uniform(x, y) }; + match m_out { + Some(_) => self.push(matrix_val(t)), + None => self.gs.ctm *= t, + } + } + "rotate" => { + let top = self.pop(); + match top { + V::Array(_) => { + let a = self.popn(); + self.push(matrix_val(Affine::rotate(a.to_radians()))); + } + v => self.gs.ctm *= Affine::rotate(v.num().unwrap_or(0.0).to_radians()), + } + } + "transform" | "itransform" | "dtransform" | "idtransform" => { + let top = self.pop(); + let (m, y) = match top { + V::Array(_) | V::Proc(_) => (matrix_of(&top).unwrap_or(Affine::IDENTITY), self.popn()), + v => (self.gs.ctm, v.num().unwrap_or(0.0)), + }; + let x = self.popn(); + let m = if n.starts_with('i') { m.inverse() } else { m }; + let p = if n.contains('d') { + let c = m.as_coeffs(); + Point::new(c[0] * x + c[2] * y, c[1] * x + c[3] * y) + } else { + m * Point::new(x, y) + }; + self.push(V::Num(p.x)); + self.push(V::Num(p.y)); + } + // Paths. + "newpath" => { + self.gs.path = BezPath::new(); + self.gs.cur = None; + self.gs.start = None; + } + "moveto" => { + let p = self.user_point(); + self.move_to(p); + } + "rmoveto" => { + let d = self.user_point(); + let c = self.current_user(); + self.move_to(Point::new(c.x + d.x, c.y + d.y)); + } + "lineto" => { + let p = self.user_point(); + self.line_to(p); + } + "rlineto" => { + let d = self.user_point(); + let c = self.current_user(); + self.line_to(Point::new(c.x + d.x, c.y + d.y)); + } + "curveto" | "rcurveto" => { + let p3 = self.user_point(); + let p2 = self.user_point(); + let p1 = self.user_point(); + if n == "rcurveto" { + let c = self.current_user(); + let o = |p: Point| Point::new(c.x + p.x, c.y + p.y); + self.curve_to(o(p1), o(p2), o(p3)); + } else { + self.curve_to(p1, p2, p3); + } + } + "closepath" => { + if self.gs.cur.is_some() { + self.gs.path.close_path(); + self.gs.cur = self.gs.start; + } + } + "arc" => self.arc(false), + "arcn" => self.arc(true), + "currentpoint" => { + let c = self.current_user(); + self.push(V::Num(c.x)); + self.push(V::Num(c.y)); + } + "pathbbox" => { + let r = kurbo::Shape::bounding_box(&self.gs.path); + let inv = self.gs.ctm.inverse(); + let (a, b) = (inv * Point::new(r.x0, r.y0), inv * Point::new(r.x1, r.y1)); + for v in [a.x.min(b.x), a.y.min(b.y), a.x.max(b.x), a.y.max(b.y)] { + self.push(V::Num(v)); + } + } + "flattenpath" | "reversepath" | "strokepath" | "initclip" => {} + "clippath" => { + let p = self.page; + self.gs.path = kurbo::Shape::to_path(&kurbo::Rect::new(p[0], p[1], p[2], p[3]), 0.1); + } + // Painting. + "fill" => self.paint(Some(FillRule::NonZero), false), + "eofill" => self.paint(Some(FillRule::EvenOdd), false), + "stroke" => self.paint(None, true), + "clip" | "eoclip" => { + let rule = if n == "clip" { FillRule::NonZero } else { FillRule::EvenOdd }; + let p = self.gs.path.clone(); + if !p.elements().iter().all(|e| matches!(e, PathEl::MoveTo(_))) { + self.b.clip(p, rule); + } + } + "rectfill" | "rectstroke" | "rectclip" => { + let paths = self.rect_path(); + if n == "rectstroke" && matches!(self.st.last(), Some(V::Array(_))) { + self.pop(); + } + let mut all = BezPath::new(); + for p in paths { + all.extend(p); + } + match n { + "rectclip" => self.b.clip(all, FillRule::NonZero), + _ => { + let saved = std::mem::replace(&mut self.gs.path, all); + self.paint(if n == "rectfill" { Some(FillRule::NonZero) } else { None }, n == "rectstroke"); + self.gs.path = saved; + } + } + } + "shfill" => { + self.pop(); + self.b.skip("smooth shading (shfill)"); + } + // Text (skipped). + "scalefont" | "makefont" | "selectfont" | "setfont" | "show" | "charpath" | "glyphshow" | "rootfont" | "setcachedevice" | "setcharwidth" => { + match n { + "scalefont" | "makefont" => { + self.pop(); + } + "selectfont" => { + self.pop(); + self.pop(); + } + "charpath" => { + self.pop(); + self.pop(); + } + "setcachedevice" => { + for _ in 0..6 { + self.pop(); + } + } + "setcharwidth" => { + self.pop(); + self.pop(); + } + "rootfont" => self.push(V::Dict(new_dict())), + _ => { + self.pop(); + } + } + if matches!(n, "show" | "charpath" | "glyphshow") { + self.b.skip("text"); + } + } + "currentfont" => self.push(V::Dict(new_dict())), + "ashow" | "widthshow" | "awidthshow" | "xshow" | "yshow" | "xyshow" | "kshow" | "cshow" => { + let k = match n { + "ashow" | "xshow" | "yshow" | "xyshow" | "kshow" | "cshow" => 3, + "widthshow" => 4, + _ => 6, + }; + for _ in 0..k.min(self.st.len()) { + self.pop(); + } + self.b.skip("text"); + } + "stringwidth" => { + self.pop(); + self.push(V::Num(0.0)); + self.push(V::Num(0.0)); + } + // Images: operands popped, inline data skipped. + "image" | "imagemask" | "colorimage" => { + let mut reads_file = false; + let check = |v: &V, rf: &mut bool| { + if matches!(v, V::File) || matches!(v, V::Proc(p) if p.iter().any(|x| matches!(x, V::Name(n) if &**n == "currentfile"))) { + *rf = true; + } + }; + if n == "colorimage" { + let ncomp = self.popn().max(1.0) as usize; + let multi = matches!(self.pop(), V::Bool(true)); + for _ in 0..if multi { ncomp.min(4) } else { 1 } { + let v = self.pop(); + check(&v, &mut reads_file); + } + for _ in 0..4 { + self.pop(); + } + } else { + match self.pop() { + V::Dict(d) => { + if let Some(ds) = d.borrow().get("DataSource") { + check(ds, &mut reads_file); + } + } + top => { + check(&top, &mut reads_file); + for _ in 0..4 { + self.pop(); + } + } + } + } + if reads_file { + src.skip_image_data(); + } + self.b.skip("image"); + } + _ => { + if self.unknown.len() < 32 && !self.unknown.iter().any(|u| u == n) { + self.unknown.push(n.to_string()); + } + } + } + Ok(()) + } +} + +/// The PostScript section of an EPS file (DOS EPS binary header handled) and its bounding box +/// (`%%HiResBoundingBox` preferred). +pub fn eps_parts(bytes: &[u8]) -> Option<(&[u8], [f64; 4])> { + let ps = if bytes.starts_with(&[0xC5, 0xD0, 0xD3, 0xC6]) && bytes.len() >= 12 { + let off = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) as usize; + let len = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize; + bytes.get(off..off.checked_add(len)?.min(bytes.len()))? + } else { + bytes + }; + let find_box = |key: &[u8]| -> Option<[f64; 4]> { + let mut from = 0; + while let Some(p) = ps[from..].windows(key.len()).position(|w| w == key) { + let s = from + p + key.len(); + let e = ps[s..].iter().position(|c| matches!(c, b'\n' | b'\r')).map_or(ps.len(), |q| s + q); + let v: Vec = String::from_utf8_lossy(&ps[s..e]).split_whitespace().filter_map(|t| t.parse().ok()).collect(); + if v.len() == 4 && v[2] > v[0] && v[3] > v[1] { + return Some([v[0], v[1], v[2], v[3]]); + } + from = s; + } + None + }; + let bbox = find_box(b"%%HiResBoundingBox:").or_else(|| find_box(b"%%BoundingBox:"))?; + Some((ps, bbox)) +} diff --git a/crates/pdf/src/tests.rs b/crates/pdf/src/tests.rs new file mode 100644 index 00000000..b4e91f2b --- /dev/null +++ b/crates/pdf/src/tests.rs @@ -0,0 +1,150 @@ +//! Test files are generated here (no third-party samples). + +use super::*; + +use crate::write::pdf; + +const CONTENT: &str = "/OC /MC0 BDC\n0 1 0 rg 0 0 200 100 re f\nEMC\n\ +/OC /MC1 BDC\nq 10 10 80 80 re W n\n1 0 0 rg 0 0 50 100 re f\nQ\n\ +/Pattern cs /P0 scn 100 0 100 50 re f\n\ +0 0 1 RG 4 w 120 80 m 180 80 l S\n\ +BT /F1 12 Tf (ignored) Tj ET\nEMC\n"; + +pub(crate) fn sample_pdf(object_stream: bool) -> Vec { + let page = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 100] /Resources << /Properties << /MC0 5 0 R /MC1 6 0 R >> /Pattern << /P0 7 0 R >> >> /Contents 4 0 R >>"; + let pattern = "<< /PatternType 2 /Shading << /ShadingType 2 /ColorSpace /DeviceRGB /Coords [100 0 200 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> /Extend [true true] >> >>"; + let mut objs = vec![ + (1, "<< /Type /Catalog /Pages 2 0 R >>".to_string(), None), + (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), None), + (3, page.to_string(), None), + (4, "<< >>".to_string(), Some(CONTENT.as_bytes().to_vec())), + (7, pattern.to_string(), None), + ]; + let ocg5 = "<< /Type /OCG /Name (Background) >>"; + let ocg6 = "<< /Type /OCG /Name >>"; // "Art" in UTF-16BE + if object_stream { + let body = format!("{ocg5} {ocg6}"); + let head = format!("5 0 6 {} ", ocg5.len() + 1); + let data = format!("{head}{body}"); + objs.push((8, format!("<< /Type /ObjStm /N 2 /First {} >>", head.len()), Some(data.into_bytes()))); + } else { + objs.push((5, ocg5.to_string(), None)); + objs.push((6, ocg6.to_string(), None)); + } + pdf(&objs, 1) +} + +fn px(doc: &Doc, x: i64, y: i64) -> [f32; 4] { + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(doc, w, h, 1.0); + img.get(x, y) +} + +#[test] +fn pdf_paths_clip_gradient_and_layers() { + for objstm in [false, true] { + let bytes = sample_pdf(objstm); + assert_eq!(sniff(&bytes), Some(Format::Pdf)); + assert_eq!(page_count(&bytes), 1); + let doc = parse(&bytes).unwrap(); + assert_eq!((doc.width, doc.height), (200.0, 100.0)); + assert_eq!(layer_names(&doc), vec!["Background", "Art"], "object stream {objstm}"); + assert!(doc.skipped.contains(&"text".to_string())); + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + let at = |x: i64, y: i64| img.get(x, y); + // Red rectangle clipped to x ≥ 10, over the green background. + assert!(at(30, 50)[0] > 0.99 && at(30, 50)[1] < 0.01, "{:?}", at(30, 50)); + assert!(at(5, 50)[1] > 0.99 && at(5, 50)[0] < 0.01, "clipped out: {:?}", at(5, 50)); + assert!(at(70, 50)[1] > 0.99, "{:?}", at(70, 50)); + // Axial shading, red → blue left to right (in the lower half: PDF y is up). + let (l, r) = (at(110, 75), at(190, 75)); + assert!(l[0] > 0.8 && l[2] < 0.2 && r[2] > 0.8 && r[0] < 0.2, "{l:?} {r:?}"); + assert!(at(150, 25)[1] > 0.99, "the gradient stays in its rectangle: {:?}", at(150, 25)); + // Stroked line (PDF y = 80 → 20 px from the top). + assert!(at(150, 20)[2] > 0.99 && at(150, 20)[1] < 0.01, "{:?}", at(150, 20)); + // One layer at a time. + let bg = layer_doc(&doc, 0); + assert!(px(&bg, 30, 50)[1] > 0.99); + let art = layer_doc(&doc, 1); + assert_eq!(px(&art, 5, 50)[3], 0.0); + // Continuous rasterisation: twice the size stays sharp at the clip edge. + let big = effectcraft_svg::rasterize(&doc, 400, 200, 2.0); + assert!(big.get(21, 100)[0] > 0.99 && big.get(18, 100)[1] > 0.99); + } +} + +#[test] +fn pdf_forms_rotation_cmyk_and_alpha() { + let content = "q 2 0 0 2 0 0 cm /Fm0 Do Q\n/GS0 gs 0 0 0 1 k 0 0 10 10 re f\n"; + let form = "0 1 1 0 k 0 0 20 20 re f"; + let objs = vec![ + (1, "<< /Type /Catalog /Pages 2 0 R >>".to_string(), None), + (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 /Rotate 90 >>".to_string(), None), + ( + 3, + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 50] /Resources << /XObject << /Fm0 5 0 R >> /ExtGState << /GS0 << /ca 0.5 >> >> >> /Contents 4 0 R >>" + .to_string(), + None, + ), + (4, "<< >>".to_string(), Some(content.as_bytes().to_vec())), + (5, "<< /Type /XObject /Subtype /Form /BBox [0 0 10 10] >>".to_string(), Some(form.as_bytes().to_vec())), + ]; + let doc = parse(&pdf(&objs, 1)).unwrap(); + // Rotated a quarter turn: 50 × 100. + assert_eq!((doc.width, doc.height), (50.0, 100.0)); + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + // The form (CMYK red, clipped to its 10×10 box, scaled ×2) covers PDF (0..20, 0..20); the + // half-transparent black square covers PDF (0..10, 0..10). /Rotate 90: PDF (x, y) → (y, x). + let red = img.get(15, 15); + assert!(red[0] > 0.99 && red[1] < 0.01 && red[3] > 0.99, "{red:?}"); + let dark = img.get(5, 5); + assert!((dark[0] - 0.5).abs() < 0.02 && dark[3] > 0.99, "{dark:?}"); + assert_eq!(img.get(30, 30)[3], 0.0, "form clipped to its bounding box"); +} + +const EPS: &str = "%!PS-Adobe-3.0 EPSF-3.0\n%%BoundingBox: 0 0 100 100\n%%EndComments\n\ +/m {moveto} bind def /l {lineto} bind def\n\ +/cm { 6 array astore concat } bind def\n\ +/box { 4 dict begin /h exch def /w exch def /y exch def /x exch def x y m w 0 rlineto 0 h rlineto w neg 0 rlineto closepath end } bind def\n\ +1 0 0 setrgbcolor\n10 10 30 30 box fill\n\ +gsave 0 0 1 setrgbcolor 70 50 20 0 360 arc fill grestore\n\ +gsave 1 0 0 1 50 0 cm 0 1 0 setrgbcolor 0 0 10 10 box fill grestore\n\ +0 setgray 2 setlinewidth 0 90 m 100 90 l stroke\n\ +/Helvetica findfont 12 scalefont setfont 5 5 moveto (text) show\n\ +showpage\n%%EOF\n"; + +#[test] +fn eps_postscript_subset() { + let doc = parse(EPS.as_bytes()).unwrap(); + assert_eq!((doc.width, doc.height), (100.0, 100.0)); + assert!(doc.skipped.contains(&"text".to_string())); + let at = |x, y| px(&doc, x, y); + assert!(at(25, 75)[0] > 0.99 && at(25, 75)[3] > 0.99, "red box {:?}", at(25, 75)); + assert!(at(70, 50)[2] > 0.99, "blue disc {:?}", at(70, 50)); + assert!(at(55, 95)[1] > 0.99, "translated green box {:?}", at(55, 95)); + let line = at(50, 10); + assert!(line[3] > 0.99 && line[0] < 0.01, "black line {line:?}"); + assert_eq!(at(95, 70)[3], 0.0); + // DOS EPS binary header (with a fake TIFF preview after the PostScript). + let ps = EPS.as_bytes(); + let mut dos = vec![0xC5, 0xD0, 0xD3, 0xC6]; + dos.extend_from_slice(&30u32.to_le_bytes()); + dos.extend_from_slice(&(ps.len() as u32).to_le_bytes()); + dos.extend_from_slice(&[0; 18]); + dos.extend_from_slice(ps); + dos.extend_from_slice(b"II*\0 not a real preview"); + let d2 = parse(&dos).unwrap(); + assert_eq!(d2.root.children.len(), doc.root.children.len()); + assert_eq!(codec("art.eps", &dos), Some("EPS")); + assert_eq!(codec("art.ai", &sample_pdf(false)), Some("AI")); + assert_eq!(parse(b"hello"), Err(Error::NotVector)); +} + +#[test] +fn filters_decode() { + assert_eq!(object::ascii85(b"<~87cURD]i,\"Ebo80~>"), b"Hello World!".to_vec()); + let z = miniz_oxide::deflate::compress_to_vec_zlib(b"abc", 6); + assert_eq!(object::inflate(&z).unwrap(), b"abc"); +} diff --git a/crates/pdf/src/write.rs b/crates/pdf/src/write.rs new file mode 100644 index 00000000..9e54bdc3 --- /dev/null +++ b/crates/pdf/src/write.rs @@ -0,0 +1,32 @@ +//! A minimal PDF writer for generating test files (EffectCraft does not export PDF). + +/// Numbered objects `(number, dictionary, stream data)` (streams Flate-compressed), a +/// cross-reference table and a trailer naming object `root` as the catalog. +pub fn pdf(objects: &[(u32, String, Option>)], root: u32) -> Vec { + let mut out = b"%PDF-1.7\n%\xE2\xE3\xCF\xD3\n".to_vec(); + let mut offsets = vec![]; + for (n, dict, stream) in objects { + offsets.push((*n, out.len())); + match stream { + Some(data) => { + let z = miniz_oxide::deflate::compress_to_vec_zlib(data, 6); + let d = format!("{} /Filter /FlateDecode /Length {} >>", dict.trim_end().trim_end_matches(">>"), z.len()); + out.extend_from_slice(format!("{n} 0 obj\n{d}\nstream\n").as_bytes()); + out.extend_from_slice(&z); + out.extend_from_slice(b"\nendstream\nendobj\n"); + } + None => out.extend_from_slice(format!("{n} 0 obj\n{dict}\nendobj\n").as_bytes()), + } + } + let xref = out.len(); + let max = objects.iter().map(|o| o.0).max().unwrap_or(0); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", max + 1).as_bytes()); + for i in 1..=max { + match offsets.iter().find(|o| o.0 == i) { + Some((_, off)) => out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes()), + None => out.extend_from_slice(b"0000000000 65535 f \n"), + } + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root {root} 0 R >>\nstartxref\n{xref}\n%%EOF\n", max + 1).as_bytes()); + out +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 470c4edc..5f90c291 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -826,9 +826,9 @@ pub struct Proxy { pub enabled: bool, } -/// A layer of a layered still (a Photoshop document) used as footage -/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// A layer of a layered still (a Photoshop document, or a PDF / Illustrator / EPS file) used as +/// footage (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct SourceLayer { /// Layer record index in the file (bottom of the stack = 0). pub index: u32, @@ -836,6 +836,10 @@ pub struct SourceLayer { /// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size. #[serde(default)] pub layer_size: bool, + /// A Photoshop smart object: the unique id of its embedded file (linked layer data), which + /// is the footage instead of the layer's pixels. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub embedded: Option, } fn one() -> u32 { @@ -852,6 +856,7 @@ pub struct Solid { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] +#[allow(clippy::large_enum_variant)] pub enum ItemKind { Folder, Comp(Arc), diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index fd9be466..e2c91e60 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -66,7 +66,7 @@ pub trait FootageSource: Send + Sync { /// Footage that can be rasterised at any scale (SVG). pub fn is_vector_footage(f: &Footage) -> bool { - f.codec == "SVG" + matches!(f.codec.as_str(), "SVG" | "PDF" | "AI" | "EPS") } /// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]). diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index 22b5fead..0536480c 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -170,6 +170,9 @@ pub struct Group { pub transform: Affine, pub opacity: f64, pub children: Vec, + /// Clipping paths in the group's space (after `transform`); the children show only where + /// every one of them covers (PDF / EPS clipping; SVG `clipPath` is not read yet). + pub clip: Vec<(BezPath, FillRule)>, } #[derive(Clone, Debug, PartialEq)] @@ -666,7 +669,7 @@ impl<'a, 'i> Parser<'a, 'i> { for c in n.children() { self.node(c, &st, &mut children, depth + 1); } - out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children })); + out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children, clip: vec![] })); } "svg" => { // Nested viewport. @@ -685,7 +688,7 @@ impl<'a, 'i> Parser<'a, 'i> { self.node(c, &st, &mut children, depth + 1); } (self.vw, self.vh) = saved; - out.push(Node::Group(Group { name: Self::element_name(n), transform: Affine::translate((x, y)) * vb, opacity, children })); + out.push(Node::Group(Group { name: Self::element_name(n), transform: Affine::translate((x, y)) * vb, opacity, children, clip: vec![] })); } "use" => { let Some(target) = self.href(n) else { return }; @@ -706,11 +709,11 @@ impl<'a, 'i> Parser<'a, 'i> { for c in target.children() { self.node(c, &sst, &mut inner, depth + 1); } - children.push(Node::Group(Group { name: Self::element_name(target), transform: vb, opacity: o, children: inner })); + children.push(Node::Group(Group { name: Self::element_name(target), transform: vb, opacity: o, children: inner, clip: vec![] })); } else { self.node(target, &st, &mut children, depth + 1); } - out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children })); + out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children, clip: vec![] })); } "path" | "rect" | "circle" | "ellipse" | "line" | "polyline" | "polygon" => { if !st.visible { @@ -750,7 +753,7 @@ impl<'a, 'i> Parser<'a, 'i> { self.node(c, &st, &mut children, depth + 1); } if !children.is_empty() { - out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children })); + out.push(Node::Group(Group { name: Self::element_name(n), transform, opacity, children, clip: vec![] })); } } } @@ -942,7 +945,7 @@ pub fn parse(bytes: &[u8]) -> Result { p.node(c, &rst, &mut children, 1); } let transform = viewbox_transform(root, dw, dh); - Ok(Doc { width: dw, height: dh, root: Group { name: "svg".into(), transform, opacity, children }, skipped: std::mem::take(&mut p.skipped) }) + Ok(Doc { width: dw, height: dh, root: Group { name: "svg".into(), transform, opacity, children, clip: vec![] }, skipped: std::mem::take(&mut p.skipped) }) } #[cfg(test)] diff --git a/crates/svg/src/raster.rs b/crates/svg/src/raster.rs index 2cfb3dd3..de82695b 100644 --- a/crates/svg/src/raster.rs +++ b/crates/svg/src/raster.rs @@ -31,6 +31,27 @@ fn draw_group(g: &Group, m: Affine, opacity: f64, dst: &mut Image) { if o <= 0.0 { return; } + if !g.clip.is_empty() { + // Draw the children on their own, keep what every clip path covers, composite. + let mut tmp = Image::new(dst.width, dst.height); + for c in &g.children { + draw_node(c, m, 1.0, &mut tmp); + } + for (path, rule) in &g.clip { + let rule = match rule { + FillRule::NonZero => effectcraft_path::FillRule::NonZero, + FillRule::EvenOdd => effectcraft_path::FillRule::EvenOdd, + }; + let cov = effectcraft_path::fill_coverage(std::slice::from_ref(path), &mat3(m), dst.width, dst.height, rule); + tmp.data.par_iter_mut().zip(cov.data.par_iter()).for_each(|(p, c)| { + for v in p.iter_mut() { + *v *= c; + } + }); + } + over(dst, &tmp, o as f32); + return; + } if g.opacity < 1.0 && g.children.len() > 1 { let mut tmp = Image::new(dst.width, dst.height); for c in &g.children { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index eed7e639..acf5a544 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -813,7 +813,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 21928b90..ad904eba 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -35,6 +35,7 @@ const LAYERS: &[(&str, u8)] = &[ ("effects", 2), ("track", 2), ("svg", 2), + ("pdf", 2), ("model", 2), ("render", 3), ("media", 3), @@ -58,6 +59,7 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("path", "keyframe"), ("path", "raster"), ("text", "path"), + ("pdf", "svg"), ("effects", "project"), ("effects", "text"), ("effects", "path"), From 7352b624d462bdb77f6f1f1c7a45e1377511b652 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:18:05 -0400 Subject: [PATCH 24/36] M13.2: PSD smart objects (SoLd/PlLd + lnk2 embedded files) import as placed footage of the embedded document or image Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/psd_import.rs | 60 ++++++++++- crates/host/tests/vector_import.rs | 37 +++++++ crates/media/src/layered.rs | 11 ++ crates/media/src/pool.rs | 5 +- crates/psd/README.md | 8 +- crates/psd/src/lib.rs | 163 ++++++++++++++++++++++++++++- crates/psd/src/tests.rs | 31 ++++++ crates/psd/src/write.rs | 57 +++++++++- 8 files changed, 365 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs index 2f012eee..555dd97b 100644 --- a/crates/engine/src/psd_import.rs +++ b/crates/engine/src/psd_import.rs @@ -6,7 +6,9 @@ //! so their blend modes act on the layers below, like Photoshop); type layers become editable //! text layers; layer masks are baked into the layer's alpha and vector masks become masks; //! layer effects map to Layer Styles; adjustment layers become adjustment layers with the -//! matching effect (best effort); clipped layers preserve the transparency underneath. +//! matching effect (best effort); clipped layers preserve the transparency underneath. Smart +//! objects whose file is embedded become footage of that file (an embedded Photoshop document's +//! merged image, or the embedded image) placed with the smart object's transform. use std::sync::Arc; @@ -164,6 +166,60 @@ fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> ( (id, (w, h)) } +/// Size of an embedded file's image: the smart object's recorded size, else the embedded +/// document's header (Photoshop, PNG). +fn embedded_size(so: &effectcraft_psd::SmartObject, data: &[u8]) -> Option<(u32, u32)> { + if let Some([w, h]) = so.size.filter(|s| s[0] >= 1.0 && s[1] >= 1.0) { + return Some((w.round() as u32, h.round() as u32)); + } + if effectcraft_psd::is_psd(data) && data.len() >= 22 { + let h = u32::from_be_bytes([data[14], data[15], data[16], data[17]]); + let w = u32::from_be_bytes([data[18], data[19], data[20], data[21]]); + return Some((w, h)); + } + if data.starts_with(b"\x89PNG") && data.len() >= 24 { + let w = u32::from_be_bytes([data[16], data[17], data[18], data[19]]); + let h = u32::from_be_bytes([data[20], data[21], data[22], data[23]]); + return Some((w, h)); + } + None +} + +/// A smart object layer as footage of its embedded file, placed by the smart object's +/// transform. `None` when the file is not embedded (linked externally) or unreadable. +fn smart_object_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, index: usize, name: &str) -> Option { + let pl = &cx.psd.layers[index]; + let so = pl.smart_object.clone()?; + let data = cx.psd.linked_data(&so.uuid)?; + let (w, h) = embedded_size(&so, data).filter(|s| s.0 > 0 && s.1 > 0)?; + let file = cx.psd.linked.iter().find(|f| f.uuid == so.uuid).map(|f| f.name.clone()).unwrap_or_default(); + let f = Footage { + path: cx.path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: cx.rate, + has_video: true, + alpha: AlphaMode::Straight, + loop_count: 1, + codec: if effectcraft_psd::is_psd(data) { "PSD".into() } else { "PSD (embedded image)".into() }, + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: true, embedded: Some(so.uuid.clone()) }), + ..Default::default() + }; + let label = if file.is_empty() { name.to_string() } else { file }; + let id = proj.add_item(&format!("{label} (Smart Object)"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); + cx.items.push(id); + let mut l = build::layer(proj, comp, name, LayerSource::Footage { item: id }, (w, h), None); + let (c, sx, sy, rot) = so.placement(w as f64, h as f64); + set_vec2(&mut l, "transform/position", c); + set_vec2(&mut l, "transform/scale", [sx * 100.0, sy * 100.0]); + if rot.abs() > 1e-9 { + set_scalar(&mut l, "transform/rotation", rot); + } + Some(l) +} + fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option { let (index, children) = match n { Node::Layer(i) => (*i, None), @@ -183,6 +239,8 @@ fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Optio let mut l = adjustment_layer(proj, comp, &name); add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name); l + } else if let Some(l) = pl.smart_object.as_ref().and_then(|_| smart_object_layer(proj, cx, comp, index, &name)) { + l } else if pl.has_pixels() || pl.fill.is_some() { let (item, size) = footage_item(proj, cx, index, &name); let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None); diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index 7b516a2a..9b77ccae 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -281,6 +281,43 @@ fn pdf_and_ai_footage_layers_and_shapes_from_vector_layer() { assert!(p[2] > p[0], "only sky under the art {p:?}"); } +#[test] +fn psd_smart_object_imports_its_embedded_document() { + // The embedded document: red left half, green right half (20×10). + let mut inner = WDoc::new(20, 10); + inner.layers = + vec![WLayer::solid("Red", Rect::new(0, 0, 10, 10), [1.0, 0.0, 0.0, 1.0]), WLayer::solid("Green", Rect::new(10, 0, 10, 10), [0.0, 1.0, 0.0, 1.0])]; + let inner_bytes = write(&inner); + let mut d = WDoc::new(64, 48); + // Placed at 2× and turned a quarter clockwise: the content's top-left corner at (40, 4). + let quad = [[40.0, 4.0], [40.0, 44.0], [20.0, 44.0], [20.0, 4.0]]; + d.layers = vec![ + WLayer::solid("Back", Rect::new(0, 0, 64, 48), [0.0, 0.0, 0.0, 1.0]), + // Placeholder pixels (grey) that the embedded file replaces. + WLayer::solid("Placed", Rect::new(20, 4, 20, 40), [0.5, 0.5, 0.5, 1.0]).with_block(smart_object_block("so-1", quad, [20.0, 10.0])), + ]; + d.linked = vec![("so-1".into(), "inner.psd".into(), *b"8BPS", inner_bytes)]; + let path = tmp("smart.psd"); + std::fs::write(&path, write(&d)).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let comp = s.project.comp(cid).unwrap(); + let placed = comp.layers.iter().find(|l| l.name == "Placed").unwrap(); + let effectcraft_project::LayerSource::Footage { item } = placed.source else { panic!("footage layer") }; + let effectcraft_project::ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + assert_eq!((f.width, f.height), (20, 10)); + assert_eq!(f.layer.as_ref().unwrap().embedded.as_deref(), Some("so-1")); + assert!(s.project.item(item).unwrap().name.contains("inner.psd")); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + // Rotated a quarter clockwise: the red half is on top (rows 4–24), green below. + let (red, green) = (img.get(30, 14), img.get(30, 34)); + assert!(red[0] > 0.95 && red[1] < 0.05, "{red:?}"); + assert!(green[1] > 0.95 && green[0] < 0.05, "{green:?}"); + assert!(img.get(10, 24)[0] < 0.01 && img.get(10, 24)[3] > 0.99, "background outside"); +} + #[test] fn eps_footage_imports() { let eps = diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs index 111140a4..b2331d81 100644 --- a/crates/media/src/layered.rs +++ b/crates/media/src/layered.rs @@ -67,6 +67,17 @@ pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result> { if effectcraft_psd::is_psd(bytes) { let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + // A smart object's embedded file: an embedded document's merged image, or an image. + if let Some(uuid) = footage.layer.as_ref().and_then(|l| l.embedded.as_deref()) { + let data = psd.linked_data(uuid).ok_or_else(|| MediaError::Decode(format!("{path}: no embedded file {uuid}")))?; + if effectcraft_psd::is_psd(data) { + let inner = effectcraft_psd::Psd::parse(data.to_vec()).map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?; + let px = inner.composite().map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?; + return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); + } + let img = image::load_from_memory(data).map_err(|e| MediaError::Decode(format!("{path} (smart object): {e}")))?; + return Ok(Some(crate::convert::dynamic_to_image(&img, op))); + } let px = match &footage.layer { Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size), None => psd.composite(), diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index ed6521c9..a0864f08 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -386,9 +386,10 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } - // A layer of a layered still is keyed by its layer index and size mode. + // A layer of a layered still is keyed by its layer index and size mode (smart objects + // by their embedded file). FootageKind::Still if footage.layer.is_some() => { - let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64); + let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64 + if l.embedded.is_some() { 1 << 40 } else { 0 }); Loc { key: key(&footage.path, l), media_t: None } } FootageKind::Still | FootageKind::Sequence | FootageKind::Model | FootageKind::Data => Loc { key: key(&footage.path, 0), media_t: None }, diff --git a/crates/psd/README.md b/crates/psd/README.md index e238b9b2..b0b19ba3 100644 --- a/crates/psd/README.md +++ b/crates/psd/README.md @@ -8,10 +8,14 @@ Implemented from Adobe's public **Adobe Photoshop File Formats Specification** ( edition): file header, colour mode data, image resources, layer and mask information (layer records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`, `lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`, -`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) +`Lr16`/`Lr32`, smart objects: `SoLd`/`SoLE`/`PlLd` placement and `lnk2`/`lnkD`/`lnk3`/`lnkE` +linked layer data with embedded files), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit; Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes. Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and -justification of the first style run. No Adobe code, sample files or assets were used; test +justification of the first style run. Smart objects import as footage of their embedded file +(a Photoshop document's merged image, or an image) placed by the smart object's corner quad +(position, scale, rotation; perspective and warps are not applied). Externally linked files +fall back to the layer's rendered pixels. No Adobe code, sample files or assets were used; test fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)). diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 885ab3c2..61cadc7d 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -204,6 +204,46 @@ pub struct SubPath { pub knots: Vec<[[f64; 2]; 3]>, } +/// A placed layer (smart object): `SoLd` / `SoLE` (placed layer data) or the older `PlLd`. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SmartObject { + /// Unique id of the linked file ([`Psd::linked`]) the layer shows. + pub uuid: String, + /// Where the content's corners sit in document pixels: top-left, top-right, bottom-right, + /// bottom-left. + pub quad: [[f64; 2]; 4], + /// The content's size in pixels (`Sz `), when recorded. + pub size: Option<[f64; 2]>, +} + +impl SmartObject { + /// The quad as an affine placement of a `w`×`h` content rectangle: (centre, scale x, scale + /// y, rotation in degrees). Perspective and skew are dropped. + pub fn placement(&self, w: f64, h: f64) -> ([f64; 2], f64, f64, f64) { + let q = self.quad; + let centre = [(q[0][0] + q[1][0] + q[2][0] + q[3][0]) / 4.0, (q[0][1] + q[1][1] + q[2][1] + q[3][1]) / 4.0]; + let top = [q[1][0] - q[0][0], q[1][1] - q[0][1]]; + let left = [q[3][0] - q[0][0], q[3][1] - q[0][1]]; + let sx = top[0].hypot(top[1]) / w.max(1e-9); + let sy = left[0].hypot(left[1]) / h.max(1e-9); + // Mirrored content (left edge turning the other way) keeps a negative y scale. + let cross = top[0] * left[1] - top[1] * left[0]; + (centre, sx, if cross < 0.0 { -sy } else { sy }, top[1].atan2(top[0]).to_degrees()) + } +} + +/// A file embedded in (or linked from) the document: linked layer data (`lnk2`, `lnkD`, +/// `lnk3`, `lnkE`). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LinkedFile { + pub uuid: String, + pub name: String, + /// File type code (`8BPS`, `PNGf`, `JPEG`…), may be blank. + pub file_type: String, + /// Embedded data (absent for externally linked files). + pub data: Option>, +} + #[derive(Clone, Debug, Default, PartialEq)] pub struct Layer { /// Index in [`Psd::layers`] (file order: bottom of the stack first). @@ -230,6 +270,8 @@ pub struct Layer { /// Layer effects (`lfx2` descriptor). pub effects: Option, pub vector_mask: Option, + /// Smart object placement. + pub smart_object: Option, /// Keys of every additional-information block the layer carries. pub keys: Vec<[u8; 4]>, } @@ -277,6 +319,8 @@ pub struct Psd { pub merged_alpha: bool, /// Indexed colour palette (256 R, then G, then B). pub palette: Vec, + /// Files embedded for smart objects. + pub linked: Vec, data: Arc<[u8]>, merged_offset: usize, } @@ -314,7 +358,20 @@ impl Psd { let n = r.u32()? as usize; r.skip(n)?; // Layer and mask information. - let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 }; + let mut psd = Psd { + psb, + width, + height, + channels, + depth, + mode, + layers: vec![], + merged_alpha: false, + palette, + linked: vec![], + data: data.clone(), + merged_offset: 0, + }; let lm_len = r.length(psb)?; let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?; if lm_len > 0 { @@ -330,6 +387,9 @@ impl Psd { } // Global additional layer information (16/32-bit layer info lives here). while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) { + if matches!(&key, b"lnk2" | b"lnkD" | b"lnk3" | b"lnkE") { + psd.linked_files(&data, start, end); + } if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start { // The block holds the layer info, normally without its length field; accept // both layouts. @@ -348,6 +408,67 @@ impl Psd { Ok(psd) } + /// The embedded bytes of linked file `uuid`. + pub fn linked_data(&self, uuid: &str) -> Option<&[u8]> { + let f = self.linked.iter().find(|f| f.uuid == uuid)?; + self.data.get(f.data.clone()?) + } + + /// Linked layer data records (spec: "Linked Layer"): length, kind (`liFD` embedded data, + /// `liFE` external file, `liFA` alias), version, unique id, original file name, type, + /// creator, data length, optional descriptors, then the data. A damaged record ends the list. + fn linked_files(&mut self, data: &[u8], start: usize, end: usize) { + let mut pos = start; + while pos + 8 <= end { + let mut r = Reader::at(&data[..end], pos); + let Ok(len) = r.u64() else { break }; + let body = r.pos; + let Some(rec_end) = body.checked_add(len as usize).filter(|e| *e <= end) else { break }; + let mut r = Reader::at(&data[..rec_end], body); + let rec = (|| -> Result { + let kind = r.tag()?; + let version = r.u32()?; + let n = r.u8()? as usize; + let uuid = String::from_utf8_lossy(r.bytes(n)?).into_owned(); + let name = descriptor::read_unicode(&mut r)?; + let file_type = String::from_utf8_lossy(&r.tag()?).trim().to_string(); + let _creator = r.tag()?; + let dlen = r.u64()? as usize; + if r.u8()? != 0 { + let _v = r.u32()?; + descriptor::read_descriptor(&mut r)?; + } + let mut out = LinkedFile { uuid, name, file_type, data: None }; + match &kind { + b"liFD" => { + let s = r.pos; + r.skip(dlen)?; + out.data = Some(s..s + dlen); + } + b"liFE" => { + let _v = r.u32()?; + descriptor::read_descriptor(&mut r)?; + if version > 3 { + r.skip(4 + 4 + 8)?; + } + let _size = r.u64()?; + if version > 2 && r.remaining() >= dlen && dlen > 0 { + let s = r.pos; + out.data = Some(s..s + dlen); + } + } + _ => {} + } + Ok(out) + })(); + match rec { + Ok(f) => self.linked.push(f), + Err(_) => break, + } + pos = rec_end.div_ceil(4) * 4; + } + } + fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> { let mut r = Reader::at(&data[..end], start); let count = r.i16()?; @@ -522,6 +643,46 @@ impl Psd { l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned())); } } + b"SoLd" | b"SoLE" => { + let _id = r.tag()?; + let _ver = r.u32()?; + let _dver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + let uuid = d.text("Idnt").unwrap_or_default().to_string(); + let nums = |k: &str| -> Option> { + match d.get(k)? { + DValue::List(l) => Some( + l.iter() + .filter_map(|v| match v { + DValue::Double(x) | DValue::UnitFloat(_, x) => Some(*x), + DValue::Integer(i) => Some(*i as f64), + _ => None, + }) + .collect(), + ), + DValue::UnitFloats(_, v) => Some(v.clone()), + _ => None, + } + }; + let t = nums("Trnf").filter(|t| t.len() == 8).unwrap_or_else(|| vec![0.0; 8]); + let size = d.obj("Sz ").and_then(|s| Some([s.num("Wdth")?, s.num("Hght")?])); + l.smart_object = Some(SmartObject { uuid, quad: [[t[0], t[1]], [t[2], t[3]], [t[4], t[5]], [t[6], t[7]]], size }); + } + b"PlLd" if l.smart_object.is_none() => { + let _id = r.tag()?; + let _ver = r.u32()?; + let n = r.u8()? as usize; + let uuid = String::from_utf8_lossy(r.bytes(n)?).into_owned(); + let _page = r.u32()?; + let _pages = r.u32()?; + let _aa = r.u32()?; + let _ty = r.u32()?; + let mut t = [0.0; 8]; + for v in &mut t { + *v = r.f64()?; + } + l.smart_object = Some(SmartObject { uuid, quad: [[t[0], t[1]], [t[2], t[3]], [t[4], t[5]], [t[6], t[7]]], size: None }); + } b"vmsk" | b"vsms" => { let _ver = r.u32()?; let flags = r.u32()?; diff --git a/crates/psd/src/tests.rs b/crates/psd/src/tests.rs index 68680fe7..cd2f9293 100644 --- a/crates/psd/src/tests.rs +++ b/crates/psd/src/tests.rs @@ -187,3 +187,34 @@ fn truncated_and_garbage_input_errors() { assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err()); assert!(Psd::parse(b"PNG".to_vec()).is_err()); } + +#[test] +fn smart_objects_and_embedded_files() { + let mut inner = WDoc::new(20, 10); + inner.layers = vec![WLayer::solid("Red", Rect::new(0, 0, 20, 10), [1.0, 0.0, 0.0, 1.0])]; + let inner_bytes = write(&inner); + let mut d = WDoc::new(64, 48); + // The content scaled 2× and rotated 90° clockwise: top-left at (40, 4). + let quad = [[40.0, 4.0], [40.0, 44.0], [20.0, 44.0], [20.0, 4.0]]; + d.layers = vec![ + WLayer::solid("Back", Rect::new(0, 0, 64, 48), [0.0, 0.0, 0.0, 1.0]), + WLayer::solid("Placed", Rect::new(20, 4, 20, 40), [0.5, 0.5, 0.5, 1.0]).with_block(smart_object_block("uuid-1", quad, [20.0, 10.0])), + ]; + d.linked = vec![("uuid-1".into(), "inner.psd".into(), *b"8BPS", inner_bytes.clone())]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.linked.len(), 1); + assert_eq!(p.linked[0].name, "inner.psd"); + assert_eq!(p.linked[0].file_type, "8BPS"); + assert_eq!(p.linked_data("uuid-1").unwrap(), &inner_bytes[..]); + let so = p.layers[1].smart_object.clone().unwrap(); + assert_eq!(so.uuid, "uuid-1"); + assert_eq!(so.quad, quad); + assert_eq!(so.size, Some([20.0, 10.0])); + let (c, sx, sy, rot) = so.placement(20.0, 10.0); + assert_eq!(c, [30.0, 24.0]); + assert!((sx - 2.0).abs() < 1e-9 && (sy - 2.0).abs() < 1e-9 && (rot - 90.0).abs() < 1e-9, "{sx} {sy} {rot}"); + // The embedded document parses on its own. + let e = Psd::parse(p.linked_data("uuid-1").unwrap().to_vec()).unwrap(); + assert_eq!((e.width, e.height), (20, 10)); + assert!(p.layers[0].smart_object.is_none()); +} diff --git a/crates/psd/src/write.rs b/crates/psd/src/write.rs index fb78cc78..160e81d3 100644 --- a/crates/psd/src/write.rs +++ b/crates/psd/src/write.rs @@ -103,11 +103,14 @@ pub struct WDoc { /// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers /// with Normal blending. pub composite: Option>, + /// Embedded files for smart objects: (unique id, file name, type code, bytes), written as + /// linked layer data (`lnk2`). + pub linked: Vec<(String, String, [u8; 4], Vec)>, } impl WDoc { pub fn new(width: u32, height: u32) -> WDoc { - WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None } + WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None, linked: vec![] } } } @@ -339,6 +342,13 @@ pub fn write(doc: &WDoc) -> Vec { lm.extend_from_slice(&li); u32b(&mut lm, 0); } + if !doc.linked.is_empty() { + let d = linked_block(&doc.linked); + lm.extend_from_slice(b"8BIM"); + lm.extend_from_slice(b"lnk2"); + u32b(&mut lm, d.len() as u32); + lm.extend_from_slice(&d); + } } u32b(&mut out, lm.len() as u32); out.extend_from_slice(&lm); @@ -372,8 +382,53 @@ pub fn write(doc: &WDoc) -> Vec { out } +/// Linked layer data records (`liFD`, version 7) for embedded files. +fn linked_block(files: &[(String, String, [u8; 4], Vec)]) -> Vec { + let mut out = vec![]; + for (uuid, name, ty, data) in files { + let mut rec = vec![]; + rec.extend_from_slice(b"liFD"); + u32b(&mut rec, 7); + rec.push(uuid.len() as u8); + rec.extend_from_slice(uuid.as_bytes()); + rec.extend_from_slice(&unicode_plain(name)); + rec.extend_from_slice(ty); + rec.extend_from_slice(b"8BIM"); + rec.extend_from_slice(&(data.len() as u64).to_be_bytes()); + rec.push(0); // no file-open descriptor + rec.extend_from_slice(data); + u32b(&mut rec, 0); // child document id (empty unicode string) + rec.extend_from_slice(&0f64.to_be_bytes()); // asset modification time + rec.push(0); // asset locked state + out.extend_from_slice(&(rec.len() as u64).to_be_bytes()); + out.extend_from_slice(&rec); + while !out.len().is_multiple_of(4) { + out.push(0); + } + } + out +} + // ---------------------------------------------------------------- block helpers +/// `SoLd`: a placed layer showing linked file `uuid` (content `size` pixels) with its corners at +/// `quad` (top-left, top-right, bottom-right, bottom-left). +pub fn smart_object_block(uuid: &str, quad: [[f64; 2]; 4], size: [f64; 2]) -> ([u8; 4], Vec) { + let list = |v: &[f64]| DValue::List(v.iter().map(|x| DValue::Double(*x)).collect()); + let t: Vec = quad.iter().flat_map(|p| [p[0], p[1]]).collect(); + let d = Descriptor::new("null") + .with("Idnt", DValue::Text(uuid.to_string())) + .with("placed", DValue::Text(uuid.to_string())) + .with("Trnf", list(&t)) + .with("nonAffineTransform", list(&t)) + .with("Sz ", DValue::Descriptor(Descriptor::new("Pnt ").with("Wdth", DValue::Double(size[0])).with("Hght", DValue::Double(size[1])))); + let mut out = b"soLD".to_vec(); + u32b(&mut out, 4); + u32b(&mut out, 16); + descriptor::write_descriptor(&mut out, &d); + (*b"SoLd", out) +} + /// `TySh`: a type layer with one style run; the text origin sits at `origin`. pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec) { let mut d = vec![]; From 054ce4a0fbf6ca94d7fc719545111a448b065edd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:18:30 -0400 Subject: [PATCH 25/36] M13.2: docs (parity, architecture) Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 1 + docs/parity.md | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..7fc1bb5a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,6 +26,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | | L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | +| L2 | `pdf` | PDF, PDF-compatible Illustrator (`.ai`) and EPS (PostScript subset) vector footage into the `svg` render tree: paths, fills, strokes, shadings, clipping, optional-content layers | | L2 | `model` | 3D models for Advanced 3D: glTF 2.0 (`.gltf`/`.glb`) and OBJ/MTL import (meshes, PBR metallic-roughness materials and textures, node hierarchy, skins, animations), parametric primitives, extruded/bevelled outline meshes and polygon triangulation | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | diff --git a/docs/parity.md b/docs/parity.md index 4437bece..68717920 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -21,8 +21,8 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; text in imported +PDF/AI files and Illustrator procset EPS; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). @@ -111,14 +111,14 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | | Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | +| Import | 94% | 1.0 | text and images inside PDF/AI files, Illustrator EPS relying on Adobe procsets, smart-object perspective/warps, PSD 3D layers (PDF / PDF-compatible AI / EPS vector footage with Continuously Rasterize, layered composition import and Create Shapes from Vector Layer, and PSD smart objects with embedded files landed in M13.2; PSD as footage/composition/retain layer sizes, SVG footage earlier) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | | Shapes | ≈ 80% | 1.5 | Lottie can't carry stroke taper/wave (stroke Taper and Wave, Dash 2/Gap 2/Dash 3/Gap 3 and radial-gradient Highlight Length/Angle landed in M13.5; pen tool for shape paths and vertex editing in M6.5) | | Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | | Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | -| Text | ≈ 91% | 0.8 | no OpenType feature panel, no extruded strokes, variable-axis animation changes outlines but not advances (vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | +| Text | ≈ 94% | 0.5 | no extruded strokes, variable-axis animation changes outlines but not advances (OpenType features — stylistic sets, discretionary ligatures, contextual / stylistic alternates, swash, titling, ordinals, fractions, figure styles, true small caps / all small caps and superior / inferior glyphs with faux fallback — per character with the Character panel's OpenType popup and `text.fontFeatures` landed in M13.2; vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | -| 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer; stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | +| 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer; collapsed precomps of another size seen through the parent's camera render (fixed in M13.2); stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | | Effects | ≈ 80% | 4.0 | GPU versions of the remaining effects (58 run on the GPU since M12.7) and the missing controls listed as partial in [effects.md](effects.md) (every After Effects effect exists since M9.11, M12.5 and M12.6; parameter names, order, twirl-downs, popups, units and defaults were aligned in M9.12) | | Interface | 74% | 5.0 | a richer Learn area (Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | From e1296a57c2c14b366e49d50b7c2e3d67daa16bcf Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:22:49 -0400 Subject: [PATCH 26/36] M13.2: Create VR Environment nests the scene as a collapsed 3D precomp in each face comp (lights copied) Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/app_more.rs | 61 +++++++++++++++----------- crates/engine/src/tests_m145.rs | 5 ++- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/crates/engine/src/commands/app_more.rs b/crates/engine/src/commands/app_more.rs index 6431d5e2..fb0473d4 100644 --- a/crates/engine/src/commands/app_more.rs +++ b/crates/engine/src/commands/app_more.rs @@ -433,8 +433,8 @@ fn ae(v: [f64; 3]) -> Vec3 { } /// Composition ▸ VR ▸ Create VR Environment: renders the active comp's 3D scene in every -/// direction. Six face comps (a 90° camera each at `position`, default the comp centre, with a -/// copy of the scene's layers), a 3:2 cube map of them, and an equirectangular output comp +/// direction. Six face comps (a 90° camera each at `position`, default the comp centre, looking +/// at the scene nested as a collapsed 3D precomp), a 3:2 cube map of them, and an equirectangular output comp /// (VR Converter, cube map → equirectangular). One undo step. fn create_vr_environment(s: &mut Session, p: &Value) -> Result { let cid = comp_id(s, p)?; @@ -462,30 +462,41 @@ fn create_vr_environment(s: &mut Session, p: &Value) -> Result { id_of(&s.execute("layer.newCamera", json!({"comp": fc, "name": format!("{fname} Camera"), "angleOfView": 90.0, "type": "oneNode"}))?, "layer")?; s.execute("prop.set", json!({"comp": fc, "layer": cam, "path": "transform/position", "value": center}))?; s.execute("prop.set", json!({"comp": fc, "layer": cam, "path": "transform/orientation", "value": o}))?; - // The scene's layers (not its cameras), in the scene's coordinates, under the face - // camera. - let layers: Vec = src.layers.iter().filter(|l| !l.is_camera()).cloned().collect(); - s.edit("Create VR Environment", None, |proj, _| { - let mut next = proj.next_id; - let mut map = std::collections::BTreeMap::new(); - let mut copies = vec![]; - for l in &layers { - let mut c = l.clone(); - super::edit::reid(&mut c, &mut next); - map.insert(l.id, c.id); - copies.push(c); - } - for c in &mut copies { - c.parent = c.parent.and_then(|p| map.get(&p).copied()); - if let Some(m) = &mut c.track_matte { - m.layer = map.get(&m.layer).copied().unwrap_or(m.layer); + // The scene, nested as a collapsed 3D precomp whose layer maps the scene's + // coordinates onto the face comp's unchanged (anchor = position): its 3D layers are + // seen through the face camera and edits to the scene show in every face. Lights + // inside a collapsed precomp do not light it (the containing comp's do), so the + // scene's lights are copied next to it. + let pre = id_of(&s.execute("layer.addItem", json!({"comp": fc, "item": cid.0, "time": 0.0}))?, "layer")?; + let mid = [src.width as f64 / 2.0, src.height as f64 / 2.0, 0.0]; + s.execute("layer.setSwitch", json!({"comp": fc, "layers": [pre], "switch": "threeD", "value": true}))?; + s.execute("layer.setSwitch", json!({"comp": fc, "layers": [pre], "switch": "collapse", "value": true}))?; + s.execute("prop.set", json!({"comp": fc, "layer": pre, "path": "transform/anchor", "value": mid}))?; + s.execute("prop.set", json!({"comp": fc, "layer": pre, "path": "transform/position", "value": mid}))?; + let lights: Vec = src.layers.iter().filter(|l| l.is_light()).cloned().collect(); + if !lights.is_empty() { + s.edit("Create VR Environment", None, |proj, _| { + let mut next = proj.next_id; + let mut map = std::collections::BTreeMap::new(); + let mut copies = vec![]; + for l in &lights { + let mut c = l.clone(); + super::edit::reid(&mut c, &mut next); + map.insert(l.id, c.id); + copies.push(c); } - } - proj.next_id = next; - let comp = proj.comp_mut(ItemId(fc)).ok_or(EngineError::NoComp)?; - comp.layers.extend(copies); - Ok(()) - })?; + for c in &mut copies { + c.parent = c.parent.and_then(|p| map.get(&p).copied()); + } + proj.next_id = next; + let comp = proj.comp_mut(ItemId(fc)).ok_or(EngineError::NoComp)?; + let at = comp.layers.len().saturating_sub(1); + for (k, c) in copies.into_iter().enumerate() { + comp.layers.insert(at + k, c); + } + Ok(()) + })?; + } faces.push(fc); } let cube = id_of( diff --git a/crates/engine/src/tests_m145.rs b/crates/engine/src/tests_m145.rs index edc8fe18..d538da63 100644 --- a/crates/engine/src/tests_m145.rs +++ b/crates/engine/src/tests_m145.rs @@ -476,8 +476,9 @@ fn create_vr_environment_and_extract_cubemap() { let c = s.project.comp(ItemId(*f)).unwrap(); assert_eq!((c.width, c.height), (64, 64)); assert!(c.layers.iter().any(|l| l.is_camera())); - assert!(c.layers.iter().any(|l| l.name == aname && l.is_3d()), "the scene's layers"); - let _ = main; + let nested = c.layers.iter().find(|l| matches!(l.source, effectcraft_project::LayerSource::Comp { item } if item == main)).expect("the scene, nested"); + assert!(nested.switches.collapse && nested.is_3d(), "collapsed 3D precomp"); + let _ = &aname; } let cube = s.project.comp(ItemId(r["cubeMap"].as_u64().unwrap())).unwrap(); assert_eq!((cube.width, cube.height, cube.layers.len()), (192, 128, 6)); From 00eb951711f38459fb8e36d4e874c5b1cd82fc45 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:24:01 -0400 Subject: [PATCH 27/36] M13.1: ScriptUI frontend (dialogs, palettes, dockable panels), script menus, example plug-in, docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ui-egui: scriptui_view draws script windows at engine layout bounds; PanelKind::ScriptPanel; File ▸ Scripts and Window menu list scripts/panels (egui + native menus) - examples/plugins/posterize-bands (WebAssembly, API v1); host plug-in tests; desktop loads /Plug-ins - MCP: script_ui and history tools; docs/plugins.md, agents.md, parity, architecture, ATTRIBUTION Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 5 + apps/effectcraft/src/main.rs | 5 + crates/automation/src/tests.rs | 26 ++ crates/automation/src/tools.rs | 32 ++ ...te Null at Selected Layers.jsx.attribution | 7 + .../scripts/Rename Layers.jsx.attribution | 7 + .../Render Queue Batch.jsx.attribution | 7 + .../Layer Tools.jsx.attribution | 7 + .../Sort Layers by In Point.jsx.attribution | 7 + crates/engine/src/commands/file_more.rs | 2 +- crates/engine/src/menus.rs | 1 + crates/host/tests/plugins.rs | 106 ++++++ crates/ui-egui/src/dock.rs | 3 + crates/ui-egui/src/dock_ui.rs | 13 + crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/menus.rs | 57 +++- crates/ui-egui/src/native_menu.rs | 24 ++ crates/ui-egui/src/panels/mod.rs | 2 + crates/ui-egui/src/panels/scriptui_view.rs | 321 ++++++++++++++++++ crates/ui-egui/tests/ui_scriptui.rs | 112 ++++++ docs/agents.md | 29 ++ docs/parity.md | 14 +- docs/plugins.md | 126 +++++++ examples/plugins/posterize-bands/Cargo.lock | 7 + examples/plugins/posterize-bands/Cargo.toml | 18 + examples/plugins/posterize-bands/src/lib.rs | 93 +++++ 26 files changed, 1026 insertions(+), 7 deletions(-) create mode 100644 crates/engine/scripts/Create Null at Selected Layers.jsx.attribution create mode 100644 crates/engine/scripts/Rename Layers.jsx.attribution create mode 100644 crates/engine/scripts/Render Queue Batch.jsx.attribution create mode 100644 crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution create mode 100644 crates/engine/scripts/Sort Layers by In Point.jsx.attribution create mode 100644 crates/host/tests/plugins.rs create mode 100644 crates/ui-egui/src/panels/scriptui_view.rs create mode 100644 crates/ui-egui/tests/ui_scriptui.rs create mode 100644 docs/plugins.md create mode 100644 examples/plugins/posterize-bands/Cargo.lock create mode 100644 examples/plugins/posterize-bands/Cargo.toml create mode 100644 examples/plugins/posterize-bands/src/lib.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index d52b19c7..71c54885 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -61,6 +61,11 @@ Screenshots of EffectCraft itself, rendered headlessly, showing only procedurall - Procedural demo content (solids, shapes, gradients, text animations): `crates/engine/src/demo.rs`. - Text animator presets (Typewriter, Fade Up Characters, Bounce In Words, Tracking In, Scramble, Blur In, Jitter, Drop In Lines): `crates/engine/presets/text_animators.json` (sidecar `.attribution`). +- Sample scripts (File ▸ Scripts) and ScriptUI panel: `crates/engine/scripts/Rename Layers.jsx`, + `crates/engine/scripts/Sort Layers by In Point.jsx`, `crates/engine/scripts/Create Null at Selected Layers.jsx`, + `crates/engine/scripts/Render Queue Batch.jsx`, `crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx` + (sidecars `.attribution`). +- Example effect plug-in: `examples/plugins/posterize-bands` (plug-in API v1, WebAssembly). ## First-party brand marks diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index b6426f5f..ac7f1be6 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -64,6 +64,11 @@ fn main() -> eframe::Result { // Settings, shortcut presets and the crash-recovery sentinel live in the platform // config directory. Agent-driven runs (`--control`) skip crash recovery. if let Some(dir) = config_dir() { + // WebAssembly effect plug-ins in /Plug-ins load before the menus are built. + let plugins = dir.join("Plug-ins"); + if plugins.is_dir() { + let _ = session.execute("effect.plugins.load", json!({"folder": plugins.to_string_lossy()})); + } session.config = Some(std::sync::Arc::new(effectcraft_engine::config::DirConfig::new(dir))); } session.load_settings(); diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index 848d3ee0..9504094a 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -265,6 +265,32 @@ fn bridge_rejects_non_loopback() { assert!(Backend::bridge("localhost:9877").is_ok()); } +#[test] +fn script_ui_and_history_tools() { + let mut sess = Session::default(); + effectcraft_script::install(&mut sess); + let mut s = McpServer::new(Backend::headless(sess)); + let code = "var d = new Window('dialog', 'Ask'); var g = d.add('group'); g.add('button', undefined, 'OK', {name: 'ok'}); \ + if (d.show() == 1) app.project.items.addComp('Answered', 64, 64, 1, 1, 24); 'done'"; + let r = call_json(&mut s, "run_script", json!({"code": code})); + assert_eq!(r["waiting"], json!(true), "{r}"); + let list = call_json(&mut s, "script_ui", json!({})); + assert_eq!(list[0]["title"], "Ask"); + let tree = call_json(&mut s, "script_ui", json!({"action": "get"})); + assert_eq!(tree["root"]["children"][0]["children"][0]["name"], "ok"); + let r = call_json(&mut s, "script_ui", json!({"action": "click", "widget": "ok"})); + assert_eq!(r["result"], json!("done"), "{r}"); + // History: undo the comp, make another, and jump back to the first on its branch. + call_json(&mut s, "undo", json!({})); + call_json(&mut s, "execute_command", json!({"command": "comp.new", "params": {"name": "Other", "width": 64, "height": 64}})); + let h = call_json(&mut s, "history", json!({})); + let branch = h["states"].as_array().unwrap().iter().find(|n| n["depth"] == 1).unwrap()["index"].clone(); + let h = call_json(&mut s, "history", json!({"goto": branch})); + let cur = h["current"].as_u64().unwrap() as usize; + assert_eq!(h["states"][cur]["depth"], 0, "the jumped-to state is now the working line"); + assert!(call_json(&mut s, "get_project", json!({})).to_string().contains("Answered")); +} + #[test] fn run_script_round_trip() { let mut sess = Session::default(); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index f9034a38..9e1bfee5 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -315,6 +315,20 @@ fn redo(b: &mut Backend, a: &Value) -> Result { history(b, a, "edit.redo") } +fn script_ui_tool(b: &mut Backend, a: &Value) -> Result { + let action = get(a, "action").and_then(Value::as_str).unwrap_or("list"); + let id = match action { + "list" => "scriptui.list", + "get" => "scriptui.get", + "click" => "scriptui.click", + "set" => "scriptui.set", + "close" => "scriptui.close", + other => return Err(Error::BadArgs(format!("unknown action `{other}` (list | get | click | set | close)"))), + }; + let p = obj(&[("window", get(a, "window")), ("widget", get(a, "widget")), ("value", get(a, "value")), ("result", get(a, "result"))]); + json_reply(b.exec(id, p)?) +} + fn history_tool(b: &mut Backend, a: &Value) -> Result { if let Some(g) = get(a, "goto") { let p = match g { @@ -554,6 +568,24 @@ static TOOLS: &[ToolDef] = &[ schema: || schema(json!({"goto": {"type": ["integer", "string"], "description": "State index or id to jump to (edit.history.goto)."}}), &[]), run: history_tool, }, + ToolDef { + name: "script_ui", + description: "Drive ScriptUI windows that scripts opened (dialogs, palettes, dockable panels). action `list` (default): open windows; `get`: a window's control tree (type, name, text, value, checked, items, selection, bounds); `click`: press a button / toggle a checkbox / pick a radio button; `set`: type text, move a slider, pick a list item (index or text); `close`: close a window (`result` for a dialog's show(), default 2 = Cancel). Controls by id, \"#id\", properties.name or text. A dialog's script resumes when the dialog closes.", + bridge_only: false, + schema: || { + schema( + json!({ + "action": {"type": "string", "enum": ["list", "get", "click", "set", "close"]}, + "window": {"type": ["integer", "string"], "description": "Window id or title (optional when one is open)."}, + "widget": {"type": ["integer", "string"], "description": "Control id, \"#id\", properties.name or text."}, + "value": {"description": "For `set`: text, number, bool, item index or item text."}, + "result": {"type": "integer", "description": "For `close`: what the dialog's show() returns."} + }), + &[], + ) + }, + run: script_ui_tool, + }, ToolDef { name: "screenshot", description: "Screenshot of the live app window, or one panel (`Composition`, `Timeline`, `EffectControls`, `Project`, ...) or element id, as PNG (longest side `max_side`, default 1600). Use it to check what the UI looks like after an action.", diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution new file mode 100644 index 00000000..ea166b70 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Create Null at Selected Layers.jsx +title: EffectCraft sample script (Create Null at Selected Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Rename Layers.jsx.attribution b/crates/engine/scripts/Rename Layers.jsx.attribution new file mode 100644 index 00000000..bc07b909 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Rename Layers.jsx +title: EffectCraft sample script (Rename Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Render Queue Batch.jsx.attribution b/crates/engine/scripts/Render Queue Batch.jsx.attribution new file mode 100644 index 00000000..5ab7be71 --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx.attribution @@ -0,0 +1,7 @@ +asset: Render Queue Batch.jsx +title: EffectCraft sample script (Render Queue Batch) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution new file mode 100644 index 00000000..6d4d572f --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution @@ -0,0 +1,7 @@ +asset: Layer Tools.jsx +title: EffectCraft sample script (Layer Tools) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx.attribution b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution new file mode 100644 index 00000000..e744d662 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution @@ -0,0 +1,7 @@ +asset: Sort Layers by In Point.jsx +title: EffectCraft sample script (Sort Layers by In Point) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 9175af59..2041d400 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -828,7 +828,7 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}] | name (an installed or sample script, see file.scripts.list)}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | name (an installed or sample script, see file.scripts.list) | steps: [{command, params}]}", always, run_script ), diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 7f14a988..84522c19 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -757,6 +757,7 @@ Effect Remove All | effect.removeAll --- Manage Effects... | effect.manage + Load Effect Plug-in... | effect.plugins.load --- @effects Animation diff --git a/crates/host/tests/plugins.rs b/crates/host/tests/plugins.rs new file mode 100644 index 00000000..c8a9b368 --- /dev/null +++ b/crates/host/tests/plugins.rs @@ -0,0 +1,106 @@ +//! Effect plug-ins in a full session: a Rust plug-in registered through the trait and the +//! example WebAssembly plug-in (examples/plugins/posterize-bands) loaded with +//! `effect.plugins.load`, both applied by id like built-ins and rendered. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::plugin::{EffectPlugin, PluginFrame, PluginManifest, PluginParam, PluginParamKind, PluginParams, register_plugin}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +struct Swap { + m: PluginManifest, +} + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { + &self.m + } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { + p.swap(0, 2); + } + } + Ok(()) + } +} + +fn setup() -> (Session, u64) { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "P", "width": 40, "height": 20, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": [1.0, 0.5, 0.0], "width": 40, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + (s, l) +} + +fn px(s: &Session) -> [f32; 4] { + let cid = s.active_comp_id().unwrap(); + s.render(cid, Tick::ZERO, Default::default()).get(20, 10) +} + +#[test] +fn rust_plugins_apply_by_id() { + let m = PluginManifest { + api: 1, + id: "org.test.swap".into(), + name: "Swap Red Blue".into(), + category: "Channel".into(), + version: "1".into(), + author: String::new(), + description: String::new(), + params: vec![PluginParam { id: "on".into(), name: "On".into(), kind: PluginParamKind::Checkbox { default: true } }], + }; + register_plugin(Arc::new(Swap { m })).unwrap(); + let (mut s, l) = setup(); + s.execute("effect.apply", json!({"layer": l, "effect": "org.test.swap"})).unwrap(); + let p = px(&s); + assert!(p[0] < 0.01 && (p[1] - 0.5).abs() < 0.01 && p[2] > 0.99, "{p:?}"); + // Its parameters are ordinary properties: set (and animate) them like any effect's. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/on", "value": false})).unwrap(); + let p = px(&s); + assert!(p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["id"] == "org.test.swap" && p["category"] == "Channel")); + assert_eq!(list["api"], 1); + assert!(s.execute("effect.list", json!({"filter": "swap red"})).unwrap().as_array().unwrap().len() == 1); +} + +/// Build the example plug-in (skipped when the wasm32 target isn't installed). +fn build_example() -> Option { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/plugins/posterize-bands"); + let target = std::env::var_os("CARGO_TARGET_DIR").map(std::path::PathBuf::from).unwrap_or_else(|| root.join("../../../target")).join("plugin-example"); + let target = if target.is_absolute() { target } else { root.join("../../..").join(target) }; + let st = std::process::Command::new(env!("CARGO")) + .current_dir(&root) + .args(["build", "--release", "--target", "wasm32-unknown-unknown", "--target-dir"]) + .arg(&target) + .status() + .ok()?; + if !st.success() { + eprintln!("skipping: the example plug-in didn't build (rustup target add wasm32-unknown-unknown)"); + return None; + } + Some(target.join("wasm32-unknown-unknown/release/posterize_bands.wasm")) +} + +#[test] +fn example_wasm_plugin_loads_and_renders() { + let Some(wasm) = build_example() else { return }; + let (mut s, l) = setup(); + let r = s.execute("effect.plugins.load", json!({"path": wasm.to_string_lossy()})).unwrap(); + assert_eq!(r["loaded"][0]["id"], "org.effectcraft.example.posterize-bands", "{r}"); + s.execute("effect.apply", json!({"layer": l, "effect": "Posterize Bands"})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/mix", "value": 0})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/levels", "value": 2})).unwrap(); + // Luminance of (1, .5, 0) is ~0.57: with 2 levels it bands to 1, scaling the colour by 1/y + // (8 bpc: red clips at 1). + let p = px(&s); + let y = 0.2126 + 0.7152 * 0.5; + assert!((p[1] - 0.5 / y as f32).abs() < 0.01 && p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + // Invert: band 0. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/invert", "value": true})).unwrap(); + assert_eq!(px(&s)[..3], [0.0, 0.0, 0.0]); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["source"].as_str().unwrap_or("").ends_with(".wasm"))); +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 91d7c335..6f045101 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -36,6 +36,8 @@ pub enum PanelKind { MaskInterpolation, ScriptConsole, EssentialGraphics, + /// A dockable ScriptUI panel (a script from the ScriptUI Panels folder), by script window id. + ScriptPanel(u32), } impl PanelKind { @@ -95,6 +97,7 @@ impl PanelKind { PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", + PanelKind::ScriptPanel(_) => "ScriptUI Panel", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/dock_ui.rs b/crates/ui-egui/src/dock_ui.rs index 44900918..e72d3989 100644 --- a/crates/ui-egui/src/dock_ui.rs +++ b/crates/ui-egui/src/dock_ui.rs @@ -18,6 +18,12 @@ impl EffectcraftApp { /// "Effect Controls Title", "Properties: Title", and the Timeline tab named after its comp. fn tab_titles(&self) -> Vec<(PanelKind, String)> { let mut out = Vec::new(); + // ScriptUI panels are named after their script. + for w in self.session.script_ui.windows.iter().filter(|w| w.kind == effectcraft_engine::scriptui::WindowKind::Panel) { + if let Some(t) = panels::scriptui_view::panel_title(self, w.id) { + out.push((PanelKind::ScriptPanel(w.id), t)); + } + } let Some(comp) = self.session.active_comp() else { return out }; let cname = self.session.active_comp_id().and_then(|id| self.session.project.item(id)).map(|i| i.name.clone()).unwrap_or_default(); out.push((PanelKind::Composition, format!("Composition {cname}"))); @@ -45,6 +51,13 @@ impl EffectcraftApp { /// Close a panel wherever it is (docked or floating). pub fn close_panel(&mut self, p: PanelKind) { + // Closing a ScriptUI panel closes its script window (its script's onClose runs). + if let PanelKind::ScriptPanel(id) = p + && self.session.script_ui.window(id).is_some() + && let Err(e) = self.session.execute("scriptui.close", serde_json::json!({"window": id})) + { + self.ui.status = e.to_string(); + } self.ui.dock.close(p); self.edit_layout(|l| l.unfloat(p)); if self.ui.maximized == Some(p) { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..9cc2113a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -834,6 +834,8 @@ impl EffectcraftApp { panels::precomp::mini_flowchart(self, &ctx); panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); + panels::scriptui_view::sync_panels(self); + panels::scriptui_view::show_windows(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index eed7e639..a6ad8542 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -456,6 +456,12 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu app.dialog = Some(crate::Dialog::About); Value::Null } + // Window ▸ : dock (or bring forward) the panel the script built. + "window.scriptPanel" => { + let id = p.get("window").and_then(Value::as_u64).ok_or("no ScriptUI panel window")? as u32; + app.show_panel(PanelKind::ScriptPanel(id)); + json!({"window": id}) + } "layer.style.options" => { crate::panels::layer_styles_dialog::open(app, &p)?; Value::Null @@ -791,7 +797,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "render.saveCurrentPreview" => ("path", Ask::Save("Preview.mp4")), - "file.runScript" => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.runScript" if params.get("name").is_none() => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.installScript" | "file.installScriptUIPanel" => ("path", Ask::Open(&["jsx", "js"])), + "effect.plugins.load" if params.get("folder").is_none() => ("path", Ask::Open(&["wasm", "wat"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), @@ -1106,6 +1114,39 @@ pub fn handle_shortcuts(app: &mut EffectcraftApp, ctx: &egui::Context) { } } +/// Menu entries listed from the session: (label, command, params). File ▸ Scripts starts with +/// the installed and sample scripts; the Window menu ends with the ScriptUI panels (as in After +/// Effects, by file name). +pub fn dynamic_entries(app: &EffectcraftApp, menu: &str) -> Vec<(String, String, Value)> { + let panels = match menu { + "Scripts" => false, + "Window" => true, + _ => return vec![], + }; + effectcraft_engine::commands::scripts::scripts(&app.session) + .into_iter() + .filter(|e| e.panel == panels) + .map(|e| { + let cmd = if panels { "window.scriptPanel" } else { "file.runScript" }; + let params = json!({"name": e.name}); + (e.name, cmd.to_string(), params) + }) + .collect() +} + +fn dynamic_rows(app: &mut EffectcraftApp, ui: &mut egui::Ui, menu: &str, clicked: &mut Option<(String, Value)>) -> usize { + let rows = dynamic_entries(app, menu); + for (i, (label, cmd, params)) in rows.iter().enumerate() { + let r = ui.add(egui::Button::new((gutter(false), label.as_str()))); + app.auto.add(&format!("menu.{}.{i}", if menu == "Window" { "scriptPanels" } else { "scripts" }), r.rect, label); + if r.clicked() { + *clicked = Some((cmd.clone(), params.clone())); + ui.close(); + } + } + rows.len() +} + /// Draw the in-window menu bar (the engine's AE menu tree). pub fn menu_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui) { let ctx = ui.ctx().clone(); @@ -1118,6 +1159,10 @@ pub fn menu_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui) { let r = ui.menu_button(label, |ui| { ui.set_min_width(if label == "Effect" { 200.0 } else { 280.0 }); menu_nodes(app, ui, children, &mut clicked); + if label == "Window" && !dynamic_entries(app, "Window").is_empty() { + ui.separator(); + dynamic_rows(app, ui, "Window", &mut clicked); + } }); app.auto.add(&format!("menu.{label}"), r.response.rect, label); } @@ -1158,6 +1203,16 @@ fn menu_nodes(app: &mut EffectcraftApp, ui: &mut egui::Ui, nodes: &[MenuNode], c menu_nodes(app, ui, children, clicked); }); } + MenuNode::Submenu { label, children } if label == "Scripts" => { + // File ▸ Scripts: installed and sample scripts, then Install… / Run Script File…. + ui.menu_button((gutter(false), label.as_str()), |ui| { + ui.set_min_width(280.0); + if dynamic_rows(app, ui, "Scripts", clicked) > 0 { + ui.separator(); + } + menu_nodes(app, ui, children, clicked); + }); + } MenuNode::Submenu { label, children } => { ui.menu_button((gutter(false), label.as_str()), |ui| { ui.set_min_width(if children.len() > 30 { 200.0 } else { 240.0 }); diff --git a/crates/ui-egui/src/native_menu.rs b/crates/ui-egui/src/native_menu.rs index ef5070b3..4558b3e2 100644 --- a/crates/ui-egui/src/native_menu.rs +++ b/crates/ui-egui/src/native_menu.rs @@ -154,7 +154,31 @@ pub fn build(app: &EffectcraftApp) -> NativeMenu { kids.push(NativeNode::Separator); } } + let dynamic = |kids: &mut Vec, menu: &str| { + for (k, (label, command, params)) in crate::menus::dynamic_entries(app, menu).into_iter().enumerate() { + kids.push(NativeNode::Item(NativeItem { + id: format!("{id}.d{k}"), + label, + command, + params, + shortcut: None, + accelerator: None, + enabled: true, + checked: None, + })); + } + }; + if label == "Scripts" { + dynamic(&mut kids, "Scripts"); + if !kids.is_empty() { + kids.push(NativeNode::Separator); + } + } rec(app, children, &id, &mut kids); + if label == "Window" && !crate::menus::dynamic_entries(app, "Window").is_empty() { + kids.push(NativeNode::Separator); + dynamic(&mut kids, "Window"); + } out.push(NativeNode::Submenu { label: label.clone(), children: kids }); } MenuNode::Item(e) => { diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index d0fb3a3e..16cc4d82 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -29,6 +29,7 @@ pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; pub mod script_console; +pub mod scriptui_view; pub mod settings; pub mod shortcut_editor; pub mod text_panels; @@ -98,6 +99,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), + PanelKind::ScriptPanel(id) => scriptui_view::panel(app, ui, rect, id), PanelKind::Flowchart => flowchart::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/scriptui_view.rs b/crates/ui-egui/src/panels/scriptui_view.rs new file mode 100644 index 00000000..580e287b --- /dev/null +++ b/crates/ui-egui/src/panels/scriptui_view.rs @@ -0,0 +1,321 @@ +//! ScriptUI windows: the dialogs, palettes and dockable panels scripts build +//! (`Session::script_ui`). Each control is drawn at the bounds the engine's ScriptUI layout +//! computed; what the user does goes back through the `scriptui.click` / `scriptui.set` / +//! `scriptui.close` commands, which run the script's handlers. Every control registers an +//! automation id `scriptui..` (and `scriptui..` when the script +//! named it). + +use effectcraft_engine::scriptui::{ScriptWindow, Widget, WidgetKind, WindowKind, layout}; +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, UiBuilder, pos2, vec2}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::dock::PanelKind; +use crate::theme::Tokens; + +enum Act { + Click(u32, u32), + Set(u32, u32, Value), + Close(u32), +} + +fn rect_of(origin: egui::Pos2, b: [f64; 4]) -> Rect { + Rect::from_min_size(origin + vec2(b[0] as f32, b[1] as f32), vec2(b[2] as f32, b[3] as f32)) +} + +fn register(app: &mut EffectcraftApp, win: u32, w: &Widget, rect: Rect) { + let label = if w.text.is_empty() { format!("{:?}", w.kind) } else { w.text.clone() }; + app.auto.add(&format!("scriptui.{win}.{}", w.id), rect, &label); + if !w.name.is_empty() { + app.auto.add(&format!("scriptui.{win}.{}", w.name), rect, &label); + } +} + +/// Draw `w`'s children (and, for containers, their children) at `origin` + bounds. +fn draw_children(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + if w.kind == WidgetKind::TabbedPanel { + // Tab headers, then the active tab only. + let r = rect_of(origin, w.bounds); + let mut x = r.min.x; + for (i, tab) in w.children.iter().enumerate() { + let tw = (tab.text.chars().count() as f32 * 7.0 + 20.0).max(48.0); + let hr = Rect::from_min_size(pos2(x, r.min.y), vec2(tw, 22.0)); + let active = i == w.active_tab; + let resp = ui.interact(hr, egui::Id::new(("sui-tab", win, tab.id)), Sense::click()); + ui.painter().rect_filled( + hr, + 3.0, + if active { + t.row_selected + } else if resp.hovered() { + t.hover + } else { + t.field_bg + }, + ); + ui.painter().text(hr.center(), Align2::CENTER_CENTER, &tab.text, Tokens::ui(12.0), t.text); + register(app, win, tab, hr); + if resp.clicked() && !active && tab.enabled { + acts.push(Act::Click(win, tab.id)); + } + x += tw + 2.0; + } + ui.painter().rect_stroke(Rect::from_min_max(pos2(r.min.x, r.min.y + 22.0), r.max), 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if let Some(tab) = w.children.get(w.active_tab) { + draw_children(app, ui, origin, win, tab, acts); + } + return; + } + for c in w.children.iter().filter(|c| c.visible) { + draw_widget(app, ui, origin, win, c, acts); + } +} + +fn draw_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + let rect = rect_of(origin, w.bounds); + register(app, win, w, rect); + let id = egui::Id::new(("sui", win, w.id)); + let tip = |r: egui::Response| if w.help_tip.is_empty() { r } else { r.on_hover_text(&w.help_tip) }; + match w.kind { + WidgetKind::Group | WidgetKind::Tab | WidgetKind::Window | WidgetKind::TabbedPanel => draw_children(app, ui, origin, win, w, acts), + WidgetKind::Panel => { + let p = ui.painter(); + p.rect_stroke(rect.shrink(1.0), 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if !w.text.is_empty() { + let g = p.layout_no_wrap(w.text.clone(), Tokens::ui(11.5), t.text_dim); + let at = pos2(rect.min.x + 10.0, rect.min.y - 1.0); + p.rect_filled(Rect::from_min_size(at - vec2(3.0, 0.0), g.size() + vec2(6.0, 0.0)), 0.0, t.panel_bg); + p.galley(at, g, t.text_dim); + } + draw_children(app, ui, origin, win, w, acts); + } + WidgetKind::Button | WidgetKind::IconButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, egui::Button::new(&w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::StaticText => { + let col = if w.enabled { t.text } else { t.text_faint }; + ui.painter().with_clip_rect(rect.expand(2.0)).text(pos2(rect.min.x, rect.min.y + 1.0), Align2::LEFT_TOP, &w.text, Tokens::ui(12.0), col); + } + WidgetKind::EditText => { + // The text being typed lives here until the field loses focus (then onChange). + let mut buf: String = ui.data(|d| d.get_temp::(id)).unwrap_or_else(|| w.text.clone()); + let focused = ui.memory(|m| m.has_focus(id)); + if !focused { + buf = w.text.clone(); + } + let te = if w.multiline { egui::TextEdit::multiline(&mut buf) } else { egui::TextEdit::singleline(&mut buf) }; + let te = te.id(id).interactive(!w.read_only).font(Tokens::ui(12.0)); + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, te)).inner; + let r = tip(r); + if r.lost_focus() && buf != w.text { + acts.push(Act::Set(win, w.id, json!(buf))); + } + ui.data_mut(|d| d.insert_temp(id, buf)); + } + WidgetKind::Checkbox => { + let mut v = w.checked; + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::Checkbox::new(&mut v, &w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::RadioButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::RadioButton::new(w.checked, &w.text))).inner; + if tip(r).clicked() && !w.checked { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::Slider | WidgetKind::Scrollbar => { + // While dragging, the value lives here; the release sends it (onChanging / onChange). + let mut v: f64 = ui.data(|d| d.get_temp::(id)).unwrap_or(w.value); + let (lo, hi) = if w.max > w.min { (w.min, w.max) } else { (0.0, 100.0) }; + let r = ui + .add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + ui.spacing_mut().slider_width = rect.width() - 4.0; + ui.add(egui::Slider::new(&mut v, lo..=hi).show_value(false)) + }) + .inner + }) + .inner; + let r = tip(r); + if r.dragged() { + ui.data_mut(|d| d.insert_temp(id, v)); + } else { + ui.data_mut(|d| d.remove::(id)); + if r.drag_stopped() || r.changed() { + acts.push(Act::Set(win, w.id, json!(v))); + } + } + } + WidgetKind::Progressbar => { + let span = if w.max > w.min { w.max - w.min } else { 100.0 }; + let f = ((w.value - w.min) / span).clamp(0.0, 1.0) as f32; + ui.put(rect, egui::ProgressBar::new(f).desired_height(rect.height())); + } + WidgetKind::DropDownList => { + let cur = w.selection.first().and_then(|i| w.items.get(*i)).cloned().unwrap_or_default(); + let mut pick: Option = None; + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + egui::ComboBox::from_id_salt(id).selected_text(cur).width(rect.width() - 8.0).show_ui(ui, |ui| { + for (i, it) in w.items.iter().enumerate() { + if it == "-" { + ui.separator(); + } else if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick + && !w.selection.contains(&i) + { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::ListBox => { + let mut pick: Option = None; + ui.painter().rect_filled(rect, 2.0, t.field_bg); + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect.shrink(2.0)), |ui| { + egui::ScrollArea::vertical().id_salt(id).max_height(rect.height() - 4.0).show(ui, |ui| { + ui.set_width(rect.width() - 8.0); + for (i, it) in w.items.iter().enumerate() { + if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::Image | WidgetKind::Unknown => { + ui.painter().rect_stroke(rect, 2.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + } + } +} + +/// Checkboxes and radio buttons sit at the left of their (possibly filled) bounds. +fn snug(r: Rect, text: &str) -> Rect { + let w = (text.chars().count() as f32 * 7.0 + 26.0).min(r.width()); + Rect::from_min_size(r.min, vec2(w, r.height())) +} + +fn perform(app: &mut EffectcraftApp, ctx: &egui::Context, acts: Vec) { + for a in acts { + let (cmd, p) = match a { + Act::Click(w, id) => ("scriptui.click", json!({"window": w, "widget": id})), + Act::Set(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v})), + Act::Close(w) => ("scriptui.close", json!({"window": w})), + }; + match crate::menus::invoke(app, ctx, cmd, p) { + Ok(r) => { + if let Some(e) = r.get("error").filter(|e| !e.is_null()) { + let msg = e["message"].as_str().unwrap_or("script error").to_string(); + app.toast = Some((msg, ctx.input(|i| i.time))); + } else if let Some(o) = r["output"].as_str().filter(|o| !o.is_empty()) { + app.ui.status = o.lines().last().unwrap_or_default().to_string(); + } + } + Err(e) => app.ui.status = e, + } + } +} + +/// The floating script windows (dialogs, palettes, windows), drawn over the app. +pub fn show_windows(app: &mut EffectcraftApp, ctx: &egui::Context) { + let windows: Vec = app.session.script_ui.windows.iter().filter(|w| w.kind != WindowKind::Panel).cloned().collect(); + if windows.is_empty() { + return; + } + let t = app.tokens; + let mut acts = vec![]; + let screen = ctx.content_rect(); + if windows.iter().any(|w| w.modal) { + // A modal dialog blocks the app behind it, as in After Effects. + let r = egui::Area::new(egui::Id::new("scriptui-modal-block")).order(egui::Order::Middle).fixed_pos(screen.min).show(ctx, |ui| { + ui.allocate_exact_size(screen.size(), Sense::click_and_drag()); + ui.painter().rect_filled(screen, 0.0, Color32::from_black_alpha(90)); + }); + let _ = r; + } + for w in &windows { + let size = vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32); + let mut open = true; + let order = if w.modal { egui::Order::Foreground } else { egui::Order::Middle }; + let resp = egui::Window::new(if w.title.is_empty() { &w.script } else { &w.title }) + .id(egui::Id::new(("scriptui-window", w.id))) + .order(order) + .collapsible(false) + .resizable(false) + .open(&mut open) + .default_pos(screen.center() - size / 2.0) + .frame(egui::Frame::window(&ctx.global_style()).fill(t.panel_bg)) + .show(ctx, |ui| { + let (rect, _) = ui.allocate_exact_size(size, Sense::hover()); + draw_children(app, ui, rect.min, w.id, &w.root, &mut acts); + rect + }); + if let Some(r) = resp { + app.auto.add(&format!("scriptui.{}", w.id), r.response.rect, &w.title); + } + if !open { + acts.push(Act::Close(w.id)); + } + } + perform(app, ctx, acts); +} + +/// A dockable ScriptUI panel's content (Window ▸ .jsx). +pub fn panel(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, id: u32) { + let t = app.tokens; + let Some(mut w) = app.session.script_ui.window(id).cloned() else { + ui.painter().text(rect.center(), Align2::CENTER_CENTER, "This ScriptUI panel has closed", Tokens::ui(12.0), t.text_faint); + return; + }; + // Docked panels fill their frame (ScriptUI lays them out at the panel's size). + let (pw, ph) = (rect.width() as f64, rect.height() as f64); + w.root.fixed_bounds = Some([0.0, 0.0, pw.max(w.root.bounds[2]), ph.max(w.root.bounds[3])]); + layout(&mut w); + let mut acts = vec![]; + let content = rect; + egui::ScrollArea::both().id_salt(("scriptui-panel", id)).max_width(content.width()).max_height(content.height()).show(ui, |ui| { + let (r, _) = ui.allocate_exact_size(vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32), Sense::hover()); + draw_children(app, ui, r.min, id, &w.root, &mut acts); + }); + app.auto.add(&format!("scriptui.{id}"), rect, &w.title); + let ctx = ui.ctx().clone(); + perform(app, &ctx, acts); +} + +/// Tab title of a script panel. +pub fn panel_title(app: &EffectcraftApp, id: u32) -> Option { + app.session.script_ui.window(id).map(|w| if w.title.is_empty() { w.script.clone() } else { w.title.clone() }) +} + +/// Close the dock panels whose script window has gone (the script closed it, or a saved +/// workspace from an earlier run). +pub fn sync_panels(app: &mut EffectcraftApp) { + let mut all = vec![]; + app.ui.dock.panels(&mut all); + for f in &app.ui.floating { + all.extend(f.panels.iter().copied()); + } + for p in all { + if let PanelKind::ScriptPanel(id) = p + && app.session.script_ui.window(id).is_none() + { + app.close_panel(p); + } + } +} diff --git a/crates/ui-egui/tests/ui_scriptui.rs b/crates/ui-egui/tests/ui_scriptui.rs new file mode 100644 index 00000000..62102904 --- /dev/null +++ b/crates/ui-egui/tests/ui_scriptui.rs @@ -0,0 +1,112 @@ +//! M13.1 UI: ScriptUI dialogs, palettes and dockable panels drawn from the session's script +//! windows (clicks reach the script's handlers), the branching History panel, and the Window / +//! File ▸ Scripts menus listing scripts (egui_kittest, UI logic only). + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui::{Event, Rect, pos2, vec2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let c = rect(h, id).center(); + h.input_mut().events.push(Event::PointerMoved(c)); + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(3); +} + +fn layer_names(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().session.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +#[test] +fn script_windows_are_drawn_and_clickable() { + let mut h = harness(); + let code = r#" + var w = new Window("palette", "Quick Solid"); + var n = w.add("edittext", undefined, "Card"); + n.characters = 10; + var go = w.add("button", undefined, "Add", { name: "add" }); + go.onClick = function () { app.project.activeItem.layers.addSolid([0, 1, 0], n.text, 50, 50, 1); }; + w.show(); + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "quick.jsx"})).unwrap(); + assert_eq!(r["ok"], true, "{r}"); + h.run_steps(3); + let win = h.state().session.script_ui.windows[0].id; + // Every control has an automation id (by id and by name). + for id in [format!("scriptui.{win}"), format!("scriptui.{win}.1"), format!("scriptui.{win}.2"), format!("scriptui.{win}.add")] { + assert!(h.state().auto.find(&id).is_some(), "missing {id}"); + } + click(&mut h, &format!("scriptui.{win}.add")); + assert_eq!(layer_names(&h), ["Card"]); + // A modal dialog: the script waits in show() until OK is clicked. + let code = r#" + var d = new Window("dialog", "Confirm"); + d.add("statictext", undefined, "Make a null?"); + var g = d.add("group"); + g.add("button", undefined, "Cancel", { name: "cancel" }); + g.add("button", undefined, "OK", { name: "ok" }); + if (d.show() === 1) app.project.activeItem.layers.addNull().name = "Yes"; + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "confirm.jsx"})).unwrap(); + assert_eq!(r["waiting"], true, "{r}"); + h.run_steps(3); + let d = h.state().session.script_ui.windows.iter().find(|w| w.modal).unwrap().id; + click(&mut h, &format!("scriptui.{d}.ok")); + assert_eq!(layer_names(&h), ["Yes", "Card"]); + assert!(h.state().session.script_ui.windows.iter().all(|w| !w.modal)); + // Window ▸ : the sample ScriptUI panel docks as its own tab. + h.state_mut().session.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + h.run_steps(3); + let panel = h.state().session.script_ui.windows.iter().find(|w| w.script == "Layer Tools.jsx").unwrap().id; + assert!(h.state().ui.dock.contains(PanelKind::ScriptPanel(panel)), "the panel docks"); + assert!(h.state().auto.find(&format!("scriptui.{panel}")).is_some()); + // The menus list scripts and panels. + let window_menu = effectcraft_ui_egui::menus::dynamic_entries(h.state(), "Window"); + assert!(window_menu.iter().any(|(l, c, _)| l == "Layer Tools.jsx" && c == "window.scriptPanel")); + let scripts = effectcraft_ui_egui::menus::dynamic_entries(h.state(), "Scripts"); + assert!(scripts.iter().any(|(l, c, _)| l == "Rename Layers.jsx" && c == "file.runScript")); + // Closing the panel's tab closes its script window. + h.state_mut().close_panel(PanelKind::ScriptPanel(panel)); + h.run_steps(2); + assert!(h.state().session.script_ui.window(panel).is_none()); + h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn history_panel_jumps_between_branches() { + let mut h = harness(); + for n in ["A", "B"] { + h.state_mut().session.execute("layer.newSolid", json!({"name": n, "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + } + h.state_mut().session.undo(); + h.state_mut().session.execute("layer.newSolid", json!({"name": "C", "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + assert_eq!(layer_names(&h), ["C", "A"]); + h.state_mut().show_panel(PanelKind::History); + h.run_steps(3); + // Original, New Composition, A, C, and B on its branch. + let states = h.state().session.history_tree(); + assert_eq!(states.len(), 5); + let b = states.iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{b}")); + assert_eq!(layer_names(&h), ["B", "A"]); + let c = h.state().session.history_tree().iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{c}")); + assert_eq!(layer_names(&h), ["C", "A"]); +} diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..da0318c4 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -220,6 +220,35 @@ In expressions, `text.sourceText.style` / `getStyleAt(i, t)` read styles and the (`setFontSize(v, start?, count?)`, `setFillColor`, `setText`, `setJustification`…) return a styled document. +### Scripts and ScriptUI windows + +Scripts that build ScriptUI windows publish them to the session; agents drive them like a user: + +1. `scriptui.list` → `[{window, title, kind: dialog|palette|window|panel, script, modal, size}]`. +2. `scriptui.get {"window": id}` → the control tree (`type`, `name`, `text`, `value`, `checked`, + `items`, `selection`, laid-out `bounds`, `handlers`…). +3. `scriptui.click {"widget": "ok"}` presses a button / toggles a checkbox / picks a radio button or + tab; `scriptui.set {"widget": "#4", "value": "Shot_"}` types into edit text, moves a slider or + picks a list item (index or text); `scriptui.close {"result": 2}` closes. Controls are addressed + by id, `#id`, `properties.name` or text; `window` can be omitted when one window is open. Each + returns the handler run's `{ok, output, error}`. + +A dialog's `show()` waits for the user: the `script.run` / `file.runScript` reply carries +`"waiting": true`, and the script continues (its final output arrives in the reply of the click +that closes the dialog). File ▸ Scripts: `file.scripts.list`, `file.runScript {"name": …}`, +`file.installScript` / `file.installScriptUIPanel {"path": …}`, `window.scriptPanel {"name": …}`. + +### History, puppet recording, plug-ins + +* `edit.history.list` lists every undo state as a tree (undoing then editing keeps the undone + states as a branch); `edit.history.goto {"index": n}` (or `id`, or `steps`) jumps to any of + them. MCP: the `history` tool. +* `puppet.recordPin {"layer": "#1", "pin": "Puppet Pin 1", "samples": [[t, x, y]…]}` records a + drag (t = seconds since it began, layer space) into Position keys at the comp frame rate from the + current time; `puppet.recordOptions {speed, smoothing, useDraftDeformation, showMesh}`. +* `effect.plugins.load {"path": "x.wasm"}` / `effect.plugins.list`: WebAssembly effect plug-ins + ([plugins.md](plugins.md)), then `effect.apply` by id like a built-in. + ## CLI Each invocation runs a headless engine with no window. It opens the demo project unless you pass diff --git a/docs/parity.md b/docs/parity.md index 4437bece..a75d37f5 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -22,9 +22,13 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and -the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and face tracking. +ScriptUI (dialogs that block `show()`, palettes, dockable ScriptUI panels, File ▸ Scripts install +and sample scripts), puppet pin recording and the remaining "better than After Effects" items — a +versioned effect plug-in API with sandboxed WebAssembly plug-ins ([plugins.md](plugins.md)), +branching undo history in the History panel, and a bit-identical rendering test across runs and +thread counts — landed in M13.1 (GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -112,10 +116,10 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | -| Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | +| Automation | ≈ 97% | 0.3 | `.jsxbin`, sockets, ScriptUI resource strings and custom `onDraw` graphics (the core object model landed in M14.4; ScriptUI windows/dialogs/dockable panels with `scriptui.*` agent commands, File ▸ Scripts install + sample scripts, and the effect plug-in API (EFF-6, WebAssembly) landed in M13.1) | | Shapes | ≈ 80% | 1.5 | Lottie can't carry stroke taper/wave (stroke Taper and Wave, Dash 2/Gap 2/Dash 3/Gap 3 and radial-gradient Highlight Length/Angle landed in M13.5; pen tool for shape paths and vertex editing in M6.5) | | Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | -| Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | +| Animation | 70% | 9.0 | puppet depth beyond pins and recording (puppet pin recording with Record Options landed in M13.1), Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 91% | 0.8 | no OpenType feature panel, no extruded strokes, variable-axis animation changes outlines but not advances (vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | | 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer; stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 00000000..5d5b39c0 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,126 @@ +# Effect plug-ins + +EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 1**). +A plug-in effect behaves exactly like a built-in one: it is listed in Effects & Presets (and the +Effect menu) under its category, applied by id (`effect.apply {"effect": "org.example.posterize"}`) +or display name, its parameters are ordinary properties (keyframes, expressions, Effect Controls, +`prop.set`), and it is saved in projects by id. + +There are two ways to write one: + +| | WebAssembly plug-in | Rust plug-in | +|---|---|---| +| What | a `.wasm` (or `.wat`) module | a type implementing `effectcraft_effects::plugin::EffectPlugin` | +| Loaded by | Effect ▸ Load Effect Plug-in…, `effect.plugins.load {path \| folder}`, or the `Plug-ins` folder next to the settings at start-up | `register_plugin(Arc::new(MyEffect))` in an app that links EffectCraft's crates | +| Sandbox | yes: no imports (no files, clock or network), fuel-limited, deterministic floats | none (it is your code) | +| Platforms | desktop and CLI (the wasmi interpreter, crate feature `effectcraft-plugin/wasm`); not the web build | everywhere | + +`effect.plugins.list` lists what is loaded: `{api, wasm, plugins: [{id, name, category, version, +author, source, params}]}`. + +## The contract + +* **Pixels** are premultiplied RGBA `f32` (0–1; values may exceed 1 in 32 bpc projects), row + major, processed **in place**. The buffer is the layer's (after masks and the effects above), + possibly at preview resolution (`scale` = buffer pixels per layer pixel) and padded. +* **Parameters** arrive flattened to `f64`s in the order they are declared: slider, angle, + checkbox (0 / 1) and popup (0-based index) take one value, point two (already in **buffer + pixels**), colour four (RGBA 0–1). +* **Time** is layer time in seconds. +* Rendering must be **deterministic** and may run on several threads at once (a WebAssembly + plug-in gets one instance per thread). +* A failed render (an error code, a trap, running out of fuel) leaves the frame unchanged. + +## The manifest + +Every plug-in describes itself with a JSON manifest: + +```json +{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +} +``` + +* `id`: stable, reverse-domain; letters, digits, `.`, `_`, `-`. `ec.` is reserved for built-ins. + Ids must be unique (a second plug-in with a taken id is refused). +* `category`: one of the Effects & Presets categories (`Blur & Sharpen`, `Stylize`, …) or a new + one, which appears after the built-in categories. +* Parameter types: `slider` (`default`, `min`, `max`, optional `sliderMin`, `sliderMax`, + `decimals`), `angle` (`default` degrees), `checkbox` (`default` bool), `popup` (`options`, + `default` index), `point` (`default` as fractions of the layer size, `[0.5, 0.5]` = centre), + `color` (`default` RGBA 0–1). Parameter ids are letters, digits and `_`. + +## WebAssembly ABI (v1) + +A module with **no imports** that exports: + +| export | signature | | +|---|---|---| +| `memory` | memory | linear memory shared with the host | +| `ec_api_version` | `() -> i32` | must return `1` | +| `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | the UTF-8 JSON manifest in memory | +| `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +| `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; return 0 on success | + +For each frame the host calls `ec_alloc` once for a buffer holding the parameters (`nparams` +little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s), fills it, +calls `ec_render`, and reads the pixels back. Each call gets a fuel budget proportional to the +frame size; a module that loops forever stops with an error instead of hanging the app. + +### Example + +[`examples/plugins/posterize-bands`](../examples/plugins/posterize-bands) is a complete plug-in in +plain Rust (no dependencies): + +```sh +cd examples/plugins/posterize-bands +cargo build --release --target wasm32-unknown-unknown +# then, in EffectCraft: Effect ▸ Load Effect Plug-in… → target/wasm32-unknown-unknown/release/posterize_bands.wasm +# or headless: +effectcraft-cli exec effect.plugins.load '{"path": "target/wasm32-unknown-unknown/release/posterize_bands.wasm"}' +``` + +Any language that compiles to WebAssembly works (C, Zig, AssemblyScript…) as long as the module +has no imports. `crates/plugin/src/tests.rs` has a plug-in written directly in WebAssembly text. + +## Rust plug-ins + +```rust +use std::sync::Arc; +use effectcraft_effects::plugin::*; + +struct Swap(PluginManifest); + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { &self.0 } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { p.swap(0, 2); } + } + Ok(()) + } +} + +register_plugin(Arc::new(Swap(manifest)))?; // before the UI builds its menus +``` + +`PluginParams` reads values by parameter id (`f`, `b`, `point`, `color`). At most 64 plug-in +effects can be registered per process. + +## Compatibility + +The API version is checked when a plug-in loads: a manifest or module for another version is +refused with a message. Within a version the contract above does not change; additions (new +parameter types, new optional exports) bump the version. diff --git a/examples/plugins/posterize-bands/Cargo.lock b/examples/plugins/posterize-bands/Cargo.lock new file mode 100644 index 00000000..c1732e16 --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "posterize-bands" +version = "1.0.0" diff --git a/examples/plugins/posterize-bands/Cargo.toml b/examples/plugins/posterize-bands/Cargo.toml new file mode 100644 index 00000000..825d704a --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "posterize-bands" +description = "Example EffectCraft effect plug-in (WebAssembly, plug-in API v1): posterizes luminance into tinted bands" +version = "1.0.0" +edition = "2024" +license = "MIT OR Apache-2.0" +publish = false + +# Standalone: build with `cargo build --release --target wasm32-unknown-unknown` in this folder. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true +panic = "abort" diff --git a/examples/plugins/posterize-bands/src/lib.rs b/examples/plugins/posterize-bands/src/lib.rs new file mode 100644 index 00000000..9af70943 --- /dev/null +++ b/examples/plugins/posterize-bands/src/lib.rs @@ -0,0 +1,93 @@ +//! Posterize Bands: an example EffectCraft effect plug-in (plug-in API v1, see +//! `docs/plugins.md`). It quantizes each pixel's luminance into `levels` bands and mixes a tint +//! colour in by `mix` percent, optionally only above a `threshold`. +//! +//! Build: `cargo build --release --target wasm32-unknown-unknown`, then load +//! `target/wasm32-unknown-unknown/release/posterize_bands.wasm` with Effect ▸ Load Effect Plug-in +//! (or `effect.plugins.load {"path": …}`), or drop it in the settings folder's `Plug-ins`. + +use std::cell::RefCell; + +/// The manifest: id, name, Effects & Presets category, parameters (flattened in this order: +/// levels, mix, tint r g b a, invert). +const MANIFEST: &str = r#"{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100, "decimals": 1}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +}"#; + +thread_local! { + static BUF: RefCell> = const { RefCell::new(Vec::new()) }; +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_api_version() -> i32 { + 1 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_ptr() -> i32 { + MANIFEST.as_ptr() as i32 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_len() -> i32 { + MANIFEST.len() as i32 +} + +/// One reusable, 8-byte-aligned buffer for the parameters and pixels. +#[unsafe(no_mangle)] +pub extern "C" fn ec_alloc(bytes: i32) -> i32 { + BUF.with(|b| { + let mut b = b.borrow_mut(); + let words = (bytes.max(0) as usize).div_ceil(8); + if b.len() < words { + b.resize(words, 0); + } + b.as_mut_ptr() as i32 + }) +} + +/// # Safety +/// The host passes pointers into the buffer `ec_alloc` returned. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ec_render(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, _time: f64, _scale: f64) -> i32 { + if nparams < 7 { + return 1; + } + let p = unsafe { std::slice::from_raw_parts(params as *const f64, nparams as usize) }; + let px = unsafe { std::slice::from_raw_parts_mut(pixels as *mut [f32; 4], (width.max(0) * height.max(0)) as usize) }; + let levels = p[0].round().max(2.0) as f32; + let mix = (p[1] / 100.0).clamp(0.0, 1.0) as f32; + let tint = [p[2] as f32, p[3] as f32, p[4] as f32]; + let invert = p[6] != 0.0; + for c in px.iter_mut() { + let a = c[3]; + if a <= 0.0 { + continue; + } + // Un-premultiply, band the luminance, re-premultiply. + let (r, g, b) = (c[0] / a, c[1] / a, c[2] / a); + let y = 0.2126 * r + 0.7152 * g + 0.0722 * b; + let mut band = (y * levels).floor().min(levels - 1.0) / (levels - 1.0); + if invert { + band = 1.0 - band; + } + for (i, ch) in [r, g, b].into_iter().enumerate() { + let banded = if y > 0.0 { ch * band / y.max(1e-6) } else { band }; + let v = banded * (1.0 - mix) + band * tint[i] * mix; + c[i] = v.max(0.0) * a; + } + } + 0 +} From d209fa001f7361301a31c8aa506b90e234de8c81 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:36:12 -0400 Subject: [PATCH 28/36] M13.3: face tracking methods, Face Track Points / Measurements, camera lens distortion, docs Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/camera_tracker.rs | 94 ++++++++++- crates/effects/src/face_track.rs | 45 +++++ crates/effects/src/lib.rs | 2 + crates/engine/src/commands/camera_cmds.rs | 18 +- crates/engine/src/commands/mask_interp.rs | 133 ++++++++++++++- crates/engine/src/lib.rs | 2 + crates/engine/src/mask_track.rs | 159 ++++++++++++++++-- crates/engine/src/tests_face.rs | 106 ++++++++++++ crates/track/src/camtrack/bundle.rs | 46 ++++- crates/track/src/camtrack/mod.rs | 109 +++++++++++- crates/track/src/camtrack/solver.rs | 125 ++++++++++++-- crates/track/src/camtrack/tests.rs | 65 ++++++- crates/track/src/face.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 14 +- crates/ui-egui/src/panels/tracker.rs | 28 ++- docs/agents.md | 27 ++- docs/architecture.md | 17 +- docs/effects.md | 10 +- docs/parity.md | 8 +- 19 files changed, 939 insertions(+), 74 deletions(-) create mode 100644 crates/effects/src/face_track.rs create mode 100644 crates/engine/src/tests_face.rs diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs index dd97b585..ff4e51fe 100644 --- a/crates/effects/src/camera_tracker.rs +++ b/crates/effects/src/camera_tracker.rs @@ -8,8 +8,12 @@ //! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing //! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. //! -//! The effect passes its input through; with **Render Track Points** on it draws the solved -//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! **Solve Lens Distortion** (Advanced; also on with Detailed Analysis) solves radial lens +//! distortion `k1, k2` jointly with the camera; **Undistort Footage** then renders the input +//! undistorted (the pinhole image the solved camera sees) so 3D layers line up with it. +//! +//! The effect passes its input through (undistorted when asked); with **Render Track Points** +//! on it draws the solved points (as the viewer shows them) into the frame. The viewer overlay, target and the Create //! commands live in the engine and the UI. use std::collections::HashMap; @@ -19,6 +23,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; +use rayon::prelude::*; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -58,6 +63,8 @@ pub fn specs() -> Vec { p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/lensDistortion", "Solve Lens Distortion", Value::Bool(false), ParamUi::Checkbox), + p("advanced/undistort", "Undistort Footage", Value::Bool(false), ParamUi::Checkbox), p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), @@ -104,13 +111,15 @@ pub fn settings(params: &Params) -> SolveSettings { hfov: f(params, "horizontalAngleOfView"), method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), deleted: del, + lens_distortion: b(params, "advanced/lensDistortion") || b(params, "advanced/detailedAnalysis"), } } /// What a solve depends on besides the tracks. pub fn solve_key(st: &SolveSettings) -> String { let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; - format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) + let lens = if st.lens_distortion { "|lens" } else { "" }; + format!("{:?}|{hfov}|{:?}|{:?}{lens}", st.shot, st.method, st.deleted) } fn fnv(bytes: &[u8]) -> u64 { @@ -165,18 +174,49 @@ pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) } -/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +/// The solved points visible on frame `k`: (id, layer pixel position in the footage, camera +/// depth). pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + projected_with(solve, k, false) +} + +/// [`projected`] into the footage, or (`undistorted`) into the undistorted image. +pub fn projected_with(solve: &CameraSolve, k: usize, undistorted: bool) -> Vec<(u32, [f64; 2], f64)> { let Some(cam) = solve.frames.get(k) else { return vec![] }; solve .visible(k) .filter_map(|p| { let c = cam.to_cam(p.pos); - cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + solve.project(k, p.pos, undistorted).map(|q| (p.id, q, c[2])) }) .collect() } +/// Whether the instance renders its input undistorted (Undistort Footage with a solved +/// distortion). +pub fn undistorts(params: &Params, solve: &CameraSolve) -> bool { + b(params, "advanced/undistort") && solve.distortion.is_some_and(|d| !d.is_none()) +} + +/// The input resampled through the solved distortion: every output pixel shows the footage +/// at its distorted position (transparent where that falls outside the layer). +fn undistort(buf: &Buf, sol: &CameraSolve) -> Buf { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Image::new(buf.img.width, buf.img.height); + let size = sol.size; + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [(x as f64 + 0.5 - buf.offset[0]) / sc, (y as f64 + 0.5 - buf.offset[1]) / sc]; + let q = sol.to_image(p); + if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] { + continue; + } + *px = buf.img.sample_bilinear_clamped(q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]); + } + }); + Buf { img: out, offset: buf.offset, scale: buf.scale } +} + /// Median of the depths. pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { let mut z: Vec = v.iter().map(|x| x.2).collect(); @@ -188,15 +228,23 @@ pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { } fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { - if !b(ctx.params, "renderTrackPoints") { + let points = b(ctx.params, "renderTrackPoints"); + if !points && !b(ctx.params, "advanced/undistort") { return buf; } let Some(sol) = solve(ctx.params) else { return buf }; if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { return buf; } + let und = undistorts(ctx.params, &sol); + if und { + buf = undistort(&buf, &sol); + } + if !points { + return buf; + } let Some(k) = sol.frame_at(ctx.time) else { return buf }; - let pts = projected(&sol, k); + let pts = projected_with(&sol, k, und); let med = median_depth(&pts); let size = f(ctx.params, "trackPointSize"); let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; @@ -287,5 +335,37 @@ mod tests { let st = settings(&Params::default()); assert_eq!(st.shot, ShotType::FixedAngle); assert!(solve_key(&st).contains("FixedAngle")); + assert!(!st.lens_distortion && !solve_key(&st).contains("lens")); + } + + #[test] + fn undistort_footage_remaps_through_the_solved_lens() { + use effectcraft_track::camtrack::Distortion; + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + distortion: Some(Distortion { k1: -0.2, k2: 0.0, radius: 40.0 }), + ..Default::default() + }; + let mut params = params_with(&sol, false); + params.values.insert("advanced/undistort".into(), Value::Bool(true)); + assert!(undistorts(¶ms, &sol)); + // A horizontal ramp: undistortion keeps the centre and pulls edge content outwards. + let mut img = Image::new(64, 48); + for (i, px) in img.data.iter_mut().enumerate() { + let x = (i % 64) as f32 / 64.0; + *px = [x, x, x, 1.0]; + } + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let at = |im: &Image, x: u32, y: u32| im.data[im.idx(x, y)][0]; + assert!((at(&out.img, 32, 24) - at(&img, 32, 24)).abs() < 0.02); + // Barrel distortion compresses the edges: the ideal pixel at x = 56 shows footage nearer the centre. + assert!(at(&out.img, 56, 24) < at(&img, 56, 24) - 0.01, "{} vs {}", at(&out.img, 56, 24), at(&img, 56, 24)); + let sp = sol.to_image([56.0, 24.0]); + assert!(sp[0] < 56.0 && (sol.to_ideal(sp)[0] - 56.0).abs() < 1e-6); } } diff --git a/crates/effects/src/face_track.rs b/crates/effects/src/face_track.rs new file mode 100644 index 00000000..7b925165 --- /dev/null +++ b/crates/effects/src/face_track.rs @@ -0,0 +1,45 @@ +//! Face tracking results (Tracker panel ▸ Method ▸ Face Tracking (Detailed Features)): +//! +//! - **Face Track Points** holds one point control per facial landmark (eyebrows, eyes and +//! pupils, nose, nostrils, mouth, chin and jaw), keyed on every tracked frame by the engine's +//! mask tracker ([`effectcraft_track::face`]); expressions and other layers can follow them. +//! - **Face Measurements** holds the measurements derived from those points (head position, +//! scale and orientation, eye openness, eyebrow raise, mouth openness, width and offset), keyed +//! by Extract & Copy Face Measurements (`track.extractFaceMeasurements`). +//! +//! Both pass their input through unchanged: they are data, like expression controls. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_track::face::{LANDMARKS, MEASUREMENTS}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, num, p, slider}; + +/// Effect ids. +pub const POINTS_ID: &str = "ec.utility.facetrackpoints"; +pub const MEASUREMENTS_ID: &str = "ec.utility.facemeasurements"; + +fn passthrough(_: &EffectCtx, b: Buf) -> Buf { + b +} + +fn measurement_ui(id: &str) -> ParamUi { + match id { + "headPositionX" | "headPositionY" => slider(-100_000.0, 100_000.0, 0.0, 1920.0, 1), + "headOrientationX" | "headOrientationY" | "headOrientationZ" => slider(-180.0, 180.0, -90.0, 90.0, 1), + _ => slider(-100_000.0, 100_000.0, 0.0, 200.0, 1), + } +} + +pub fn specs() -> Vec { + let points: Vec = LANDMARKS + .iter() + // Point defaults are fractions of the layer size: the canonical face in the middle. + .map(|(id, name, q)| p(id, name, Value::Vec2([0.5 + 0.25 * q[0], 0.5 + 0.25 * q[1]]), ParamUi::Point)) + .collect(); + let measurements: Vec = MEASUREMENTS.iter().map(|(id, name)| p(id, name, num(0.0), measurement_ui(id))).collect(); + vec![ + EffectSpec { id: POINTS_ID, name: "Face Track Points", category: "Utility", params: points, render: passthrough, gpu: false, float: true }, + EffectSpec { id: MEASUREMENTS_ID, name: "Face Measurements", category: "Utility", params: measurements, render: passthrough, gpu: false, float: true }, + ] +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..febdd9da 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -22,6 +22,7 @@ mod distort; mod distort2; mod distort3; pub mod distort4; +pub mod face_track; mod generate; mod generate2; mod generate3; @@ -412,6 +413,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(puppet::specs()); v.extend(warp_stab::specs()); v.extend(camera_tracker::specs()); + v.extend(face_track::specs()); v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs index 5d454aed..68cd14fb 100644 --- a/crates/engine/src/commands/camera_cmds.rs +++ b/crates/engine/src/commands/camera_cmds.rs @@ -23,7 +23,7 @@ pub fn specs() -> Vec { "Track Camera", ["Animation"], None, - "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}", can_track, track_camera ), @@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { return Err(bad("track.camera", "aov: 1..170 degrees")); } let detailed = b_p(p, "detailed"); + let lens = b_p(p, "lensDistortion"); + let undistort = b_p(p, "undistort"); let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); let uid = match existing { Some(u) => u, @@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? } }; - if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() { s.edit("3D Camera Tracker Settings", None, |proj, _| { let g = proj .comp_mut(cid) @@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { if let Some(d) = detailed { set("advanced/detailedAnalysis", KV::Bool(d)); } + if let Some(d) = lens { + set("advanced/lensDistortion", KV::Bool(d)); + } + if let Some(d) = undistort { + set("advanced/undistort", KV::Bool(d)); + } Ok(()) })?; } @@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result { out["shotType"] = json!(settings.shot.label()); out["solveMethod"] = json!(settings.method.label()); out["deleted"] = json!(settings.deleted.len()); + out["lensDistortionSolved"] = json!(settings.lens_distortion); let solve = ct::solve(¶ms); out["solved"] = json!(solve.is_some()); if let Some(sv) = &solve { @@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result { out["focalLength"] = json!(f); out["horizontalAngleOfView"] = json!(sv.hfov(f)); out["groundPlane"] = json!(sv.ground.is_some()); + out["lensDistortion"] = match &sv.distortion { + Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}), + None => Value::Null, + }; + out["undistort"] = json!(ct::undistorts(¶ms, sv)); if let Some(pl) = placed(&s.project, cid, lid, uid) { let k = pl.frame_at(s.time_of(cid)); let (pos, ori, zoom) = pl.camera(k); diff --git a/crates/engine/src/commands/mask_interp.rs b/crates/engine/src/commands/mask_interp.rs index 5df40c6d..a4548944 100644 --- a/crates/engine/src/commands/mask_interp.rs +++ b/crates/engine/src/commands/mask_interp.rs @@ -1,5 +1,6 @@ -//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask -//! Interpolation (Window ▸ Mask Interpolation). +//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face +//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation +//! (Window ▸ Mask Interpolation). use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching}; @@ -20,7 +21,7 @@ pub fn specs() -> Vec { "Track Mask", ["Animation"], None, - "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", has_mask, track_mask ), @@ -29,10 +30,19 @@ pub fn specs() -> Vec { "Mask Tracking Method", [], None, - "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}", + "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}", super::always, mask_method ), + cmd!( + "track.extractFaceMeasurements", + "Extract & Copy Face Measurements", + [], + None, + "{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys", + has_face_points, + extract_face + ), cmd!( "mask.interpolate", "Apply Mask Interpolation", @@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> { fn method_p(p: &Value, cmd: &str) -> Result> { match str_p(p, "method") { - Some(m) => MaskMethod::from_name(m) - .map(Some) - .ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))), + Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| { + bad( + cmd, + format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"), + ) + }), None => Ok(None), } } @@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result { })) } +// ---------------------------------------------------------------- face measurements + +fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> { + use effectcraft_effects::face_track::POINTS_ID; + let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID)); + if p.get("layer").is_some() { + return layer_p(s, p, "track.extractFaceMeasurements").ok(); + } + let cid = s.active_comp_id()?; + let comp = s.project.comp(cid)?; + let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l)); + sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id)) +} + +fn has_face_points(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into()) +} + +/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them +/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard. +fn extract_face(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID}; + use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure}; + const CMD: &str = "track.extractFaceMeasurements"; + let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?; + let mut times: Vec = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect(); + times.sort(); + times.dedup(); + if times.is_empty() { + times.push(layer.layer_time(s.time_of(cid))); + } + let rows: Vec<(Tick, [f64; 14])> = { + let pts_at = |t: Tick| { + let mut l = [[0.0; 2]; N]; + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) { + l[i] = v; + } + } + l + }; + let reference = face_frame(&pts_at(times[0])).1; + times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect() + }; + let comp_times: Vec = times.iter().map(|t| layer.comp_time(*t)).collect(); + let fd = comp.frame_duration(); + let uid = s.edit("Extract Face Measurements", None, |proj, _| { + let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?; + let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?; + for (k, (id, _)) in MEASUREMENTS.iter().enumerate() { + let Some(pr) = g.get_mut(id) else { continue }; + pr.keys.clear(); + for (t, m) in &rows { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k]))); + } + } + Ok(uid) + })?; + // Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes). + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO); + let mut clip = vec![]; + for (id, _) in MEASUREMENTS { + let Some(pr) = mg.get(id) else { continue }; + let Some(path) = layer.props.match_path_of(pr.uid) else { continue }; + let keys = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = layer.comp_time(k.time) - earliest; + k + }) + .collect(); + clip.push(crate::KeyClip { layer: lid, path, keys }); + } + s.state.key_clipboard = clip; + s.state.clip_is_keys = true; + // The same data as text (tab-separated, one row per frame) for other applications. + let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime"); + for (_, name) in MEASUREMENTS { + text.push('\t'); + text.push_str(name); + } + text.push('\n'); + let mut table = vec![]; + for ((_, m), ct) in rows.iter().zip(&comp_times) { + let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 }; + text.push_str(&format!("{frame}\t{:.4}", ct.seconds())); + for v in m { + text.push_str(&format!("\t{v:.3}")); + } + text.push('\n'); + let obj: serde_json::Map = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect(); + table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj})); + } + s.bump(); + Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text})) +} + // ---------------------------------------------------------------- mask interpolation /// Mask Interpolation panel options (kept in the editor state, serde for agents). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 057c7d10..538e8844 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -753,6 +753,8 @@ mod tests_effects; #[cfg(test)] mod tests_essential; #[cfg(test)] +mod tests_face; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..a5741052 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -6,6 +6,12 @@ //! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the //! chosen motion model), and every frame's motion is applied to the mask's vertices and //! tangents, so the shape's Bezier structure is kept. +//! +//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn +//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path +//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track +//! Points** effect on the layer (created on first use), from which Extract & Copy Face +//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,11 +37,22 @@ pub enum MaskMethod { PositionScaleRotation, PositionScaleRotationSkew, Perspective, + /// Face Tracking (Outline Only). + FaceOutline, + /// Face Tracking (Detailed Features). + FaceDetailed, } impl MaskMethod { - pub const ALL: [MaskMethod; 5] = - [MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective]; + pub const ALL: [MaskMethod; 7] = [ + MaskMethod::Position, + MaskMethod::PositionScale, + MaskMethod::PositionScaleRotation, + MaskMethod::PositionScaleRotationSkew, + MaskMethod::Perspective, + MaskMethod::FaceOutline, + MaskMethod::FaceDetailed, + ]; pub fn label(self) -> &'static str { match self { MaskMethod::Position => "Position", @@ -43,8 +60,14 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "Position, Scale & Rotation", MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew", MaskMethod::Perspective => "Perspective", + MaskMethod::FaceOutline => "Face Tracking (Outline Only)", + MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)", } } + /// One of the Face Tracking methods. + pub fn is_face(self) -> bool { + matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed) + } pub fn id(self) -> &'static str { match self { MaskMethod::Position => "position", @@ -52,6 +75,8 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "positionScaleRotation", MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew", MaskMethod::Perspective => "perspective", + MaskMethod::FaceOutline => "faceOutline", + MaskMethod::FaceDetailed => "faceDetailed", } } pub fn from_name(s: &str) -> Option { @@ -67,6 +92,7 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => Model::Similarity, MaskMethod::PositionScaleRotationSkew => Model::Affine, MaskMethod::Perspective => Model::Homography, + MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity, } } } @@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath { } } -/// One tracked frame: (layer time, comp time, mask path). -type MaskFrame = (Tick, Tick, ShapePath); +/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom +/// tangents. `like` lends its feather points. +pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath { + let m = poly.len(); + let n = n.clamp(3, m.max(3)); + let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect(); + let tan: Vec<[f64; 2]> = (0..n) + .map(|i| { + let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]); + [(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0] + }) + .collect(); + ShapePath { + vertices: v, + in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(), + out_tangents: tan, + closed: true, + feather: if like.len() == n { like.feather.clone() } else { vec![] }, + } +} + +/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features). +type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>); #[derive(Default)] pub struct MaskTrackShared { @@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + if w.method.is_face() { + return run_face(&w, shared, first, &frame_at, layer, t0); + } let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); let mut path = w.path.clone(); - lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None)); let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { @@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { break; }; path = transform_path(&path, &step.motion); - lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None)); let mut s = lock(&shared.state); s.done = k as u64; s.elapsed = t0.elapsed().as_secs_f64(); @@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { s.finished = true; } +/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks. +fn run_face( + w: &MaskWork, + shared: &MaskTrackShared, + first: (Arc, [f64; 2]), + frame_at: &(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync), + layer: &effectcraft_project::Layer, + t0: web_time::Instant, +) { + let detailed = w.method == MaskMethod::FaceDetailed; + let b = trk::mask::bounds(&flatten(&w.path, 8)); + let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else { + let mut s = lock(&shared.state); + s.error = Some("no face found inside the mask (draw the mask around a face)".into()); + s.finished = true; + return; + }; + let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks)); + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts)); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + let Some(fit) = res else { + lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds())); + break; + }; + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts)); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + crate::offload::report(s.done, s.total); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// The layer's Face Track Points effect (created when missing). Returns its uid. +pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option { + face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID) +} + +/// The layer's effect `id`, applied when missing (data effects of face tracking). +pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option { + let spec = effectcraft_effects::find(id)?; + let l = p.comp(comp)?.layer(layer)?; + if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) { + return Some(g.uid); + } + let (w, h) = effectcraft_render::source_size(p, l); + let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] }; + let mut next = p.next_id; + let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size); + let uid = g.uid; + let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?; + fx.children.push(g.into()); + p.next_id = next; + Some(uid) +} + fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) { + let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None }; let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return }; - let Some(g) = layer.props.find_group_mut(job.mask) else { return }; - let Some(pr) = g.get_mut("path") else { return }; - for (lt, _, path) in frames { - effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + if let Some(g) = layer.props.find_group_mut(job.mask) + && let Some(pr) = g.get_mut("path") + { + for (lt, _, path, _) in frames { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + } + } + if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) { + for (lt, _, _, pts) in frames { + let Some(pts) = pts else { continue }; + for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() { + if let Some(pr) = g.get_mut(id) { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i]))); + } + } + } } } @@ -242,7 +379,7 @@ impl Session { return Err("a track analysis is already running".into()); } self.poll_mask_track(); - if !wait && self.offloads() { + if !wait && self.offloads() && !work.method.is_face() { let MaskWork { comp, layer, path, method, times, .. } = work; return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } diff --git a/crates/engine/src/tests_face.rs b/crates/engine/src/tests_face.rs new file mode 100644 index 00000000..dad97935 --- /dev/null +++ b/crates/engine/src/tests_face.rs @@ -0,0 +1,106 @@ +//! Face tracking end to end through the commands, on synthetic face footage generated in-test +//! (a moving, rotating face that opens its mouth and blinks). + +use effectcraft_keyframe::{ShapePath, Value as KV}; +use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks}; +use serde_json::json; + +use crate::mask_track::MaskMethod; +use crate::tests_track::{H, W, frame_time, setup}; + +fn face(f: u32) -> FaceParams { + let t = f as f64; + FaceParams { + center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()], + height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()), + roll: 12.0 * (t * 0.18).sin(), + yaw: 10.0 * (t * 0.12).sin(), + mouth_open: 0.5 + 0.45 * (t * 0.3).sin(), + eye_open: [0.85, 0.85], + ..Default::default() + } +} + +#[test] +fn face_tracking_keys_outline_points_and_measurements() { + let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5)); + assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed)); + assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline)); + // A rough mask around the face. + let c = face(0).center; + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let mask = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24, "{r}"); + assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed); + let comp = s.active_comp().unwrap(); + let layer = comp.layer(clip).unwrap(); + // The mask follows the face outline. + let path = layer.props.find_group(mask).unwrap().get("path").unwrap(); + assert_eq!(path.keys.len(), 25); + for f in [0, 12, 24] { + let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() }; + let fp = face(f); + let (sn, cs) = fp.roll.to_radians().sin_cos(); + for v in &p.vertices { + let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height]; + let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]); + let r = ((u / 0.75).powi(2) + w * w).sqrt(); + assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}"); + } + } + // Face Track Points: one keyed point per landmark, on the face. + let fx = layer.effects().unwrap(); + let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points"); + let mut worst: f64 = 0.0; + for f in [0, 6, 12, 18, 24] { + let truth = synth_landmarks(&face(f)); + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + let pr = pts.get(id).unwrap(); + assert_eq!(pr.keys.len(), 25, "{id}"); + let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() }; + worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height); + } + } + assert!(worst < 0.1, "worst landmark error {worst} face heights"); + // Extract & Copy Face Measurements. + assert!(s.is_enabled("track.extractFaceMeasurements")); + let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness")); + assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len()); + assert!(s.state.clip_is_keys); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements"); + let mo = m.get("mouthOpenness").unwrap(); + assert_eq!(mo.keys.len(), 25); + // Mouth openness follows the footage: widest open where the generator opens it most. + let open = |f: u32| mo.value_at(frame_time(f)).as_f64(); + let (fmax, fmin) = ( + (0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + (0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + ); + assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin)); + let roll = m.get("headOrientationZ").unwrap(); + for f in [0, 10, 20] { + assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0); + } + // One undo step for the extraction. + assert!(s.undo()); + assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID)); + // Outline Only keys just the mask. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let m2 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap(); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2); + // A mask without a face reports it. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap(); + let m3 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true})); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty()); +} diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 72cb2f17..077f04b7 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -10,6 +10,7 @@ use rayon::prelude::*; +use super::Distortion; use super::geometry::Pose; use super::linalg::*; @@ -19,6 +20,8 @@ pub struct BaCam { pub pose: Pose, /// Focal length (pixels). pub f: f64, + /// Lens distortion (shared by all cameras; adjusted when [`BaOpts::distortion`]). + pub dist: Distortion, } /// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal @@ -50,11 +53,13 @@ pub struct BaOpts { /// Huber threshold (pixels). pub huber: f64, pub max_iter: usize, + /// Adjust the shared radial distortion `(k1, k2)` too. + pub distortion: bool, } impl Default for BaOpts { fn default() -> Self { - BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30, distortion: false } } } @@ -77,7 +82,8 @@ pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { if p[2] <= 1e-9 { return None; } - Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) + let q = c.dist.distort([c.f * p[0] / p[2], c.f * p[1] / p[2]]); + Some([q[0] - uv[0], q[1] - uv[1]]) } /// Total robust cost. @@ -108,6 +114,8 @@ struct Layout { fidx: Vec, /// Index of the shared focal parameter. shared: Option, + /// Index of `k1` (then `k2`) when the distortion is adjusted. + dist: Option, n: usize, rot_only: bool, } @@ -132,7 +140,11 @@ fn layout(ncam: usize, o: &BaOpts) -> Layout { n += 1; n - 1 }); - Layout { start, fidx, shared, n, rot_only: o.fix_centers } + let dist = o.distortion.then(|| { + n += 2; + n - 2 + }); + Layout { start, fidx, shared, dist, n, rot_only: o.fix_centers } } /// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). @@ -204,12 +216,21 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> continue; } let iz = 1.0 / xc[2]; - let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; - let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let p0 = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let pd = c.dist.distort(p0); + let r = [pd[0] - ob.uv[0], pd[1] - ob.uv[1]]; let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); let w = if e <= o.huber { 1.0 } else { o.huber / e }; - let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + let a0 = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + // Through the distortion: d r = D d p. + let (dj, dk) = c.dist.jacobian(p0); + let a = [0, 1].map(|i| [0, 1, 2].map(|k| dj[i][0] * a0[0][k] + dj[i][1] * a0[1][k])); + let p = [dj[0][0] * p0[0] + dj[0][1] * p0[1], dj[1][0] * p0[0] + dj[1][1] * p0[1]]; cam_jac(&l, ci, c, xc, &a, p, &mut jc); + if let Some(g) = l.dist { + jc.push((g, [dk[0][0], dk[1][0]])); + jc.push((g + 1, [dk[0][1], dk[1][1]])); + } // Point Jacobian: A R. let jx = if o.fix_points { [[0.0; 3]; 2] @@ -236,7 +257,8 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; // Each observation of a point is on another camera: only the shared focal // length repeats. - let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + let repeats = Some(*ia) == l.shared || l.dist.is_some_and(|g| *ia == g || *ia == g + 1); + let found = if repeats { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), @@ -385,6 +407,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> if let Some(g) = l.shared { c.f *= dc[g].clamp(-0.5, 0.5).exp(); } + if let Some(g) = l.dist { + c.dist.k1 += dc[g].clamp(-0.2, 0.2); + c.dist.k2 += dc[g + 1].clamp(-0.2, 0.2); + } } let mut np: Vec = pts.to_vec(); if !o.fix_points { @@ -445,7 +471,11 @@ mod tests { let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); let f = 900.0; let cams_true: Vec = (0..6) - .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .map(|i| BaCam { + pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, + f, + dist: Distortion::NONE, + }) .collect(); let mut obs = vec![]; for (ci, c) in cams_true.iter().enumerate() { diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 76c0637f..e8cd28da 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -28,8 +28,15 @@ //! tripod solution when it explains the tracks as well as the general one (a quick rotation fit //! at the general solution's focal length skips the tripod search when it clearly fails). //! -//! The camera model has square pixels, the principal point at the layer centre and no lens -//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! 5. **Lens distortion** (Solve Lens Distortion, or Detailed Analysis): radial distortion +//! `p_d = p (1 + k1 ρ² + k2 ρ⁴)` (ρ = distance from the principal point over half the frame +//! diagonal; Brown's model without the tangential terms) is solved jointly with the cameras +//! and points in a final bundle adjustment on the raw tracks; the whole solve is then repeated +//! on tracks undistorted with that estimate (the linear initialisations assume a pinhole +//! camera) and adjusted jointly again. The solve stores `k1, k2`; the effect can undistort +//! its output so composited 3D layers line up. +//! +//! The camera model has square pixels and the principal point at the layer centre. Camera space is x right, y down, z forward (After Effects' convention); a camera //! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. //! //! The solve is stored in a canonical frame: the first solved frame's camera is at the origin @@ -105,6 +112,69 @@ impl CameraTracks { } } +/// Radial lens distortion around the principal point (see the module docs). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Distortion { + pub k1: f64, + pub k2: f64, + /// Normalisation radius ρ = 1 (pixels): half the frame diagonal; 0 = no distortion. + pub radius: f64, +} + +impl Distortion { + pub const NONE: Distortion = Distortion { k1: 0.0, k2: 0.0, radius: 0.0 }; + + /// No distortion for frames of `size` (the radius set, coefficients zero). + pub fn for_size(size: [f64; 2]) -> Distortion { + Distortion { k1: 0.0, k2: 0.0, radius: 0.5 * size[0].hypot(size[1]) } + } + pub fn is_none(&self) -> bool { + self.radius <= 0.0 || (self.k1 == 0.0 && self.k2 == 0.0) + } + fn rho2(&self, p: [f64; 2]) -> f64 { + (p[0] * p[0] + p[1] * p[1]) / (self.radius * self.radius) + } + /// Ideal → distorted (pixels relative to the principal point). + pub fn distort(&self, p: [f64; 2]) -> [f64; 2] { + if self.radius <= 0.0 { + return p; + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + [p[0] * d, p[1] * d] + } + /// Distorted → ideal (fixed-point iteration). + pub fn undistort(&self, q: [f64; 2]) -> [f64; 2] { + if self.is_none() { + return q; + } + let mut p = q; + for _ in 0..30 { + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + if d.abs() < 1e-6 { + break; + } + p = [q[0] / d, q[1] / d]; + } + p + } + /// `d distort / d p` (2 × 2, row-major) and `d distort / d (k1, k2)` (rows x, y). + pub fn jacobian(&self, p: [f64; 2]) -> ([[f64; 2]; 2], [[f64; 2]; 2]) { + if self.radius <= 0.0 { + return ([[1.0, 0.0], [0.0, 1.0]], [[0.0; 2]; 2]); + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + // d(r2)/dp = 2 p / R². + let dd = (self.k1 + 2.0 * self.k2 * r2) * 2.0 / (self.radius * self.radius); + let j = [[d + p[0] * dd * p[0], p[0] * dd * p[1]], [p[1] * dd * p[0], d + p[1] * dd * p[1]]]; + let k = [[p[0] * r2, p[0] * r2 * r2], [p[1] * r2, p[1] * r2 * r2]]; + (j, k) + } +} + /// 3D Camera Tracker ▸ Shot Type. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -178,6 +248,8 @@ pub struct SolveSettings { pub method: SolveMethod, /// Track ids the user deleted. pub deleted: Vec, + /// Solve radial lens distortion jointly with the cameras. + pub lens_distortion: bool, } /// The solved camera of one frame. @@ -261,6 +333,9 @@ pub struct CameraSolve { pub points: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub ground: Option, + /// The solved lens distortion (Solve Lens Distortion). + #[serde(skip_serializing_if = "Option::is_none")] + pub distortion: Option, } impl CameraSolve { @@ -355,6 +430,36 @@ impl CameraSolve { } } + /// An ideal (pinhole) image point → where it appears in the footage (layer pixels), through + /// the solved lens distortion. + pub fn to_image(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.distort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// A footage point → its ideal (undistorted) position. + pub fn to_ideal(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.undistort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// Project world point `x` on frame `k`: through the lens distortion into the footage, or + /// (`undistorted`) into the undistorted image. + pub fn project(&self, k: usize, x: V3, undistorted: bool) -> Option<[f64; 2]> { + let q = self.frames.get(k)?.project(self.size, x)?; + Some(if undistorted { q } else { self.to_image(q) }) + } + /// The first solved frame. pub fn first_solved(&self) -> Option { self.frames.iter().position(|c| c.solved) diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index 85edd2bd..5ed7b81b 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; use super::geometry::{self, Pose}; use super::linalg::*; -use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use super::{CameraSolve, CameraTracks, Distortion, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; use crate::fit::{Model, ransac}; fn debug() -> bool { @@ -142,6 +142,8 @@ struct Cfg { focal: FocalMode, /// Registration estimates a focal length per frame. per_frame_f: bool, + /// The global adjustments also adjust the lens distortion. + distortion: bool, } const F_PRIOR: f64 = 30.0; @@ -183,7 +185,7 @@ impl Recon { &mut cams, &mut pts, &obs, - &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters, distortion: cfg.distortion }, ); for (k, c) in frames.iter().zip(&cams) { self.cams.insert(*k, *c); @@ -289,6 +291,7 @@ impl Recon { fixed: vec![false], huber, max_iter: 40, + distortion: false, }; let mut best: Option<(usize, BaCam)> = None; // Robust start (large Huber threshold), then tighten; from the prediction and, failing @@ -371,7 +374,11 @@ fn predict(r: &Recon, k: usize) -> Option { match (before, after) { (Some((ka, a)), Some((kb, b))) => { let t = (k - ka) as f64 / (kb - ka) as f64; - Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + Some(BaCam { + pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, + f: a.f * (b.f / a.f).powf(t), + dist: a.dist, + }) } (Some((ka, a)), None) => { // Constant velocity from the two previous cameras. @@ -381,7 +388,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: a.f, dist: a.dist }) } None => Some(a), } @@ -392,7 +399,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: b.f, dist: b.dist }) } None => Some(b), }, @@ -459,8 +466,8 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let (a, b) = (kfs[0], kfs[j]); let mut r = Recon::default(); - r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); - r.cams.insert(b, BaCam { pose: init.0, f }); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); + r.cams.insert(b, BaCam { pose: init.0, f, dist: Distortion::NONE }); r.pts = init.1.clone(); let inner = |r: &mut Recon, frames: &[usize]| { let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; @@ -495,7 +502,7 @@ fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f /// Tripod reconstruction over keyframes `kfs` at focal length `f`. fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let mut r = Recon::default(); - r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); for w in kfs.windows(2) { if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { @@ -519,7 +526,7 @@ fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&Atomi rel = geometry::procrustes(&ia, &ib)?; } let pa = r.cams[&a].pose.r; - r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f, dist: Distortion::NONE }); } r.triangulate(d, Kind::Tripod, 0.0, 20.0); let frames: Vec = r.cams.keys().copied().collect(); @@ -640,8 +647,101 @@ fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { if best.1 <= errs[i] { best.0.exp() } else { fs[i] } } +/// A solve before its canonical output. +struct Core { + d: Data, + r: Recon, + method: SolveMethod, + kind: Kind, + cfg: Cfg, + focal: FocalMode, + kfs: Vec, +} + +/// `tracks` with every position undistorted by `dist`. +fn undistorted(tracks: &CameraTracks, dist: &Distortion) -> CameraTracks { + let c = [tracks.size[0] * 0.5, tracks.size[1] * 0.5]; + let mut t = tracks.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = dist.undistort([p[0] as f64 - c[0], p[1] as f64 - c[1]]); + *p = [(q[0] + c[0]) as f32, (q[1] + c[1]) as f32]; + } + } + t +} + /// Solve the camera of a tracked clip. pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + if !s.lens_distortion { + let c = solve_core(tracks, s, cancel)?; + return Ok(finish(&c.d, tracks, s, c.r, c.method, c.kind)); + } + // Lens distortion: solve, adjust (k1, k2) jointly on the raw tracks, then solve again on + // tracks undistorted with the estimate and adjust jointly once more. + let mut dist = Distortion::for_size(tracks.size); + let mut out = None; + for pass in 0..2 { + let und = (pass > 0).then(|| undistorted(tracks, &dist)); + let c = solve_core(und.as_ref().unwrap_or(tracks), s, cancel)?; + let d = Data::new(tracks, &s.deleted); + d.set_noise(c.d.noise()); + let mut r = c.r; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + let cfg = Cfg { distortion: true, ..c.cfg }; + let all: Vec = r.cams.keys().copied().collect(); + let frames: Vec = if all.len() <= 160 { all.clone() } else { c.kfs.iter().copied().filter(|k| r.cams.contains_key(k)).collect() }; + if frames.len() < 2 { + return err("the camera could not be solved (not enough parallax or features)"); + } + // The pinhole solve's outliers include the distorted frame edges: start from all + // observations (the Huber loss keeps real outliers in check), then reject again. + r.outliers.clear(); + r.adjust(&d, &frames, &cfg, c.focal, 30); + r.outliers.clear(); + r.reject(&d); + r.adjust(&d, &frames, &cfg, c.focal, 30); + dist = r.cams[&frames[0]].dist; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + if frames.len() < all.len() { + // Refine the other frames against the adjusted points and distortion. + let base = r.clone(); + let rcfg = Cfg { distortion: false, ..cfg }; + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &rcfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, cam) in refined { + if let Some(cam) = cam { + r.cams.insert(k, cam); + } + } + r.reject(&d); + } + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return err("cancelled"); + } + if debug() { + eprintln!("lens distortion pass {pass}: k1 {:.5} k2 {:.5}", dist.k1, dist.k2); + } + let mut sol = finish(&d, tracks, s, r, c.method, c.kind); + sol.distortion = Some(dist); + out = Some(sol); + } + out.ok_or_else(|| SolveError("the camera could not be solved".into())) +} + +fn solve_core(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { let d = Data::new(tracks, &s.deleted); if d.n < 2 { return err("the clip must be at least two frames long"); @@ -653,7 +753,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); let variable = s.shot == ShotType::VariableZoom; - let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false, distortion: false }; let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; // The focal-length search runs on the first keyframes only. let mini: Vec = kfs.iter().copied().take(10).collect(); @@ -738,7 +838,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } let (att, _) = results.swap_remove(pick); let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; - let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable, distortion: false }; let mut r = att.recon; let focal = match (fixed_f, variable) { (Some(_), _) => FocalMode::Fixed, @@ -845,7 +945,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (m, _) => m, }; lap("final adjustment"); - Ok(finish(&d, tracks, s, r, method, kind)) + Ok(Core { d, r, method, kind, cfg, focal, kfs: kf_frames }) } /// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). @@ -949,5 +1049,6 @@ fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: frames, points, ground: None, + distortion: None, } } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 4b5a8346..ff92cfe4 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -105,7 +105,7 @@ fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { } fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { - SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } + SolveSettings { shot, hfov: 0.0, method, deleted: vec![], lens_distortion: false } } #[test] @@ -149,7 +149,8 @@ fn specified_angle_of_view_is_kept() { let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); let tr = tracks_of(&pts, &t, 6, true); let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![], lens_distortion: false }, None).unwrap(); assert!((s.focal() - f).abs() < 1e-6); assert!(s.average_error < 0.5); let (pe, _) = path_error(&s, &t); @@ -226,7 +227,9 @@ fn mostly_flat_scene_and_ground_plane() { } // Ground plane from the solved (coplanar) points: every point lies on it. let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let mut s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![], lens_distortion: false }, None) + .expect("solve"); let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); let g = s.ground_from(&ids, 0).expect("ground"); let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); @@ -251,7 +254,9 @@ fn deleted_tracks_are_ignored_and_cancel_stops() { let tr = tracks_of(&pts, &t, 14, true); let deleted: Vec = (0..40).collect(); let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone(), lens_distortion: false }, None) + .unwrap(); assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); let stop = std::sync::atomic::AtomicBool::new(true); let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); @@ -436,3 +441,55 @@ fn solve_saved_tracks() { t0.elapsed() ); } + +/// Pass every track position through a lens distortion (around the frame centre). +fn distort_tracks(t: &CameraTracks, d: &Distortion) -> CameraTracks { + let mut t = t.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = d.distort([p[0] as f64 - W / 2.0, p[1] as f64 - H / 2.0]); + *p = [(q[0] + W / 2.0) as f32, (q[1] + H / 2.0) as f32]; + } + } + t +} + +#[test] +fn lens_distortion_is_recovered_in_the_bundle_adjustment() { + let f = 1000.0; + let t = moving_camera(36, |_| f); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let truth = Distortion { k1: -0.12, k2: 0.03, radius: 0.5 * W.hypot(H) }; + let tr = distort_tracks(&tracks_of(&pts, &t, 8, true), &truth); + let plain = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("pinhole solve"); + let lens = solve(&tr, &SolveSettings { lens_distortion: true, ..settings(ShotType::FixedAngle, SolveMethod::Typical) }, None).expect("lens solve"); + let d = lens.distortion.expect("distortion solved"); + eprintln!( + "pinhole error {:.3} px (focal {:.1}); with distortion {:.3} px (focal {:.1}), k1 {:.4} k2 {:.4}", + plain.average_error, + plain.focal(), + lens.average_error, + lens.focal(), + d.k1, + d.k2 + ); + assert!(plain.distortion.is_none()); + assert!((d.k1 - truth.k1).abs() < 0.005, "k1 {}", d.k1); + assert!((d.k2 - truth.k2).abs() < 0.01, "k2 {}", d.k2); + // The distortion model explains the tracks: about the noise level, far below the pinhole fit. + assert!(lens.average_error < 0.3, "{}", lens.average_error); + assert!(lens.average_error < 0.5 * plain.average_error, "{} vs {}", lens.average_error, plain.average_error); + assert!((lens.focal() - f).abs() < 0.02 * f, "focal {}", lens.focal()); + let (path, rot) = path_error(&lens, &t); + assert!(path < 0.01 && rot < 0.3, "path {path} rot {rot}"); + // Projection through the lens lands on the (distorted) tracks; undistorting inverts it. + let p = [100.0, 80.0]; + let q = lens.to_image(p); + let back = lens.to_ideal(q); + assert!((back[0] - p[0]).abs() < 1e-6 && (back[1] - p[1]).abs() < 1e-6); + let sp = &lens.points[0]; + let k = sp.first as usize; + let obs = tr.track(sp.id).and_then(|t| t.at(k as u32)).expect("observed"); + let proj = lens.project(k, sp.pos, false).expect("in front"); + assert!((proj[0] - obs[0]).hypot(proj[1] - obs[1]) < 2.0, "{proj:?} vs {obs:?}"); +} diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs index 78344a14..62073d0a 100644 --- a/crates/track/src/face.rs +++ b/crates/track/src/face.rs @@ -624,6 +624,11 @@ impl FaceTracker { }; let mut t = FaceTracker { skin, prev: guess, expect: canon }; let fit = t.fit(frame)?; + // A face shows at least two of: left eye, right eye, mouth. + let found = [EYE[0][4], EYE[1][4], MOUTH[0]].iter().filter(|i| fit.detected[**i]).count(); + if found < 2 { + return None; + } t.accept(&fit); Some((t, fit)) } diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs index 41fb7204..c9c40f05 100644 --- a/crates/ui-egui/src/panels/camera_tracker_ui.rs +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -38,6 +38,8 @@ struct Active { show_3d: bool, point_size: f64, target_size: f64, + /// Undistort Footage is on: points project without the lens distortion. + undistort: bool, /// Layer pixels → comp pixels. l2c: Mat3, frame: usize, @@ -66,6 +68,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye let t = tracks.as_ref()?; (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) }); + let undistort = solve.as_ref().is_some_and(|sv| ct::undistorts(¶ms, sv)); return Some(Active { layer: *lid, uid: *uid, @@ -74,6 +77,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + undistort, l2c: l2c(ectx, l), frame: frame.unwrap_or(0), }); @@ -98,18 +102,20 @@ fn l2c_scale(m: &Mat3) -> f32 { /// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { - let Some(cam) = sv.frames.get(a.frame) else { return }; + if sv.frames.get(a.frame).is_none() { + return; + } let r = t.size * 0.5 * a.target_size / 100.0; for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { let ring = ct::target_circle(t.center, t.normal, r * frac, 48); - let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + let pts: Vec = ring.iter().filter_map(|x| sv.project(a.frame, *x, a.undistort)).map(|q| to_screen(a, map, q)).collect(); if pts.len() < 3 { return; } let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); } - if let Some(c) = cam.project(sv.size, t.center) { + if let Some(c) = sv.project(a.frame, t.center, a.undistort) { let c = to_screen(a, map, c); p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); } @@ -168,7 +174,7 @@ pub fn viewer_hook( let mut hits: Vec = vec![]; match (&a.solve, a.show_3d) { (Some(sv), true) => { - let pts = ct::projected(sv, a.frame); + let pts = ct::projected_with(sv, a.frame, a.undistort); let med = ct::median_depth(&pts); for (id, q, z) in pts { let sp = to_screen(&a, map, q); diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index 364e114e..02c59636 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -404,6 +404,27 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p bx += 34.0; } y += 34.0; + // Face Tracking (Detailed Features): Extract & Copy Face Measurements. + let has_points = app + .session + .active_comp() + .and_then(|c| c.layer(layer)) + .and_then(|l| l.effects()) + .is_some_and(|fx| fx.groups().any(|g| g.match_id == effectcraft_engine::effects::face_track::POINTS_ID)); + if method == MaskMethod::FaceDetailed || has_points { + let r = Rect::from_min_size(pos2(x0, y), vec2(w.min(260.0), 24.0)); + if button(app, ui, r, "Extract & Copy Face Measurements", has_points && !running, "tracker.mask.extractFace") { + match crate::menus::invoke(app, ctx, "track.extractFaceMeasurements", json!({"layer": layer.0})) { + Ok(v) => { + if let Some(text) = v.get("clipboard").and_then(|t| t.as_str()) { + ctx.copy_text(text.to_string()); + } + } + Err(e) => app.ui.status = e, + } + } + y += 32.0; + } if let Some(pr) = app.session.mask_track_progress() { let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); p.rect_filled(bar, 3.0, t.field_bg); @@ -419,7 +440,12 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p ); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } else { - p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint); + let hint = if method.is_face() { + "Finds the face inside the mask and keys its outline (and, with Detailed Features, Face Track Points)." + } else { + "Tracks the pixels inside the mask and keys its Mask Path." + }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, hint, Tokens::ui(11.0), t.text_faint); } } diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..c25aa399 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -105,11 +105,18 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ `track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and tangents move with the fitted motion). `method` is `position`, `positionScale`, -`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew` -or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current +`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`, +`perspective`, `faceOutline` (Face Tracking (Outline Only): the mask, drawn around a face, becomes +the face outline on every frame) or `faceDetailed` (Face Tracking (Detailed Features): also keys a +Face Track Points effect with one point per landmark — eyebrows, eyes, pupils, nose tip, +nostrils, mouth corners and lips, chin, jaw); `direction` is `forward|backward|frameForward|frameBackward` from the current time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app (progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is -one undo step; `"wait": true` blocks. +one undo step; `"wait": true` blocks. `track.extractFaceMeasurements {layer?}` (Extract & Copy Face +Measurements) keys a Face Measurements effect from the Face Track Points (head position, scale and +orientation X/Y/Z, eye openness, eyebrow distance from eye, mouth openness, width and offset), puts +those keys on the keyframe clipboard and returns them per frame plus a tab-separated `clipboard` +text. `mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both @@ -138,7 +145,11 @@ the app re-analyses automatically (headless: run `warp.analyze` again). Settings properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion, 1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`, `effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges), -`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. +`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. Method 3 +(Subspace Warp, the default) bends a mesh per frame to the smoothed feature trajectories, so +scenes with depth stabilise where one perspective transform cannot; +`effects/#1/advanced/rollingShutterRipple` (0 Automatic Reduction, 1 Enhanced Reduction: a finer, +softer mesh) tunes it. ```sh effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \ @@ -150,13 +161,17 @@ effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ### 3D Camera Tracker Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: -fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?, +undistort?, wait?}`) apply Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: "Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. `camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, `averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` -(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +(position, orientation, zoom), `groundPlane` and the solved `lensDistortion` (`k1`, `k2`, normalised by +half the frame diagonal) when Solve Lens Distortion (`effects/#1/advanced/lensDistortion`, also on +with Detailed Analysis) is on; `effects/#1/advanced/undistort` renders the footage undistorted so +the solved camera's 3D layers line up. `camera.points {time?}` lists the solved points visible now with their `id`, `comp` position, `depth`, `world` position and `error`. Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..f54c2b9c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -228,6 +228,13 @@ tracks them with pyramidal Lucas–Kanade and fits the chosen model (position RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes. +The two **Face Tracking** methods (`effectcraft_track::face`) fit a face inside the mask instead: +a skin colour model from the first frame, the face outline along rays from the skin component's +centre (an area-moment ellipse gives centre, size and roll), facial features as the non-skin +components inside it, and an active-shape point distribution model trained on synthetic face +shapes (our own generator, no external weights) that fills and checks the landmarks. The outline +keys the Mask Path; Detailed Features keys a Face Track Points effect, and +`track.extractFaceMeasurements` derives a keyed Face Measurements effect (and copies its keys). **Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every @@ -236,7 +243,10 @@ translation / similarity / homography fits) as JSON in the effect's hidden Analy with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears analyses whose key no longer matches and queues them; the desktop app re-analyses them in the background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing, -auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the +auto-scale; cached per analysis and settings) and warps each frame. Subspace Warp also stores long +feature trajectories in the analysis, smooths them in a low-rank subspace (Liu et al. 2011) and +warps each frame with a content-preserving mesh (Liu et al. 2009) fitted to the smoothed +positions, rendered as a per-pixel mesh lookup; Synthesize Edges fills the borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. @@ -267,7 +277,10 @@ the essential matrix (normalised 8-point in RANSAC) or a planar homography decom resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod -pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +pans, chosen by Auto Detect when it explains the tracks as well; with Solve Lens Distortion (or +Detailed Analysis) radial distortion `k1, k2` is adjusted jointly in a final bundle adjustment on +the raw tracks and the solve repeated on tracks undistorted with that estimate; Undistort Footage +renders the input through the inverse model. Tracks and solve are stored as JSON in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is diff --git a/docs/effects.md b/docs/effects.md index 84735286..fd29541b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 306 effects with 2488 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 59 effects also run on the GPU compositor with identical results. -- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 262 implemented in full, 42 partial (what is missing is listed). +- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 264 implemented in full, 42 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -302,7 +302,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| 3D Camera Tracker | `ec.perspective.cameratracker` | 10 | | 32 | Implemented | +| 3D Camera Tracker | `ec.perspective.cameratracker` | 12 | | 32 | Implemented | | 3D Glasses | `ec.perspective.3dglasses` | 8 | | 32 | Implemented | | Bevel Alpha | `ec.perspective.bevelalpha` | 4 | | 32 | Implemented | | Bevel Edges | `ec.perspective.beveledges` | 4 | | 32 | Implemented | @@ -416,6 +416,8 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Overbrights | `ec.utility.ccoverbrights` | 2 | | 32 | Implemented | | Cineon Converter | `ec.utility.cineon` | 7 | | 32 | Implemented | | Color Profile Converter | `ec.utility.colorprofileconverter` | 7 | | 32 | Implemented | +| Face Measurements | `ec.utility.facemeasurements` | 14 | | 32 | Implemented | +| Face Track Points | `ec.utility.facetrackpoints` | 26 | | 32 | Implemented | | Grow Bounds | `ec.utility.growbounds` | 1 | | 32 | Implemented | | HDR Compander | `ec.utility.hdrcompander` | 3 | | 32 | Implemented | | HDR Highlight Compression | `ec.utility.hdrcompression` | 1 | | 32 | Implemented | diff --git a/docs/parity.md b/docs/parity.md index 4437bece..2d03442b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -17,13 +17,13 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, -Tracking 88%, Effects 85%, Web 65%. +Tracking 96%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable +approximated effects (Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for @@ -124,7 +124,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | +| Tracking | ≈ 96% | 0.3 | face tracking is a classical (skin model + feature components + shape model) fitter, not a learned detector: profile views and occluded faces are weak; Rolling Shutter Ripple is approximated by Subspace Warp's mesh density (face tracking (Outline Only / Detailed Features with Face Track Points and Extract & Copy Face Measurements), Subspace Warp's content-preserving mesh warp on subspace-smoothed trajectories, and radial lens distortion (k1, k2) in the camera bundle adjustment with Undistort Footage landed in M13.3; Rolling Shutter Repair in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing From ca7f7675535522d295ef0572d3a5019dd731dc62 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:50:59 -0400 Subject: [PATCH 29/36] M13.1: clippy (history test) Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/tests_history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/tests_history.rs b/crates/engine/src/tests_history.rs index 326c37c0..967c29e3 100644 --- a/crates/engine/src/tests_history.rs +++ b/crates/engine/src/tests_history.rs @@ -46,7 +46,7 @@ fn undo_then_new_step_keeps_a_branch() { let branch: Vec<&Value> = st.iter().filter(|n| n["depth"] == 1).collect(); assert_eq!(branch.len(), 2, "B and C sit on a branch: {st:?}"); // Jump to C on the old branch. - let c = st.iter().filter(|n| n["depth"] == 1).last().unwrap()["index"].as_u64().unwrap(); + let c = st.iter().rfind(|n| n["depth"] == 1).unwrap()["index"].as_u64().unwrap(); let r = s.execute("edit.history.goto", json!({"index": c})).unwrap(); assert_eq!(r["label"], "New Solid", "{r}"); assert_eq!(names(&s), ["C", "B", "A"]); From 81ac7151bc38a6f85022a7e3f7d881fcfbcb446c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:53:56 -0400 Subject: [PATCH 30/36] M13.3: CameraSolve distortion field in UI test after merge Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/tests/ui_camera_tracker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs index 45374fba..963ce3b6 100644 --- a/crates/ui-egui/tests/ui_camera_tracker.rs +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -41,6 +41,7 @@ fn app() -> (EffectcraftApp, u64, u64) { }) .collect(), ground: None, + distortion: None, }; s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); s.camera_pending.clear(); From 1242a775fed8c65893731cf6ee01c38d6f26da4c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:55:31 -0400 Subject: [PATCH 31/36] M10.2: Render Queue completeness (render settings, output module, templates, logs, overflow, notify) Render Settings: field render with 3:2 pulldown phases, effects/solo/guide layer/color depth/frame blending/motion blur overrides, custom time sampling (duration), storage overflow through a quota hook. Output Module: crop (edges or region of interest), resize (presets, lock aspect, quality), straight/premultiplied/alpha-only channels, post-render actions, include project link, audio channels and PCM formats, WebM bitrate and key frame interval. Render Settings / Output Module templates with built-ins and defaults per slot (Edit > Templates dialogs), render logs next to the output, Notify on completion. All through renderQueue.* commands and persisted in the project. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + README.md | 7 +- crates/engine/src/commands/comp_more.rs | 40 +- crates/engine/src/commands/proxy.rs | 19 +- crates/engine/src/commands/render_queue.rs | 520 ++++++++++-- crates/engine/src/lib.rs | 3 + crates/engine/src/render_queue.rs | 51 +- crates/engine/src/tests_stubs.rs | 117 +++ crates/export/Cargo.toml | 1 + crates/export/src/encode.rs | 80 +- crates/export/src/lib.rs | 170 ++-- crates/export/src/pipeline.rs | 327 ++++++++ crates/export/src/webm.rs | 79 +- crates/export/tests/render_settings.rs | 310 ++++++++ crates/export/tests/roundtrip.rs | 28 +- crates/host/src/lib.rs | 18 +- crates/project/src/lib.rs | 17 +- crates/project/src/render_queue.rs | 743 +++++++++++++++++- crates/project/src/render_templates.rs | 323 ++++++++ crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/menus.rs | 14 +- crates/ui-egui/src/panels/dialogs.rs | 1 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/render_queue.rs | 259 +++++- crates/ui-egui/src/panels/rq_templates.rs | 166 ++++ crates/ui-egui/tests/render_queue_snapshot.rs | 23 + docs/parity.md | 4 +- docs/web.md | 2 - 28 files changed, 3096 insertions(+), 230 deletions(-) create mode 100644 crates/export/src/pipeline.rs create mode 100644 crates/export/tests/render_settings.rs create mode 100644 crates/project/src/render_templates.rs create mode 100644 crates/ui-egui/src/panels/rq_templates.rs diff --git a/Cargo.lock b/Cargo.lock index c7934100..1dcfbb95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,7 @@ name = "effectcraft-export" version = "0.1.0" dependencies = [ "effectcraft-color", + "effectcraft-keyframe", "effectcraft-media", "effectcraft-opusenc", "effectcraft-project", diff --git a/README.md b/README.md index 3bff64fb..3a9cabbc 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,11 @@ tools, and Front, Top, Left and Custom views. ## Export A Render Queue like After Effects', with Render Settings and Output Modules: **H.264** MP4 and -**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, PNG, JPEG, TIFF and 32-bit EXR -sequences, and animated GIF. The same queue runs from the command line. Every encoder is +**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, **WebM** (VP9 with inter frames and +alpha, Opus), PNG, JPEG, TIFF and 32-bit EXR sequences, animated GIF and WAV/AIFF. Field +rendering with 3:2 pulldown, effect/solo/guide/depth overrides, crop, region of interest and +resize, Render Settings and Output Module templates with defaults, post-render actions, storage +overflow and render logs. The same queue runs from the command line. Every encoder is FilmCraft's pure-Rust code; there is no FFmpeg inside. **Lottie** goes both ways: File ▸ Export ▸ Lottie JSON… writes a composition (precomps, shape, diff --git a/crates/engine/src/commands/comp_more.rs b/crates/engine/src/commands/comp_more.rs index e3057f67..0f07fd67 100644 --- a/crates/engine/src/commands/comp_more.rs +++ b/crates/engine/src/commands/comp_more.rs @@ -82,7 +82,37 @@ pub(crate) fn encode_png(rgba: &[u8], w: u32, h: u32) -> std::result::Result Result { + use effectcraft_project::render_queue::{RenderQueueItem, TimeSpan}; + use effectcraft_project::render_templates::TemplateSlot; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(s.time()); + let mut it = RenderQueueItem::new(0, cid); + it.settings = s.project.render_templates.default_render_settings(TemplateSlot::Still); + it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; + it.output = s.project.render_templates.default_output_module(TemplateSlot::Still); + if let Some(path) = str_p(p, "path") { + it.output.output = path.to_string(); + } else if !it.output.format.is_sequence() { + it.output.output = "[compName]_frame.[fileExtension]".into(); + } + let id = s.edit("Save Frame As", None, |proj, _| { + it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; + let id = it.id; + proj.render_queue.push(it); + Ok(id) + })?; + let path = s.project.render_queue.last().and_then(|i| s.resolve_output(i)); + Ok(json!({"item": id, "queued": true, "output": path, "time": t.seconds()})) +} + fn save_frame(s: &mut Session, p: &Value) -> Result { + if p.get("queue").and_then(Value::as_bool) == Some(true) { + return queue_frame(s, p); + } let path = str_p(p, "path").ok_or_else(|| bad("comp.saveFrameAs", "missing `path` (.png)"))?.to_string(); let cid = comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; @@ -124,7 +154,15 @@ pub fn specs() -> Vec { vec![ cmd!("comp.cropToRegionOfInterest", "Crop Comp to Region of Interest", ["Composition"], None, "{comp?}", has_roi, crop_to_roi), cmd!("comp.cropToLayerBounds", "Crop Comp to Selected Layer(s) Bounds", ["Composition"], None, "{layers?}", has_layers, crop_to_layers), - cmd!("comp.saveFrameAs", "File...", ["Composition", "Save Frame As"], Some("Cmd+Alt+S"), "{path (.png), time?, scale?}", has_comp, save_frame), + cmd!( + "comp.saveFrameAs", + "File...", + ["Composition", "Save Frame As"], + Some("Cmd+Alt+S"), + "{path (.png; with queue: an output path or template), time?, scale?, queue?: bool (add a Render Queue item from the Frame Default templates instead of writing now)}", + has_comp, + save_frame + ), cmd!("comp.responsiveTime", "Responsive Design — Time", [], None, "{op: intro|outro|workArea}", has_comp, responsive_time), ] } diff --git a/crates/engine/src/commands/proxy.rs b/crates/engine/src/commands/proxy.rs index 607dd5dd..3e1126a9 100644 --- a/crates/engine/src/commands/proxy.rs +++ b/crates/engine/src/commands/proxy.rs @@ -4,6 +4,7 @@ //! item's switch). use effectcraft_project::render_queue::{Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, TimeSpan}; +use effectcraft_project::render_templates::TemplateSlot; use effectcraft_project::{Footage, ItemId, ItemKind, Proxy}; use effectcraft_time::Tick; use serde_json::{Value, json}; @@ -159,18 +160,28 @@ fn create_proxy(s: &mut Session, p: &Value) -> Result { k => return Err(bad(C, format!("kind: still|movie, not `{k}`"))), }; let mut it = RenderQueueItem::new(0, cid); + // The Movie / Still Proxy Default templates, at draft quality and the proxy resolution. + let templates = &s.project.render_templates; + let slot = if still { TemplateSlot::StillProxy } else { TemplateSlot::MovieProxy }; + it.settings = templates.default_render_settings(slot); it.settings.quality = RenderQuality::Draft; it.settings.resolution = p.get("resolution").and_then(Value::as_f64).unwrap_or(0.5).clamp(0.05, 1.0); if still { // One frame at the current time (the comp's poster frame when not the active comp). let t = if s.active_comp_id() == Some(cid) { s.time() } else { comp.poster_time }; it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; - it.output = OutputModule::for_format(OutputFormat::PngSequence); - it.output.output = "[compName]_proxy_[#####].png".into(); + it.output = templates.default_output_module(slot); + if !it.output.format.is_sequence() { + it.output = OutputModule::for_format(OutputFormat::PngSequence); + } + it.output.output = "[compName]_proxy_[#####].[fileExtension]".into(); } else { it.settings.time_span = TimeSpan::LengthOfComp; - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; + it.output = templates.default_output_module(slot); + if !it.output.format.is_movie() { + it.output = OutputModule::for_format(OutputFormat::ProRes); + it.output.prores_profile = ProResProfile::P4444; + } it.output.output = "[compName]_proxy.[fileExtension]".into(); } it.output.channels = Channels::Rgba; diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index 91467ea5..fcf088ac 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -1,16 +1,20 @@ //! Render Queue: Composition ▸ Add to Render Queue, the queue's Render Settings / Output Module / -//! Output To, Render and Stop. +//! Output To / Log, Render and Stop, Notify, storage overflow folders, and the Render Settings / +//! Output Module templates (Edit ▸ Templates). //! //! Items are addressed by `item` (the stable id from `renderQueue.add` / `renderQueue.list`) or //! `index` (1-based, the # column). use effectcraft_project::render_queue::{ - AudioOutput, Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, RenderSettings, RenderStatus, TimeSpan, + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, DEFAULT_SEQUENCE_TEMPLATE, DEFAULT_TEMPLATE, EffectsMode, FieldRender, + OutputFormat, OutputModule, PostRenderAction, ProResProfile, Pulldown, RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderSettings, + RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, post_render_parse, }; +use effectcraft_project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; use effectcraft_time::{FrameRate, Tick}; use serde_json::{Value, json}; -use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, str_p}; +use super::{CommandSpec, always, b_p, bad, comp_id, f_p, has_comp, str_p}; use crate::{EngineError, Result, Session, cmd, query}; fn can_add(s: &Session) -> std::result::Result<(), String> { @@ -54,16 +58,32 @@ fn time_p(p: &Value, k: &str) -> Option { f_p(p, k).map(Tick::from_seconds_f64) } -/// Apply Render Settings parameters. -fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result { +/// Parse an enum parameter. +fn enum_p(p: &Value, k: &str, cmd: &str, parse: fn(&str) -> Option, help: &str) -> Result> { + match p.get(k) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(v)) => parse(v).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Bool(b)) => parse(if *b { "on" } else { "off" }).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Number(n)) => parse(&n.to_string()).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(_) => Err(bad(cmd, format!("{k}: {help}"))), + } +} + +/// Apply Render Settings parameters (`template` first, then the individual settings). Any +/// individual change makes the settings "Custom". +fn apply_settings(rs: &mut RenderSettings, templates: &RenderTemplates, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + *rs = templates.render_settings(t).ok_or_else(|| bad(cmd, format!("no Render Settings template “{t}”")))?; + any = true; + } + let before = rs.clone(); if let Some(q) = str_p(p, "quality") { rs.quality = match q.to_ascii_lowercase().as_str() { "best" => RenderQuality::Best, "draft" => RenderQuality::Draft, _ => return Err(bad(cmd, "quality: best|draft")), }; - any = true; } match p.get("resolution") { Some(Value::String(r)) => { @@ -74,15 +94,21 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "quarter" => 0.25, _ => return Err(bad(cmd, "resolution: full|half|third|quarter|")), }; - any = true; } Some(Value::Number(n)) => { rs.resolution = n.as_f64().unwrap_or(1.0).clamp(0.01, 4.0); - any = true; } _ => {} } - let (start, end) = (time_p(p, "start"), time_p(p, "end")); + let (start, mut end) = (time_p(p, "start"), time_p(p, "end")); + if let Some(d) = time_p(p, "duration") { + // Time Span ▸ Custom ▸ Duration: from the given (or current custom) start. + let a = start.unwrap_or(match rs.time_span { + TimeSpan::Custom { start, .. } => start, + _ => Tick::ZERO, + }); + end = Some(a + d); + } if let Some(ts) = str_p(p, "timeSpan") { rs.time_span = match ts.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { "workarea" | "workareaonly" => TimeSpan::WorkArea, @@ -90,14 +116,12 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "custom" => TimeSpan::Custom { start: start.unwrap_or(Tick::ZERO), end: end.unwrap_or(Tick::from_seconds_f64(1.0)) }, _ => return Err(bad(cmd, "timeSpan: workArea|comp|custom")), }; - any = true; } else if start.is_some() || end.is_some() { let (a, b) = match rs.time_span { TimeSpan::Custom { start, end } => (start, end), _ => (Tick::ZERO, Tick(i64::MAX / 4)), }; rs.time_span = TimeSpan::Custom { start: start.unwrap_or(a), end: end.unwrap_or(b) }; - any = true; } if let TimeSpan::Custom { start, end } = rs.time_span && end <= start @@ -106,61 +130,193 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result } if let Some(u) = str_p(p, "proxyUse") { rs.proxy_use = effectcraft_project::render_queue::ProxyUse::parse(u).ok_or_else(|| bad(cmd, "proxyUse: current|all|comp|none"))?; - any = true; } match p.get("frameRate") { Some(Value::Null) => { rs.frame_rate = None; - any = true; } Some(v) => { let f = v.as_f64().filter(|f| *f > 0.0 && *f <= 1000.0).ok_or_else(|| bad(cmd, "frameRate: fps > 0 or null (comp rate)"))?; rs.frame_rate = Some(FrameRate::from_f64(f)); - any = true; } None => {} } - if let Some(b) = b_p(p, "motionBlur") { - rs.motion_blur = b; - any = true; + match p.get("motionBlur") { + Some(Value::Bool(b)) => { + rs.motion_blur = *b; + if *b && rs.motion_blur_mode == SwitchOverride::OffForAll { + rs.motion_blur_mode = SwitchOverride::Current; + } + } + Some(_) => { + let m = enum_p(p, "motionBlur", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")?.unwrap_or_default(); + rs.motion_blur = m != SwitchOverride::OffForAll; + rs.motion_blur_mode = m; + } + None => {} + } + if let Some(m) = enum_p(p, "frameBlending", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")? { + rs.frame_blending = m; + } + if let Some(f) = enum_p(p, "fieldRender", cmd, FieldRender::parse, "off|upper|lower")? { + rs.field_render = f; + } + if let Some(f) = enum_p(p, "pulldown", cmd, Pulldown::parse, "off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW")? { + rs.pulldown = f; + } + if let Some(e) = enum_p(p, "effects", cmd, EffectsMode::parse, "current|allOn|allOff")? { + rs.effects = e; + } + if let Some(e) = enum_p(p, "solo", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.solo = e; + } + if let Some(e) = enum_p(p, "guideLayers", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.guide_layers = e; + } + if let Some(e) = enum_p(p, "colorDepth", cmd, ColorDepth::parse, "current|8|16|32")? { + rs.color_depth = e; } if let Some(b) = b_p(p, "skipExisting") { rs.skip_existing = b; + } + if let Some(b) = b_p(p, "storageOverflow") { + rs.storage_overflow = b; + } + if *rs != before { + rs.name = "Custom".into(); any = true; } Ok(any) } -/// Apply Output Module parameters. -fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { +/// Apply a template to a module, keeping its output path (with the extension fixed). +fn apply_om_template(om: &mut OutputModule, t: &OutputModule) { + let keep = om.output.clone(); + let old = om.format; + *om = t.clone(); + om.output = keep; + let f = om.format; + om.format = old; + om.set_format(f); + om.channels = t.channels; + om.name = t.name.clone(); +} + +/// Apply Output Module parameters (`template` first). `roi`: the viewer's region of interest +/// (captured by `crop.useRoi`). +fn apply_output(om: &mut OutputModule, templates: &RenderTemplates, roi: Option<[f64; 4]>, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + let t = templates.output_module(t).ok_or_else(|| bad(cmd, format!("no Output Module template “{t}”")))?; + apply_om_template(om, &t); + any = true; + } + let before = om.clone(); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); - any = true; } if let Some(c) = str_p(p, "channels") { om.channels = match c.to_ascii_lowercase().replace([' ', '+'], "").as_str() { "rgb" => Channels::Rgb, "rgba" | "rgbalpha" => Channels::Rgba, - _ => return Err(bad(cmd, "channels: rgb|rgba")), + "alpha" | "alphaonly" => Channels::Alpha, + _ => return Err(bad(cmd, "channels: rgb|rgba|alpha")), }; if om.channels == Channels::Rgba && !om.format.supports_alpha() { return Err(bad(cmd, format!("{} has no alpha channel", om.format.label()))); } - any = true; + } + if let Some(c) = str_p(p, "color").or(str_p(p, "alphaMode")) { + om.alpha_mode = match c.to_ascii_lowercase().replace([' ', '(', ')'], "").as_str() { + "straight" | "straightunmatted" => AlphaMode::Straight, + "premultiplied" | "premultipliedmatted" | "premul" => AlphaMode::Premultiplied, + _ => return Err(bad(cmd, "color: straight|premultiplied")), + }; } if let Some(q) = f_p(p, "quality") { om.quality = q.clamp(1.0, 100.0) as u8; - any = true; } if let Some(b) = f_p(p, "bitrate") { - om.bitrate_kbps = b.clamp(100.0, 500_000.0) as u32; - any = true; + om.bitrate_kbps = b.clamp(50.0, 500_000.0) as u32; + } + if let Some(b) = b_p(p, "webmBitrate") { + om.webm_bitrate = b; + } + if let Some(k) = p.get("keyframeInterval").and_then(Value::as_u64) { + om.keyframe_interval = k.min(100_000) as u32; } if let Some(pp) = str_p(p, "proresProfile") { om.prores_profile = ProResProfile::from_name(pp).ok_or_else(|| bad(cmd, "proresProfile: proxy|lt|standard|hq|4444|4444xq"))?; - any = true; + } + match p.get("crop") { + Some(Value::Bool(b)) => om.crop.enabled = *b, + Some(Value::Object(c)) => { + om.crop.enabled = c.get("enabled").and_then(Value::as_bool).unwrap_or(true); + for (k, v) in [("top", &mut om.crop.top), ("left", &mut om.crop.left), ("bottom", &mut om.crop.bottom), ("right", &mut om.crop.right)] { + if let Some(n) = c.get(k).and_then(Value::as_i64) { + *v = n.clamp(-20_000, 20_000) as i32; + } + } + if let Some(u) = c.get("useRoi").and_then(Value::as_bool) { + om.crop.use_roi = u; + if u { + om.crop.roi = c.get("roi").and_then(|r| serde_json::from_value::<[f64; 4]>(r.clone()).ok()).or(roi); + if om.crop.roi.is_none() { + return Err(bad( + cmd, + "Use Region of Interest: the composition has no region of interest (View ▸ Region of Interest) — pass crop.roi [x, y, w, h]", + )); + } + } + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "crop: bool or {enabled?, useRoi?, roi?: [x,y,w,h], top?, left?, bottom?, right?}")), + } + // Flat keys (dialog forms): cropTop/cropLeft/cropBottom/cropRight, resizeWidth/resizeHeight. + for (k, v) in [("cropTop", &mut om.crop.top), ("cropLeft", &mut om.crop.left), ("cropBottom", &mut om.crop.bottom), ("cropRight", &mut om.crop.right)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as i64).clamp(-20_000, 20_000) as i32; + om.crop.enabled = true; + om.crop.use_roi = false; + } + } + for (k, v) in [("resizeWidth", &mut om.resize.width), ("resizeHeight", &mut om.resize.height)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as u32).clamp(1, 30_000); + om.resize.enabled = true; + } + } + match p.get("resize") { + Some(Value::Bool(b)) => om.resize.enabled = *b, + Some(Value::Object(r)) => { + om.resize.enabled = r.get("enabled").and_then(Value::as_bool).unwrap_or(true); + if let Some(name) = r.get("preset").and_then(Value::as_str) { + let (_, w, h) = + RESIZE_PRESETS.iter().find(|(n, _, _)| n.eq_ignore_ascii_case(name)).ok_or_else(|| bad(cmd, format!("no resize preset “{name}”")))?; + (om.resize.width, om.resize.height) = (*w, *h); + om.resize.lock_aspect = false; + } + if let Some(l) = r.get("lockAspect").and_then(Value::as_bool) { + om.resize.lock_aspect = l; + } + if let Some(w) = r.get("width").and_then(Value::as_u64) { + om.resize.width = (w as u32).clamp(1, 30_000); + } + if let Some(h) = r.get("height").and_then(Value::as_u64) { + om.resize.height = (h as u32).clamp(1, 30_000); + } + if let Some(q) = r.get("quality").and_then(Value::as_str) { + om.resize.quality = match q.to_ascii_lowercase().as_str() { + "low" | "bilinear" => ResizeQuality::Low, + "high" | "bicubic" => ResizeQuality::High, + _ => return Err(bad(cmd, "resize.quality: low|high")), + }; + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "resize: bool or {enabled?, preset?, width?, height?, lockAspect?, quality?: low|high}")), } if let Some(a) = p.get("audio") { om.audio = match a { @@ -174,15 +330,29 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { }, _ => return Err(bad(cmd, "audio: auto|on|off")), }; - any = true; } if let Some(r) = p.get("sampleRate").and_then(Value::as_u64) { om.audio_sample_rate = (r as u32).clamp(8_000, 192_000); - any = true; + } + match p.get("audioChannels") { + Some(Value::Number(n)) => om.audio_channels = if n.as_u64() == Some(1) { 1 } else { 2 }, + Some(Value::String(c)) => { + om.audio_channels = match c.to_ascii_lowercase().as_str() { + "mono" | "1" => 1, + "stereo" | "2" => 2, + _ => return Err(bad(cmd, "audioChannels: mono|stereo")), + } + } + _ => {} + } + if let Some(f) = enum_p(p, "audioFormat", cmd, AudioFormat::parse, "16|24|32 (float)")? { + om.audio_format = f; } if let Some(l) = b_p(p, "loop") { om.gif_loop = l; - any = true; + } + if let Some(l) = b_p(p, "includeProjectLink") { + om.include_project_link = l; } if let Some(o) = str_p(p, "output").or(str_p(p, "path")) { om.output = o.to_string(); @@ -194,6 +364,14 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { om.set_format(f); om.output = keep; } + } + if *om != before { + // Output To alone doesn't make the module custom. + let mut o = om.clone(); + o.output = before.output.clone(); + if o != before { + om.name = "Custom".into(); + } any = true; } Ok(any) @@ -227,7 +405,7 @@ fn requeue(it: &mut RenderQueueItem) { fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { let comp = s.project.comp(it.comp); let name = s.project.item(it.comp).map(|i| i.name.clone()); - let (w, h) = comp.map(|c| it.settings.output_size(c)).map(|(w, h)| it.output.format.coded_size(w, h)).unwrap_or((0, 0)); + let (w, h) = comp.map(|c| it.output.output_size(c, &it.settings)).unwrap_or((0, 0)); let frames = comp.map(|c| it.settings.frame_count(c)).unwrap_or(0); let mut v = serde_json::to_value(it).unwrap_or(Value::Null); if let Some(o) = v.as_object_mut() { @@ -249,6 +427,10 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { o.insert("outputModules".into(), json!(it.extra_outputs.len() + 1)); o.insert("extraOutputs".into(), json!(extra)); o.insert("postRenderAction".into(), json!(it.post_render.label())); + o.insert("logLabel".into(), json!(it.log.label())); + if let Some(p) = s.resolve_output(it) { + o.insert("logPath".into(), json!(effectcraft_project::render_queue::log_path(&p))); + } o.insert("width".into(), json!(w)); o.insert("height".into(), json!(h)); o.insert("frames".into(), json!(frames)); @@ -259,16 +441,25 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); + // New items start from the Movie Default templates (Edit ▸ Templates). + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::Movie); + it.output = templates.default_output_module(TemplateSlot::Movie); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; - it.output = OutputModule::for_format(f); + if f != it.output.format { + it.output = OutputModule::for_format(f); + } } - apply_settings(&mut it.settings, p, "renderQueue.add")?; + apply_settings(&mut it.settings, &templates, p, "renderQueue.add")?; let mut op = p.clone(); if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut it.output, &op, "renderQueue.add")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, &op, "renderQueue.add")?; + if let Some(l) = enum_p(p, "log", "renderQueue.add", RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")? { + it.log = l; + } let id = s.edit("Add to Render Queue", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; let id = it.id; @@ -305,7 +496,7 @@ fn set_render(s: &mut Session, p: &Value) -> Result { fn set_render_settings(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setRenderSettings")?; let mut rs = s.project.render_queue[i].settings.clone(); - if !apply_settings(&mut rs, p, "renderQueue.setRenderSettings")? { + if !apply_settings(&mut rs, &s.project.render_templates, p, "renderQueue.setRenderSettings")? { return Err(bad("renderQueue.setRenderSettings", "nothing to change")); } s.edit("Render Settings", None, |proj, _| { @@ -321,12 +512,20 @@ fn set_output_module(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setOutputModule")?; let m = module_p(s, i, p, "renderQueue.setOutputModule")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - if !apply_output(&mut om, p, "renderQueue.setOutputModule")? { + let changed = apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, p, "renderQueue.setOutputModule")?; + let post = match str_p(p, "postRenderAction") { + Some(a) => Some(post_render_parse(a).ok_or_else(|| bad("renderQueue.setOutputModule", "postRenderAction: none|import|importAndReplace|setProxy"))?), + None => None, + }; + if !changed && post.is_none() { return Err(bad("renderQueue.setOutputModule", "nothing to change")); } s.edit("Output Module Settings", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; + if let Some(a) = post { + it.post_render = a; + } requeue(it); Ok(()) })?; @@ -338,7 +537,7 @@ fn set_output(s: &mut Session, p: &Value) -> Result { let path = str_p(p, "path").or(str_p(p, "output")).ok_or_else(|| bad("renderQueue.setOutput", "pass `path` (file path or template)"))?; let m = module_p(s, i, p, "renderQueue.setOutput")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - apply_output(&mut om, &json!({"output": path}), "renderQueue.setOutput")?; + apply_output(&mut om, &s.project.render_templates, None, &json!({"output": path}), "renderQueue.setOutput")?; s.edit("Output To", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; @@ -452,7 +651,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut om, &op, "render.addOutputModule")?; + apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, &op, "render.addOutputModule")?; s.edit("Add Output Module", None, |proj, _| { let it = &mut proj.render_queue[i]; it.extra_outputs.push(om); @@ -469,11 +668,11 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { fn pre_render(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; - it.output.channels = Channels::Rgba; + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::PreRender); + it.output = templates.default_output_module(TemplateSlot::PreRender); it.output.output = "[compName]_prerender.[fileExtension]".into(); - apply_output(&mut it.output, p, "render.preRender")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, p, "render.preRender")?; it.post_render = PostRenderAction::ImportAndReplace; let id = s.edit("Pre-render", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; @@ -499,13 +698,202 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { let mut item = RenderQueueItem::new(0, cid); item.output = OutputModule::for_format(format); item.output.output = path.clone(); - let job = - crate::ExportJob { project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), accel: s.accel.as_deref(), item: &item, path: &path }; + let job = crate::ExportJob { + project: &s.project, + footage: s.footage.as_ref(), + expr: s.expr.as_deref(), + accel: s.accel.as_deref(), + item: &item, + path: &path, + storage: s.storage_quota.as_deref(), + label: "Save Current Preview".into(), + }; let r = exporter.export(&job, &mut |_, _| true).map_err(EngineError::Other)?; s.toast(format!("Saved preview to {}", r.path)); Ok(serde_json::to_value(&r).unwrap_or_default()) } +// ---------------------------------------------------------------- templates (Edit ▸ Templates) + +fn kind_p(p: &Value, cmd: &str) -> Result { + str_p(p, "kind").and_then(TemplateKind::parse).ok_or_else(|| bad(cmd, "kind: renderSettings|outputModule")) +} + +fn name_p<'a>(p: &'a Value, cmd: &str) -> Result<&'a str> { + str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or_else(|| bad(cmd, "pass `name`")) +} + +/// Templates as JSON (lists, built-in flags, defaults per slot). +fn templates_json(t: &RenderTemplates) -> Value { + let rs: Vec = t + .render_settings_list() + .iter() + .map(|r| json!({"name": r.name, "builtin": RenderTemplates::is_builtin(TemplateKind::RenderSettings, &r.name), "settings": r})) + .collect(); + let om: Vec = t + .output_module_list() + .iter() + .map(|m| json!({"name": m.name, "builtin": RenderTemplates::is_builtin(TemplateKind::OutputModule, &m.name), "summary": format!("{} · {}", m.format.label(), m.channels.label()), "module": m})) + .collect(); + let slots = |k: TemplateKind| -> Value { + TemplateSlot::ALL.iter().map(|sl| (format!("{sl:?}"), json!(t.default_name(k, *sl)))).collect::>().into() + }; + json!({ + "renderSettings": rs, + "outputModules": om, + "defaults": {"renderSettings": slots(TemplateKind::RenderSettings), "outputModules": slots(TemplateKind::OutputModule)}, + "slots": TemplateSlot::ALL.iter().map(|sl| json!({"id": format!("{sl:?}"), "label": sl.label()})).collect::>(), + "resizePresets": RESIZE_PRESETS.iter().map(|(n, w, h)| json!({"name": n, "width": w, "height": h})).collect::>(), + }) +} + +fn templates(s: &mut Session, _: &Value) -> Result { + Ok(templates_json(&s.project.render_templates)) +} + +/// Save a template: from an item's settings (`item`/`index`, `module`), or the slot default, +/// adjusted by `params` (the same keys as setRenderSettings / setOutputModule). +fn save_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.saveTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + if name.eq_ignore_ascii_case("custom") { + return Err(bad(C, "“Custom” is reserved")); + } + let item = if p.get("item").is_some() || p.get("index").is_some() { Some(item_index(s, p, C)?) } else { None }; + let params = p.get("params").cloned().unwrap_or(Value::Null); + let from = str_p(p, "from"); + let mut t = s.project.render_templates.clone(); + match kind { + TemplateKind::RenderSettings => { + let mut rs = match (item, from) { + (Some(i), _) => s.project.render_queue[i].settings.clone(), + (None, Some(f)) => t.render_settings(f).ok_or_else(|| bad(C, format!("no Render Settings template “{f}”")))?, + (None, None) => t.default_render_settings(TemplateSlot::Movie), + }; + if params.is_object() { + apply_settings(&mut rs, &t, ¶ms, C)?; + } + t.save_render_settings(&name, rs); + } + TemplateKind::OutputModule => { + let mut om = match (item, from) { + (Some(i), _) => module_ref(&s.project.render_queue[i], module_p(s, i, p, C)?).clone(), + (None, Some(f)) => t.output_module(f).ok_or_else(|| bad(C, format!("no Output Module template “{f}”")))?, + (None, None) => t.default_output_module(TemplateSlot::Movie), + }; + if params.is_object() { + apply_output(&mut om, &t, s.state.region_of_interest, ¶ms, C)?; + } + // Templates name files after the composition, not a particular path. + om.output = if om.format.is_sequence() { DEFAULT_SEQUENCE_TEMPLATE } else { DEFAULT_TEMPLATE }.into(); + t.save_output_module(&name, om); + } + } + s.edit("Save Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"saved": name, "templates": templates_json(&s.project.render_templates)})) +} + +fn delete_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.deleteTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.delete(kind, &name).map_err(|e| bad(C, e))?; + s.edit("Delete Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"deleted": name})) +} + +fn set_template_default(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setTemplateDefault"; + let kind = kind_p(p, C)?; + let slot = str_p(p, "slot").and_then(TemplateSlot::parse).ok_or_else(|| bad(C, "slot: movie|still|preRender|movieProxy|stillProxy"))?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.set_default(kind, slot, &name).map_err(|e| bad(C, e))?; + s.edit("Template Default", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"slot": format!("{slot:?}"), "name": name})) +} + +/// Apply templates to an item: `renderSettings` and/or `outputModule` (to `module`). +fn apply_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.applyTemplate"; + let i = item_index(s, p, C)?; + let m = module_p(s, i, p, C)?; + let t = s.project.render_templates.clone(); + let rs = match str_p(p, "renderSettings") { + Some(n) => Some(t.render_settings(n).ok_or_else(|| bad(C, format!("no Render Settings template “{n}”")))?), + None => None, + }; + let om = match str_p(p, "outputModule") { + Some(n) => { + let tpl = t.output_module(n).ok_or_else(|| bad(C, format!("no Output Module template “{n}”")))?; + let mut om = module_ref(&s.project.render_queue[i], m).clone(); + apply_om_template(&mut om, &tpl); + Some(om) + } + None => None, + }; + if rs.is_none() && om.is_none() { + return Err(bad(C, "pass `renderSettings` and/or `outputModule` (template names)")); + } + s.edit("Apply Template", None, |proj, _| { + let it = &mut proj.render_queue[i]; + if let Some(rs) = rs { + it.settings = rs; + } + if let Some(om) = om { + *module_mut(it, m) = om; + } + requeue(it); + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_log(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setLog"; + let i = item_index(s, p, C)?; + let log = enum_p(p, "log", C, RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")?.ok_or_else(|| bad(C, "pass `log`"))?; + s.edit("Render Queue: Log", None, |proj, _| { + proj.render_queue[i].log = log; + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_notify(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "notify").or(b_p(p, "value")).unwrap_or(!s.project.render_prefs.notify); + s.edit("Render Queue: Notify", None, |proj, _| { + proj.render_prefs.notify = on; + Ok(()) + })?; + Ok(json!({"notify": on})) +} + +fn set_overflow(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setOverflowFolders"; + let folders: Vec = match p.get("folders") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).map(str::trim).filter(|f| !f.is_empty()).map(String::from).collect(), + Some(Value::Null) | None => vec![], + _ => return Err(bad(C, "folders: [path, …]")), + }; + s.edit("Storage Overflow Folders", None, |proj, _| { + proj.render_prefs.overflow_folders = folders.clone(); + Ok(()) + })?; + Ok(json!({"folders": s.project.render_prefs.overflow_folders})) +} + pub fn specs() -> Vec { vec![ cmd!( @@ -513,7 +901,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, template?: Render Settings template, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, log?: errorsOnly|plusSettings|plusPerFrameInfo, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, proxyUse?, effects?, solo?, guideLayers?, colorDepth?, frameBlending?, fieldRender?, pulldown?, motionBlur?, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool, channels?: rgb|rgba|alpha, color?, bitrate?: kbps, webmBitrate?, keyframeInterval?, proresProfile?, crop?, resize?, includeProjectLink?, audio?: auto|on|off, sampleRate?, audioChannels?, audioFormat?, loop?: bool (values as in setRenderSettings / setOutputModule)}", can_add, add ), @@ -524,7 +912,7 @@ pub fn specs() -> Vec { "Render Settings...", [], None, - "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none}", + "{item?|index?, template?: name, quality?: best|draft, resolution?: full|half|third|quarter|scale, proxyUse?: current|all|comp|none, effects?: current|allOn|allOff, solo?: current|allOff, guideLayers?: current|allOff, colorDepth?: current|8|16|32, frameBlending?: current|onForChecked|offForAll, fieldRender?: off|upper|lower, pulldown?: off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW, motionBlur?: bool|current|onForChecked|offForAll, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool}", has_items, set_render_settings ), @@ -533,7 +921,7 @@ pub fn specs() -> Vec { "Output Module Settings...", [], None, - "{item?|index?, module?: n (1 = first), format?, channels?: rgb|rgba, quality?: 1-100, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool, output?}", + "{item?|index?, module?: n (1 = first), template?: name, format?, channels?: rgb|rgba|alpha, color?: straight|premultiplied, quality?: 1-100, bitrate?: kbps, webmBitrate?: bool, keyframeInterval?: frames, proresProfile?: proxy|lt|standard|hq|4444|4444xq, crop?: bool|{useRoi?, roi?, top?, left?, bottom?, right?}, cropTop?, cropLeft?, cropBottom?, cropRight?, resize?: bool|{preset?, width?, height?, lockAspect?, quality?: low|high}, resizeWidth?, resizeHeight?, postRenderAction?: none|import|importAndReplace|setProxy, includeProjectLink?: bool, audio?: auto|on|off, sampleRate?, audioChannels?: mono|stereo, audioFormat?: 16|24|32, loop?: bool, output?}", has_items, set_output_module ), @@ -562,6 +950,46 @@ pub fn specs() -> Vec { cmd!("renderQueue.render", "Render", [], None, "{wait?: bool (default true; the UI renders in the background)}", has_queued, render), cmd!("renderQueue.stop", "Stop Rendering", [], None, "{}", rendering, stop), query!("renderQueue.list", "Render Queue Items", "{}", list), + query!("renderQueue.templates", "Render Templates", "{}", templates), + cmd!( + "renderQueue.saveTemplate", + "Save Template...", + [], + None, + "{kind: renderSettings|outputModule, name, item?|index? (save from this item), from?: template to copy, module?, params?: {setRenderSettings / setOutputModule keys}}", + always, + save_template + ), + cmd!("renderQueue.deleteTemplate", "Delete Template", [], None, "{kind: renderSettings|outputModule, name}", always, delete_template), + cmd!( + "renderQueue.setTemplateDefault", + "Set Template Default", + [], + None, + "{kind: renderSettings|outputModule, slot: movie|still|preRender|movieProxy|stillProxy, name}", + always, + set_template_default + ), + cmd!( + "renderQueue.applyTemplate", + "Apply Template", + [], + None, + "{item?|index?, renderSettings?: template name, outputModule?: template name, module?: n}", + has_items, + apply_template + ), + cmd!("renderQueue.setLog", "Render Queue: Log", [], None, "{item?|index?, log: errorsOnly|plusSettings|plusPerFrameInfo}", has_items, set_log), + cmd!("renderQueue.setNotify", "Notify When Done", [], None, "{notify?: bool (toggles)}", always, set_notify), + cmd!( + "renderQueue.setOverflowFolders", + "Storage Overflow Folders", + [], + None, + "{folders: [path, …] (used in order when the output volume is full)}", + always, + set_overflow + ), query!("renderQueue.formats", "Output Formats", "{}", formats), ] } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9333329a..d7a89160 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -275,6 +275,8 @@ pub struct Session { pub importer: Option>, /// Render Queue encoder (the export layer); `None` = export unavailable. pub exporter: Option>, + /// Free-space hook for Render Settings ▸ Use Storage Overflow (`None`: never full). + pub storage_quota: Option>, /// The running (or finished, not yet polled) render. pub render_job: Option, /// The running (or finished, not yet polled) track analysis. @@ -357,6 +359,7 @@ impl Default for Session { expr_check: None, importer: None, exporter: None, + storage_quota: None, render_job: None, track_job: None, mask_job: None, diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index 9dd20e8d..3de9d8e5 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -27,6 +27,10 @@ pub struct ExportJob<'a> { pub accel: Option<&'a dyn effectcraft_render::Accelerator>, /// Resolved output path (templates expanded except the `#` frame-number run). pub path: &'a str, + /// Free-space hook (Use Storage Overflow). + pub storage: Option<&'a dyn effectcraft_project::render_queue::StorageQuota>, + /// "#n Comp Name", for the render log. + pub label: String, } /// What an export wrote. @@ -39,6 +43,12 @@ pub struct ExportResult { pub bytes: u64, pub seconds: f64, pub audio: bool, + /// The render log written next to the output, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub log: Option, + /// Files that went to storage overflow folders. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow: Vec, } /// Encodes render queue items (implemented by the export layer). @@ -127,6 +137,7 @@ struct Work { expr: Option>, accel: Option>, exporter: Arc, + storage: Option>, /// Per queue item: one (item with that output module, resolved path) per output module. items: Vec>, } @@ -143,7 +154,15 @@ pub(crate) fn run_items( shared: &JobShared, notify: &mut dyn FnMut(&JobShared), ) { - let work = Work { project: s.project.clone(), footage: s.footage.clone(), expr: s.expr.clone(), accel: s.accel.clone(), exporter, items }; + let work = Work { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + accel: s.accel.clone(), + exporter, + storage: s.storage_quota.clone(), + items, + }; run_work(work, shared, notify); } @@ -174,7 +193,18 @@ fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { // first module's file is the item's output. let mut r: Result = Err("no output module".into()); for (mi, (mitem, path)) in modules.iter().enumerate() { - let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), accel: w.accel.as_deref(), item: mitem, path }; + let name = w.project.item(mitem.comp).map(|i| i.name.clone()).unwrap_or_default(); + let index = w.project.render_queue.iter().position(|i| i.id == mitem.id).map_or(0, |i| i + 1); + let job = ExportJob { + project: &w.project, + footage: w.footage.as_ref(), + expr: w.expr.as_deref(), + accel: w.accel.as_deref(), + item: mitem, + path, + storage: w.storage.as_deref(), + label: format!("#{index} {name} (output module {})", mi + 1), + }; let rr = w.exporter.export(&job, &mut |done, total| { { let mut s = lock(&shared.state); @@ -235,8 +265,7 @@ impl Session { if tpl.is_empty() { return None; } - let (w, h) = item.settings.output_size(comp); - let (w, h) = item.output.format.coded_size(w, h); + let (w, h) = item.output.output_size(comp, &item.settings); let rate = item.settings.rate(comp); let n = item.settings.frame_count(comp) as i64; let f0 = item.settings.first_frame(comp); @@ -329,7 +358,15 @@ impl Session { self.poll_render(); return Ok(ids); } - let work = Work { project: self.project.clone(), footage: self.footage.clone(), expr: self.expr.clone(), accel: self.accel.clone(), exporter, items }; + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + accel: self.accel.clone(), + exporter, + storage: self.storage_quota.clone(), + items, + }; let shared = Arc::new(JobShared::default()); if wait { run_work(work, &shared, &mut |_| {}); @@ -426,6 +463,10 @@ impl Session { } else { format!("Rendered {} item(s) in {:.1} s", st.items_total, st.elapsed) }; + if self.project.render_prefs.notify { + // Notify: the frontend plays a sound and shows a system notification. + self.events.push(Event::Frontend { command: "renderQueue.notify".into(), params: serde_json::json!({"message": msg, "error": failed > 0}) }); + } self.events.push(Event::Toast { message: msg, error: failed > 0 }); } for (id, path) in post { diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 997552bc..18e65cb0 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -312,6 +312,123 @@ fn pre_render_imports_and_replaces() { assert!(matches!(s.project.comp(outer).unwrap().layers[0].source, LayerSource::Comp { .. })); } +#[test] +fn post_render_actions_import_and_set_proxy() { + let (mut s, _) = rq(); + let cid = s.active_comp_id().unwrap(); + let items_before = s.project.items.len(); + // Import: the rendered file becomes a footage item. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/post/a.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": a["item"], "postRenderAction": "import"})).unwrap(); + assert_eq!(s.project.render_queue[0].post_render, PostRenderAction::Import); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), items_before + 1); + assert!(s.project.items.values().any(|i| matches!(&i.kind, effectcraft_project::ItemKind::Footage(f) if f.path == "/tmp/post/a.mov"))); + // Set Proxy: the comp gets the render as its proxy. + let b = s.execute("renderQueue.add", json!({"output": "/tmp/post/b.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": b["item"], "postRenderAction": "Set Proxy"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + let px = s.project.item(cid).unwrap().proxy.clone().expect("proxy set"); + assert_eq!(px.footage.path, "/tmp/post/b.mov"); + // None: nothing happens. + let n = s.project.items.len(); + let c = s.execute("renderQueue.add", json!({"output": "/tmp/post/c.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "none"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), n); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "explode"})).is_err()); +} + +#[test] +fn templates_save_apply_default_and_persist() { + let (mut s, _) = rq(); + let t = s.execute("renderQueue.templates", json!({})).unwrap(); + let names: Vec<&str> = t["renderSettings"].as_array().unwrap().iter().map(|r| r["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert!(t["outputModules"].as_array().unwrap().iter().any(|m| m["name"] == "High Quality with Alpha")); + assert_eq!(t["defaults"]["renderSettings"]["Movie"], "Best Settings"); + // A new item starts from the Movie defaults. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/t/a.mp4"})).unwrap(); + assert_eq!(a["renderSettingsSummary"], "Best Settings"); + assert_eq!(a["outputModuleSummary"], "H.264 - Match Render Settings - 15 Mbps"); + // Edit it: Custom; save templates from it; make them the defaults. + let r = s + .execute( + "renderQueue.setRenderSettings", + json!({"item": a["item"], "fieldRender": "upper", "pulldown": "WSSWW", "colorDepth": 16, "effects": "allOff", "solo": "allOff", "guideLayers": "current", "frameBlending": "offForAll", "motionBlur": "onForChecked", "duration": 0.5, "storageOverflow": false}), + ) + .unwrap(); + assert_eq!(r["renderSettingsSummary"], "Custom"); + assert_eq!(r["settings"]["field_render"], "UpperFirst"); + s.execute( + "renderQueue.setOutputModule", + json!({"item": a["item"], "format": "png", "channels": "rgba", "color": "premultiplied", "crop": {"top": 2, "bottom": 2}, "resize": {"width": 320, "height": 200, "lockAspect": false, "quality": "low"}}), + ) + .unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "renderSettings", "name": "Interlaced 16", "item": a["item"]})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Small PNG", "item": a["item"], "params": {"quality": 50}})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "renderSettings", "slot": "movie", "name": "Interlaced 16"})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Small PNG"})).unwrap(); + assert!(s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Nope"})).is_err()); + let b = s.execute("renderQueue.add", json!({"output": "/tmp/t/b_[####].png"})).unwrap(); + assert_eq!(b["renderSettingsSummary"], "Interlaced 16"); + assert_eq!(b["outputModuleSummary"], "Small PNG"); + let it = s.project.render_queue.last().unwrap().clone(); + assert_eq!(it.output.resize.width, 320); + assert_eq!(it.output.output, "/tmp/t/b_[####].png"); + assert_eq!((b["width"].as_u64(), b["height"].as_u64()), (Some(320), Some(200))); + // Apply built-ins to an item; the output path keeps its name with the new extension. + let r = s.execute("renderQueue.applyTemplate", json!({"item": a["item"], "renderSettings": "DV Settings", "outputModule": "High Quality"})).unwrap(); + assert_eq!(r["renderSettingsSummary"], "DV Settings"); + assert_eq!(r["outputModuleSummary"], "High Quality"); + assert!(r["outputPath"].as_str().unwrap().ends_with("a.mov"), "{r}"); + // Persisted with the project. + let p = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + assert_eq!(p.render_templates, s.project.render_templates); + assert_eq!(p.render_queue, s.project.render_queue); + // Delete: saved ones go; built-ins stay. Undo restores. + s.execute("renderQueue.deleteTemplate", json!({"kind": "outputModule", "name": "Small PNG"})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_none()); + assert!(s.execute("renderQueue.deleteTemplate", json!({"kind": "renderSettings", "name": "Best Settings"})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_some()); +} + +#[test] +fn save_frame_as_queues_with_the_frame_defaults() { + let (mut s, _) = rq(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "still", "name": "TIFF Sequence with Alpha"})).unwrap(); + let r = s.execute("comp.saveFrameAs", json!({"queue": true, "time": 1.0})).unwrap(); + assert_eq!(r["queued"], true); + let it = s.project.render_queue.last().unwrap(); + assert_eq!(it.output.format, OutputFormat::TiffSequence); + assert_eq!(it.settings.name, "Current Settings"); + let comp = s.project.comp(it.comp).unwrap(); + assert_eq!(it.settings.frame_count(comp), 1); + assert_eq!(it.settings.frame_time(comp, 0), Tick::from_seconds_f64(1.0)); +} + +#[test] +fn log_notify_and_overflow_settings() { + let (mut s, _) = rq(); + let a = s.execute("renderQueue.add", json!({"output": "/tmp/n/a.mp4", "log": "plusPerFrameInfo"})).unwrap(); + assert_eq!(a["logLabel"], "Plus Per Frame Info"); + assert_eq!(a["logPath"], "/tmp/n/a_RenderLog.txt"); + s.execute("renderQueue.setLog", json!({"item": a["item"], "log": "plusSettings"})).unwrap(); + assert_eq!(s.project.render_queue[0].log, effectcraft_project::render_queue::RenderLog::PlusSettings); + s.execute("renderQueue.setOverflowFolders", json!({"folders": ["/tmp/o1", " ", "/tmp/o2"]})).unwrap(); + assert_eq!(s.project.render_prefs.overflow_folders, ["/tmp/o1", "/tmp/o2"]); + assert_eq!(s.execute("renderQueue.setNotify", json!({})).unwrap()["notify"], true); + s.drain_events(); + s.execute("renderQueue.render", json!({})).unwrap(); + let ev = s.drain_events(); + assert!(ev.iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify")), "{ev:?}"); + s.execute("renderQueue.setNotify", json!({"notify": false})).unwrap(); + s.execute("renderQueue.setRender", json!({"index": 1, "render": true})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert!(!s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify"))); +} + #[test] fn save_current_preview_writes_a_movie() { let (mut s, ex) = rq(); diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 15f9b17d..6900c204 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -32,3 +32,4 @@ effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } filmcraft-vp9 = { workspace = true } +effectcraft-keyframe = { workspace = true } diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index d6cf3150..b0566d9b 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -4,12 +4,12 @@ use std::io::Write; use effectcraft_project::Comp; -use effectcraft_project::render_queue::{Channels, OutputFormat, ProResProfile}; +use effectcraft_project::render_queue::{AudioFormat, Channels, OutputFormat, ProResProfile}; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_isobmff::{Brand, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; use rayon::prelude::*; -use crate::{ExportError, Job, Report, Result, State, batch_size, render_frame, rgba8, wants_audio}; +use crate::{Cx, ExportError, Report, Result, State, batch_size, wants_audio}; fn enc(e: impl std::fmt::Display) -> ExportError { ExportError::Encode(e.to_string()) @@ -208,29 +208,60 @@ enum Audio { Pcm, } -const AUDIO_CHANNELS: u32 = 2; +fn deinterleave(buf: &[f32], channels: usize) -> Vec> { + (0..channels).map(|c| buf.chunks_exact(channels).map(|p| p[c]).collect()).collect() +} -fn deinterleave(buf: &[f32]) -> [Vec; 2] { - let l = buf.chunks_exact(2).map(|p| p[0]).collect(); - let r = buf.chunks_exact(2).map(|p| p[1]).collect(); - [l, r] +/// The comp's audio over `n` samples from `start`, interleaved with the module's channel count +/// (mono = the average of left and right). +pub(crate) fn mix(cx: &Cx, start: effectcraft_time::Tick, n: usize, sr: u32) -> Vec { + let st = effectcraft_render::audio::mix_comp(&cx.project, cx.footage, cx.expr, cx.comp, start, n, sr); + if cx.output.audio_channels == 1 { st.chunks_exact(2).map(|p| (p[0] + p[1]) * 0.5).collect() } else { st } +} + +/// PCM bytes of samples in `fmt` (little or big endian). +pub(crate) fn pcm_bytes(buf: &[f32], fmt: AudioFormat, big_endian: bool) -> Vec { + let mut out = Vec::with_capacity(buf.len() * 4); + for &s in buf { + let s = s.clamp(-1.0, 1.0); + match fmt { + AudioFormat::S16 => { + let v = (s * 32767.0).round() as i16; + out.extend_from_slice(&if big_endian { v.to_be_bytes() } else { v.to_le_bytes() }); + } + AudioFormat::S24 => { + let v = (s * 8_388_607.0).round() as i32; + let b = v.to_le_bytes(); + if big_endian { + out.extend_from_slice(&[b[2], b[1], b[0]]); + } else { + out.extend_from_slice(&b[..3]); + } + } + AudioFormat::F32 => out.extend_from_slice(&if big_endian { s.to_be_bytes() } else { s.to_le_bytes() }), + } + } + out } // ---------------------------------------------------------------- the movie type Writer<'f, 's> = Mp4Writer<&'f mut crate::out::Out<'s>>; -pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let fmt = job.output.format; - let channels = if fmt == OutputFormat::ProRes { job.output.channels } else { Channels::Rgb }; + let channels = match job.output.channels { + Channels::Rgba if fmt != OutputFormat::ProRes => Channels::Rgb, + c => c, + }; let mut venc: Box = match fmt { OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?), - _ => Box::new(ProRes::new(w, h, job.output.prores_profile, channels)), + _ => Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels })), }; let brand = if fmt == OutputFormat::H264 { Brand::Mp4 } else { Brand::Mov }; let batch = batch_size(); - let render = |k: u64| -> Vec { rgba8(&render_frame(job, comp, k), comp, channels, w, h) }; + let render = |k: u64| -> Vec { job.pixels(&job.frame(comp, k), comp, channels, w, h) }; // Encode the first batch before creating tracks: encoders finalise their config on frame 1. let first_end = batch.min(st.total); @@ -242,7 +273,8 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R drop(first); st.advance(first_end)?; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; let opts = WriterOptions::new(brand); let movie_ts = opts.movie_timescale.max(1) as i128; let mut mux: Writer = Mp4Writer::new(&mut file, opts).map_err(mux_err)?; @@ -257,15 +289,22 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let vt = mux.add_track(vcfg).map_err(mux_err)?; let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); let with_audio = wants_audio(job); + let chans = if job.output.audio_channels == 1 { 1u32 } else { 2 }; + let afmt = job.output.audio_format; let mut audio = None; if with_audio { let (a, entry, start) = if brand == Brand::Mp4 { - let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, AUDIO_CHANNELS as usize, 320_000)).map_err(enc)?; - let entry = SampleEntry::aac(e.audio_specific_config(), AUDIO_CHANNELS, sr); + let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, chans as usize, 160_000 * chans)).map_err(enc)?; + let entry = SampleEntry::aac(e.audio_specific_config(), chans, sr); let priming = e.priming_samples() as i64; (Audio::Aac(Box::new(e)), entry, Some(priming)) } else { - let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: AUDIO_CHANNELS, sample_rate: sr as f64 }; + let (bits, float) = match afmt { + AudioFormat::S16 => (16, false), + AudioFormat::S24 => (24, false), + AudioFormat::F32 => (32, true), + }; + let pcm = PcmConfig { bits, float, big_endian: false, signed: true, channels: chans, sample_rate: sr as f64 }; (Audio::Pcm, SampleEntry::pcm(pcm), None) }; let mut c = TrackConfig::new(entry, sr); @@ -290,21 +329,18 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R } let n = (end - *cursor) as usize; let start = Tick(((*cursor as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - let buf = effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, sr); + let buf = mix(job, start, n, sr); *cursor = end; match a { Audio::Aac(e) => { - let planar = deinterleave(&buf); + let planar = deinterleave(&buf, chans as usize); let refs: Vec<&[f32]> = planar.iter().map(Vec::as_slice).collect(); for au in e.encode(&refs) { mux.write_sample(*at, WriteSample { data: &au, duration: 1024, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } Audio::Pcm => { - let mut pcm = Vec::with_capacity(n * 4); - for s in buf { - pcm.extend_from_slice(&((s.clamp(-1.0, 1.0) * 32767.0).round() as i16).to_le_bytes()); - } + let pcm = pcm_bytes(&buf, afmt, false); mux.write_sample(*at, WriteSample { data: &pcm, duration: n as u32, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } @@ -336,7 +372,7 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let out = mux.finish().map_err(mux_err)?; out.flush().map_err(mux_err)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index c9196625..74b12c76 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -2,13 +2,21 @@ //! //! | Format | Container / files | Video | Alpha | Audio | //! |---|---|---|---|---| -//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), stereo | -//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | 16-bit PCM, stereo | +//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), mono/stereo | +//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | PCM 16/24-bit or 32-bit float, mono/stereo | //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | //! | WebM | WebM (Matroska) | VP9 profile 0 (`effectcraft-vp9enc`: key + inter frames, a key frame every 2 s, loop filter), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | -//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | PCM 16/24-bit (WAV also 32-bit float), mono/stereo (audio only) | +//! +//! Every frame goes through [`pipeline`]: the Render Settings overrides (Effects, Solo Switches, +//! Guide Layers, Frame Blending, Motion Blur, Color Depth), field rendering (two fields half a +//! frame apart woven line by line, with optional 3:2 pulldown from film frames at 4/5 of the +//! rate), then the Output Module's Crop (edges or the region of interest) and Resize. Channels +//! may be RGB, RGB + Alpha (straight or premultiplied) or the alpha matte alone. With Use Storage +//! Overflow and a [`StorageQuota`] hook, files that don't fit go to the overflow folders. A +//! render log (Errors Only / Plus Settings / Plus Per Frame Info) is written next to the output. //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -20,19 +28,21 @@ mod encode; mod out; +mod pipeline; mod webm; use web_time::Instant; -use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderQuality, RenderSettings, sequence_path}; +use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderLog, RenderQuality, RenderSettings, sequence_path}; use effectcraft_project::{Comp, ItemId, Project}; use effectcraft_raster::Image; -use effectcraft_render::{ExprHost, FootageSource, RenderOpts, Renderer}; -use effectcraft_time::Tick; +use effectcraft_render::{ExprHost, FootageSource}; use rayon::prelude::*; pub use effectcraft_project::render_queue; +pub use effectcraft_project::render_queue::StorageQuota; pub use out::Sink; +pub(crate) use pipeline::Cx; #[derive(Debug, thiserror::Error)] pub enum ExportError { @@ -69,6 +79,21 @@ pub struct Job<'a> { pub path: &'a str, /// Hand finished files to this instead of writing them to disk (web downloads, tests). pub sink: Option<&'a Sink>, + /// Render log, storage overflow. + pub options: JobOptions<'a>, +} + +/// Job options beyond the Render Settings and Output Module. +#[derive(Default)] +pub struct JobOptions<'a> { + /// The render log written next to the output. + pub log: RenderLog, + /// What the log calls the job (e.g. "#2 Main Comp"). + pub label: String, + /// Free-space hook for Use Storage Overflow (`None`: never full). + pub storage: Option<&'a dyn StorageQuota>, + /// Overflow folders, in order. + pub overflow: Vec, } /// Progress after a batch of frames. @@ -100,6 +125,10 @@ pub struct Report { pub seconds: f64, pub bytes: u64, pub audio: bool, + /// The render log, when one was written. + pub log: Option, + /// Files that went to an overflow folder. + pub overflow: Vec, } /// Formats this build can write (all of them; kept as the hook for optional encoders). @@ -107,10 +136,9 @@ pub fn available_formats() -> Vec { OutputFormat::ALL.to_vec() } -/// The size of the written frames for a job (H.264 rounds to even). +/// The size of the written frames for a job (crop, resize; H.264 rounds to even). pub fn output_size(comp: &Comp, settings: &RenderSettings, output: &OutputModule) -> (u32, u32) { - let (w, h) = settings.output_size(comp); - output.format.coded_size(w, h) + output.output_size(comp, settings) } /// Whether the job writes an audio track. @@ -125,19 +153,30 @@ pub fn wants_audio(job: &Job) -> bool { /// Run the export (blocking). `progress` is called after every batch; returning `false` cancels. pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result { - let comp = job.project.comp(job.comp).ok_or(ExportError::NoComp)?; let t0 = Instant::now(); - let total = job.settings.frame_count(comp); + let cx = Cx::new(job); + let mut r = export_with(&cx, progress, t0); + let log = cx.write_log(&r, t0.elapsed().as_secs_f64()); + if let Ok(rep) = &mut r { + rep.log = log; + rep.overflow = cx.overflowed.lock().unwrap_or_else(|e| e.into_inner()).clone(); + } + r +} + +fn export_with(cx: &Cx, progress: &mut dyn FnMut(&Progress) -> bool, t0: Instant) -> Result { + let comp = cx.comp().ok_or(ExportError::NoComp)?; + let total = cx.settings.frame_count(comp); let mut st = State { t0, total, done: 0, progress }; st.advance(0)?; - let (w, h) = output_size(comp, job.settings, job.output); - let mut report = match job.output.format { - OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, - OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, - OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, - OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, - OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, - f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, + let (w, h) = output_size(comp, cx.settings, cx.output); + let mut report = match cx.output.format { + OutputFormat::H264 | OutputFormat::ProRes => encode::movie(cx, comp, w, h, &mut st)?, + OutputFormat::Gif => gif_export(cx, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(cx, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(cx, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(cx, comp, true, &mut st)?, + f if f.is_sequence() => sequence(cx, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; report.seconds = t0.elapsed().as_secs_f64(); @@ -170,37 +209,8 @@ pub(crate) fn batch_size() -> u64 { rayon::current_num_threads().clamp(2, 16) as u64 } -/// Render output frame `i` of the job at the settings' resolution. -pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { - let t: Tick = job.settings.frame_time(comp, i); - let opts = RenderOpts { - scale: job.settings.resolution.clamp(0.01, 4.0), - motion_blur: job.settings.motion_blur, - guides: false, - draft: job.settings.quality == RenderQuality::Draft, - // Output renders always look through the comp's active camera. - view: None, - backend: effectcraft_render::Backend::Auto, - roi: None, - proxy: job.settings.proxy_use, - }; - let mut r = Renderer::new(job.project, job.footage, opts); - r.expr = job.expr; - r.accel = job.accel; - r.comp_frame(job.comp, t) -} - -/// 8-bit RGBA of the requested channels (RGB: over the comp background, opaque; RGBA: straight), -/// cropped/padded to `w`×`h`. -pub(crate) fn rgba8(img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { - let px = match channels { - Channels::Rgb => img.to_rgba8_over(comp.background), - Channels::Rgba => img.to_rgba8(), - }; - fit(px, img.width, img.height, w, h) -} - -fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { +/// Crop/pad 8-bit RGBA to `w`×`h` (codec rounding). +pub(crate) fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { if iw == w && ih == h { return px; } @@ -217,10 +227,13 @@ fn first_frame_number(job: &Job, comp: &Comp) -> i64 { job.settings.first_frame(comp) } -fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { - let f0 = first_frame_number(job, comp); - let fmt = job.output.format; - let channels = if fmt.supports_alpha() { job.output.channels } else { Channels::Rgb }; +fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let f0 = first_frame_number(cx, comp); + let fmt = cx.output.format; + let channels = match cx.output.channels { + Channels::Rgba if !fmt.supports_alpha() => Channels::Rgb, + c => c, + }; let batch = batch_size(); let mut bytes = 0u64; let mut i = 0; @@ -229,13 +242,18 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result> = (i..end) .into_par_iter() .map(|k| { - let path = sequence_path(job.path, f0 + k as i64); - if job.settings.skip_existing && out::exists(job.sink, &path) { + let path = sequence_path(cx.path, f0 + k as i64); + if cx.settings.skip_existing && out::exists(cx.sink, &path) { return Ok(0); } - let img = render_frame(job, comp, k); - let mut f = out::create(job.sink, &path)?; - write_still(&mut f, fmt, &img, comp, channels, w, h, job.output.quality)?; + let img = cx.frame(comp, k); + // Encode in memory first: the storage hook decides where it fits. + let mut buf = std::io::Cursor::new(Vec::new()); + write_still(cx, &mut buf, fmt, &img, comp, channels, w, h, cx.output.quality)?; + let data = buf.into_inner(); + let path = cx.place(&path, data.len() as u64); + let mut f = out::create(cx.sink, &path)?; + std::io::Write::write_all(&mut f, &data).map_err(io)?; f.finish() }) .collect(); @@ -245,11 +263,21 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result Result<()> { +fn write_still( + cx: &Cx, + file: &mut W, + fmt: OutputFormat, + img: &Image, + comp: &Comp, + channels: Channels, + w: u32, + h: u32, + quality: u8, +) -> Result<()> { use image::{ExtendedColorType, ImageEncoder, ImageFormat}; match fmt { OutputFormat::ExrSequence => { @@ -260,6 +288,10 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, for x in 0..w { let p = if x < img.width && y < img.height { img.data[img.idx(x, y)] } else { [0.0; 4] }; let (c, a) = match channels { + Channels::Alpha => { + let a = p[3].clamp(0.0, 1.0); + ([lin(a); 3], 1.0) + } Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); let s = |c: f32| if a > 0.0 { lin((c / a).clamp(0.0, 1.0)) * a } else { 0.0 }; @@ -278,16 +310,17 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, image::DynamicImage::ImageRgba32F(buf).write_to(file, ImageFormat::OpenExr).map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::JpegSequence => { - let px = rgba8(img, comp, Channels::Rgb, w, h); + let px = cx.pixels(img, comp, if channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }, w, h); let rgb: Vec = px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(); image::codecs::jpeg::JpegEncoder::new_with_quality(file, quality.clamp(1, 100)) .write_image(&rgb, w, h, ExtendedColorType::Rgb8) .map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::PngSequence | OutputFormat::TiffSequence => { - let px = rgba8(img, comp, channels, w, h); + let px = cx.pixels(img, comp, channels, w, h); let (data, ct) = match channels { Channels::Rgba => (px, ExtendedColorType::Rgba8), + Channels::Alpha => (px.chunks_exact(4).map(|p| p[0]).collect(), ExtendedColorType::L8), Channels::Rgb => (px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(), ExtendedColorType::Rgb8), }; if fmt == OutputFormat::PngSequence { @@ -301,11 +334,12 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, } } -fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +fn gif_export(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { if w > u16::MAX as u32 || h > u16::MAX as u32 { return Err(ExportError::Unsupported("GIF frames are limited to 65535 px".into())); } - let mut file = out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = out::create(job.sink, &path)?; let mut enc = gif::Encoder::new(&mut file, w as u16, h as u16, &[]).map_err(|e| ExportError::Encode(e.to_string()))?; enc.set_repeat(if job.output.gif_loop { gif::Repeat::Infinite } else { gif::Repeat::Finite(0) }).map_err(|e| ExportError::Encode(e.to_string()))?; let rate = job.settings.rate(comp); @@ -320,8 +354,8 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< let frames: Vec> = (i..end) .into_par_iter() .map(|k| { - let img = render_frame(job, comp, k); - let mut px = rgba8(&img, comp, job.output.channels, w, h); + let img = job.frame(comp, k); + let mut px = job.pixels(&img, comp, job.output.channels, w, h); gif::Frame::from_rgba_speed(w as u16, h as u16, &mut px, 10) }) .collect(); @@ -340,7 +374,7 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< } drop(enc); let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs new file mode 100644 index 00000000..76087f2a --- /dev/null +++ b/crates/export/src/pipeline.rs @@ -0,0 +1,327 @@ +//! The frame pipeline shared by every output format: Render Settings overrides (effects, solo, +//! guide layers, frame blending, motion blur), field rendering with 3:2 pulldown, colour depth, +//! the Output Module's crop and resize, alpha handling, storage overflow and the render log. + +use std::sync::Mutex; + +use effectcraft_project::render_queue::{ + AlphaMode, Channels, ColorDepth, CurrentOrOff, EffectsMode, FrameSample, RenderLog, ResizeQuality, SwitchOverride, log_path, +}; +use effectcraft_project::{Comp, Project}; +use effectcraft_raster::Image; +use effectcraft_render::{RenderOpts, Renderer}; +use effectcraft_time::Tick; +use web_time::Instant; + +use crate::{Job, RenderQuality, out}; + +/// A job being exported: the job, the project with the Render Settings overrides applied, and +/// per-frame timing for the render log. +pub(crate) struct Cx<'a> { + pub job: &'a Job<'a>, + pub project: Project, + /// (output frame, seconds) of every rendered frame (Plus Per Frame Info logs). + pub frame_times: Option>>, + /// Files written to an overflow folder. + pub overflowed: Mutex>, +} + +impl<'a> std::ops::Deref for Cx<'a> { + type Target = Job<'a>; + fn deref(&self) -> &Job<'a> { + self.job + } +} + +/// The project as the Render Settings render it: Effects All On / All Off, Solo Switches All +/// Off, Frame Blending and Motion Blur "On for Checked Layers" (the composition switches on). +pub(crate) fn apply_overrides(p: &mut Project, job: &Job) { + let s = job.settings; + let ids: Vec<_> = p.comps().map(|(i, _)| *i).collect(); + for id in ids { + let Some(c) = p.comp_mut(id) else { continue }; + match s.frame_blending { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_frame_blending = true, + SwitchOverride::OffForAll => c.enable_frame_blending = false, + } + match s.motion_blur_override() { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_motion_blur = true, + SwitchOverride::OffForAll => c.enable_motion_blur = false, + } + for l in c.layers.iter_mut() { + match s.effects { + EffectsMode::Current => {} + EffectsMode::AllOn => l.switches.effects = true, + EffectsMode::AllOff => l.switches.effects = false, + } + if s.solo == CurrentOrOff::AllOff { + l.switches.solo = false; + } + } + } +} + +impl<'a> Cx<'a> { + pub fn new(job: &'a Job<'a>) -> Cx<'a> { + let mut project = job.project.clone(); + apply_overrides(&mut project, job); + let frame_times = (job.options.log == RenderLog::PlusPerFrameInfo).then(|| Mutex::new(Vec::new())); + Cx { job, project, frame_times, overflowed: Mutex::new(vec![]) } + } + + pub fn comp(&self) -> Option<&Comp> { + self.project.comp(self.job.comp) + } + + fn render_at(&self, t: Tick) -> Image { + let s = self.job.settings; + let opts = RenderOpts { + scale: s.resolution.clamp(0.01, 4.0), + motion_blur: s.motion_blur_override() != SwitchOverride::OffForAll, + guides: s.guide_layers == CurrentOrOff::Current, + draft: s.quality == RenderQuality::Draft, + // Output renders always look through the comp's active camera. + view: None, + backend: effectcraft_render::Backend::Auto, + roi: None, + proxy: s.proxy_use, + }; + let mut r = Renderer::new(&self.project, self.job.footage, opts); + r.expr = self.job.expr; + r.accel = self.job.accel; + r.comp_frame(self.job.comp, t) + } + + /// Output frame `i`: sampled (fields, pulldown), at the colour depth, cropped and resized. + pub fn frame(&self, comp: &Comp, i: u64) -> Image { + let t0 = Instant::now(); + let img = match self.job.settings.sample(comp, i) { + FrameSample::Progressive(t) => self.render_at(t), + FrameSample::Fields { first, second, upper_first } => { + let a = self.render_at(first); + if second == first { a } else { interleave(&a, &self.render_at(second), upper_first) } + } + }; + let mut img = img; + // Color Depth: renders run at the project's depth; another depth quantises the frame. + let depth = self.job.settings.color_depth; + if depth != ColorDepth::Current + && let Some(levels) = depth.resolve(self.project.settings.bit_depth).levels() + { + img.quantize(levels); + } + let img = self.crop_resize(img); + if let Some(ft) = &self.frame_times { + ft.lock().unwrap_or_else(|e| e.into_inner()).push((i, t0.elapsed().as_secs_f64())); + } + img + } + + /// Output Module ▸ Crop, then Resize. + fn crop_resize(&self, img: Image) -> Image { + let om = self.job.output; + let scale = self.job.settings.resolution.clamp(0.01, 4.0); + let (t, l, b, r) = om.crop.edges(img.width, img.height, scale); + let img = if (t, l, b, r) == (0, 0, 0, 0) { img } else { crop(&img, t, l, b, r) }; + if !om.resize.enabled { + return img; + } + let (w, h) = om.frame_size(img.width, img.height, 1.0); + if (w, h) == (img.width, img.height) { + return img; + } + match om.resize.quality { + ResizeQuality::Low => effectcraft_raster::warp::resample(&img, w, h), + ResizeQuality::High => bicubic(&img, w, h), + } + } + + /// 8-bit RGBA of the requested channels cropped/padded to `w`×`h` (codec rounding): RGB over + /// the comp background (opaque), RGB + Alpha straight or premultiplied, or the alpha matte. + pub fn pixels(&self, img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { + let px = match channels { + Channels::Rgb => img.to_rgba8_over(comp.background), + Channels::Rgba => match self.job.output.alpha_mode { + AlphaMode::Straight => img.to_rgba8(), + AlphaMode::Premultiplied => premultiplied8(img), + }, + Channels::Alpha => img + .data + .iter() + .flat_map(|p| { + let a = (p[3].clamp(0.0, 1.0) * 255.0 + 0.5) as u8; + [a, a, a, 255] + }) + .collect(), + }; + crate::fit(px, img.width, img.height, w, h) + } + + /// Where a file goes: `path`, or (Use Storage Overflow) the first overflow folder with room + /// for `bytes` when the quota hook says the output volume is full. + pub fn place(&self, path: &str, bytes: u64) -> String { + let Some(q) = self.job.options.storage else { return path.to_string() }; + if !self.job.settings.storage_overflow || q.has_room(path, bytes) { + return path.to_string(); + } + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + for dir in &self.job.options.overflow { + let p = std::path::Path::new(dir).join(&name).to_string_lossy().to_string(); + if q.has_room(&p, bytes) { + self.overflowed.lock().unwrap_or_else(|e| e.into_inner()).push(p.clone()); + return p; + } + } + path.to_string() + } + + /// Write the render log next to the output, as the Log setting asks. Returns its path. + pub fn write_log(&self, result: &Result, seconds: f64) -> Option { + let level = self.job.options.log; + // Errors Only: a log for failures (a user stop is not an error). + if level == RenderLog::ErrorsOnly && matches!(result, Ok(_) | Err(crate::ExportError::Cancelled)) { + return None; + } + let comp = self.comp(); + let path = log_path(self.job.path); + let mut s = String::new(); + s.push_str("EffectCraft Render Log\n"); + s.push_str(&format!("Item: {}\n", if self.job.options.label.is_empty() { "(render)" } else { &self.job.options.label })); + s.push_str(&format!("Output: {}\n", self.job.path)); + match result { + Ok(r) => s.push_str(&format!("Result: Done ({} frames, {}×{}, {} bytes) in {seconds:.2} s\n", r.frames, r.width, r.height, r.bytes)), + Err(e) => s.push_str(&format!("Result: Failed\nError: {e}\n")), + } + let over = self.overflowed.lock().unwrap_or_else(|e| e.into_inner()); + if !over.is_empty() { + s.push_str(&format!("Storage overflow: {} file(s) written to overflow folders (first: {})\n", over.len(), over[0])); + } + if level != RenderLog::ErrorsOnly { + s.push_str("\nRender Settings:\n"); + for (k, v) in self.job.settings.describe(comp) { + s.push_str(&format!(" {k}: {v}\n")); + } + s.push_str("\nOutput Module:\n"); + for (k, v) in self.job.output.describe() { + s.push_str(&format!(" {k}: {v}\n")); + } + } + if let (RenderLog::PlusPerFrameInfo, Some(ft), Some(c)) = (level, &self.frame_times, comp) { + let mut v = ft.lock().unwrap_or_else(|e| e.into_inner()).clone(); + v.sort_by_key(|(i, _)| *i); + s.push_str("\nFrames:\n"); + let f0 = self.job.settings.first_frame(c); + for (i, secs) in v { + let t = self.job.settings.frame_time(c, i); + s.push_str(&format!(" Frame {} ({:.3} s): rendered in {:.1} ms\n", f0 + i as i64, t.seconds(), secs * 1000.0)); + } + } + let mut f = out::create(self.job.sink, &path).ok()?; + std::io::Write::write_all(&mut f, s.as_bytes()).ok()?; + f.finish().ok()?; + Some(path) + } +} + +/// Weave two fields: the dominant field's lines (even lines when `upper_first`) from `first`. +pub(crate) fn interleave(first: &Image, second: &Image, upper_first: bool) -> Image { + let mut out = first.clone(); + let w = out.width as usize; + for y in 0..out.height as usize { + let from_first = (y % 2 == 0) == upper_first; + if !from_first && y < second.height as usize && second.width as usize == w { + out.data[y * w..(y + 1) * w].copy_from_slice(&second.data[y * w..(y + 1) * w]); + } + } + out +} + +/// Remove `t`/`l`/`b`/`r` pixels from the edges (negative values add transparent pixels). +pub(crate) fn crop(img: &Image, t: i32, l: i32, b: i32, r: i32) -> Image { + let w = (img.width as i32 - l - r).max(1) as u32; + let h = (img.height as i32 - t - b).max(1) as u32; + let mut out = Image::new(w, h); + for y in 0..h as i32 { + let sy = y + t; + if sy < 0 || sy >= img.height as i32 { + continue; + } + for x in 0..w as i32 { + let sx = x + l; + if sx >= 0 && sx < img.width as i32 { + let o = out.idx(x as u32, y as u32); + out.data[o] = img.data[img.idx(sx as u32, sy as u32)]; + } + } + } + out +} + +/// Bicubic resize (Resize Quality: High). +fn bicubic(src: &Image, w: u32, h: u32) -> Image { + // Shrinking a lot: bilinear with its box pre-filter first, then bicubic for the last step. + if src.width > w * 2 && src.height > h * 2 { + let mid = effectcraft_raster::warp::resample(src, w * 2, h * 2); + return bicubic(&mid, w, h); + } + let mut out = Image::new(w, h); + let (sx, sy) = (src.width as f64 / w as f64, src.height as f64 / h as f64); + for y in 0..h { + for x in 0..w { + let o = out.idx(x, y); + // Clamp to the edge so borders stay opaque. + let fx = ((x as f64 + 0.5) * sx).clamp(0.5, src.width as f64 - 0.5); + let fy = ((y as f64 + 0.5) * sy).clamp(0.5, src.height as f64 - 0.5); + let mut p = src.sample_bicubic(fx, fy); + p[3] = p[3].min(1.0); + out.data[o] = p; + } + } + out +} + +/// Premultiplied 8-bit RGBA (colour matted with black). +fn premultiplied8(img: &Image) -> Vec { + img.data + .iter() + .flat_map(|p| { + let a = p[3].clamp(0.0, 1.0); + [0, 1, 2].map(|c| (p[c].clamp(0.0, a) * 255.0 + 0.5) as u8).into_iter().chain([(a * 255.0 + 0.5) as u8]) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interleave_takes_alternate_lines() { + let a = Image::filled(3, 4, [1.0, 0.0, 0.0, 1.0]); + let b = Image::filled(3, 4, [0.0, 0.0, 1.0, 1.0]); + let up = interleave(&a, &b, true); + let lo = interleave(&a, &b, false); + for y in 0..4u32 { + assert_eq!(up.data[up.idx(1, y)][0], if y % 2 == 0 { 1.0 } else { 0.0 }, "upper first: line {y}"); + assert_eq!(lo.data[lo.idx(1, y)][0], if y % 2 == 1 { 1.0 } else { 0.0 }, "lower first: line {y}"); + } + } + + #[test] + fn crop_and_pad() { + let mut img = Image::new(4, 3); + for (i, p) in img.data.iter_mut().enumerate() { + *p = [i as f32, 0.0, 0.0, 1.0]; + } + let c = crop(&img, 1, 1, 0, 1); + assert_eq!((c.width, c.height), (2, 2)); + assert_eq!(c.data[0][0], 5.0); + let p = crop(&img, -1, 0, 0, 0); + assert_eq!((p.width, p.height), (4, 4)); + assert_eq!(p.data[0], [0.0; 4]); + assert_eq!(p.data[4][0], 0.0); + assert_eq!(p.data[5][0], 1.0); + } +} diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index 65ca1189..af38eaeb 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -15,11 +15,12 @@ use std::io::{Seek, SeekFrom, Write}; use effectcraft_project::Comp; -use effectcraft_project::render_queue::Channels; +use effectcraft_project::render_queue::{AudioFormat, Channels}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use rayon::prelude::*; -use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; +use crate::encode::{mix, pcm_bytes}; +use crate::{Cx, Report, Result, State, batch_size, io, wants_audio}; const OPUS_RATE: u32 = 48_000; const OPUS_BITRATE: u32 = 192_000; @@ -124,9 +125,10 @@ fn cluster(frames: &mut [Frame]) -> Vec { out } -pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn webm(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; + let colour_channels = if job.output.channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }; let quality = job.output.quality.clamp(1, 100); let fps = rate.as_f64().max(1.0); let cfg = effectcraft_vp9enc::EncoderConfig { @@ -134,20 +136,22 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re height: h, quality, full_range: false, - keyframe_interval: (fps * 2.0).round().max(1.0) as u32, - target_kbps: None, + keyframe_interval: if job.output.keyframe_interval > 0 { job.output.keyframe_interval } else { (fps * 2.0).round().max(1.0) as u32 }, + target_kbps: job.output.webm_bitrate.then_some(job.output.bitrate_kbps.max(50)), frame_rate: fps, loop_filter: true, }; let mut color_enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); let mut alpha_enc = alpha.then(|| effectcraft_vp9enc::Vp9Encoder::new(cfg.clone())); let with_audio = wants_audio(job); - let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let opus_channels = if job.output.audio_channels == 1 { 1usize } else { 2 }; + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(opus_channels as _, OPUS_BITRATE)); let total = st.total; let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; let duration_ms = frame_ms(total) as f64; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; // EBML header. let mut head = vec![]; let mut eb = vec![]; @@ -201,7 +205,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re el_uint(&mut a, 0x56BB, 80_000_000); let mut au = vec![]; el_float(&mut au, 0xB5, OPUS_RATE as f64); - el_uint(&mut au, 0x9F, 2); + el_uint(&mut au, 0x9F, opus_channels as u64); el(&mut a, 0xE1, &au); el(&mut tracks, TRACK_ENTRY, &a); } @@ -220,10 +224,10 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re if end > cursor { let n = (end - cursor) as usize; let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); - pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + pcm_left.extend(mix(job, start, n, OPUS_RATE)); cursor = end; } - let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * opus_channels; if finish && !pcm_left.is_empty() { // Pad the last packet (pre-skip + padding cover the encoder delay). let pad = (fs - pcm_left.len() % fs) % fs + fs; @@ -244,7 +248,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and // alpha side by side. let pixels: Vec> = - (i..end).into_par_iter().map(|k| rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h)).collect(); + (i..end).into_par_iter().map(|k| job.pixels(&job.frame(comp, k), comp, if alpha { Channels::Rgba } else { colour_channels }, w, h)).collect(); let mut frames: Vec = Vec::with_capacity(pixels.len()); for (j, px) in pixels.iter().enumerate() { let (color, a) = rayon::join( @@ -288,7 +292,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re file.write_all(&size).map_err(io)?; file.seek(SeekFrom::End(0)).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } // ---------------------------------------------------------------- audio-only outputs @@ -306,14 +310,25 @@ fn ieee_extended(v: f64) -> [u8; 10] { out } -/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. -pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { +/// WAV or AIFF: the comp's audio over the render span (Audio Output: sample rate, mono/stereo, +/// 16/24-bit integer or 32-bit float; AIFF has no float in plain AIFF, so it writes 24-bit). +pub(crate) fn audio_file(job: &Cx, comp: &Comp, aiff: bool, st: &mut State) -> Result { let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let channels = if job.output.audio_channels == 1 { 1u16 } else { 2 }; + let fmt = match job.output.audio_format { + AudioFormat::F32 if aiff => AudioFormat::S24, + f => f, + }; + let bytes_per_sample: u16 = match fmt { + AudioFormat::S16 => 2, + AudioFormat::S24 => 3, + AudioFormat::F32 => 4, + }; let (span_start, span_end) = job.settings.span(comp); let start = span_start.to_units_floor(sr as i64); let end = span_end.to_units_floor(sr as i64); let n = (end - start).max(0) as usize; - let mut pcm = Vec::with_capacity(n * 4); + let mut pcm = Vec::with_capacity(n * (channels * bytes_per_sample) as usize); // Mix in one-second blocks, reporting progress in frames. let block = sr as usize; let mut done = 0usize; @@ -322,10 +337,7 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> while done < n { let k = block.min(n - done); let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { - let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; - pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); - } + pcm.extend(pcm_bytes(&mix(job, t, k, sr), fmt, aiff)); done += k; let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); if now > reported && st.total > 0 { @@ -336,13 +348,14 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> if reported < st.total { st.advance(st.total - reported)?; } + let block_align = channels * bytes_per_sample; + let frames = (pcm.len() / block_align as usize) as u32; let mut out = vec![]; - let frames = (pcm.len() / 4) as u32; if aiff { let mut comm = vec![]; - comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&channels.to_be_bytes()); comm.extend_from_slice(&frames.to_be_bytes()); - comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&(bytes_per_sample * 8).to_be_bytes()); comm.extend_from_slice(&ieee_extended(sr as f64)); let ssnd_len = 8 + pcm.len(); out.extend_from_slice(b"FORM"); @@ -359,24 +372,30 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> out.push(0); } } else { + // WAVE_FORMAT_PCM (1) or WAVE_FORMAT_IEEE_FLOAT (3). + let tag: u16 = if fmt == AudioFormat::F32 { 3 } else { 1 }; out.extend_from_slice(b"RIFF"); - out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(&((36 + pcm.len() + pcm.len() % 2) as u32).to_le_bytes()); out.extend_from_slice(b"WAVEfmt "); out.extend_from_slice(&16u32.to_le_bytes()); - out.extend_from_slice(&1u16.to_le_bytes()); - out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&tag.to_le_bytes()); + out.extend_from_slice(&channels.to_le_bytes()); out.extend_from_slice(&sr.to_le_bytes()); - out.extend_from_slice(&(sr * 4).to_le_bytes()); - out.extend_from_slice(&4u16.to_le_bytes()); - out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(&(sr * block_align as u32).to_le_bytes()); + out.extend_from_slice(&block_align.to_le_bytes()); + out.extend_from_slice(&(bytes_per_sample * 8).to_le_bytes()); out.extend_from_slice(b"data"); out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); out.extend_from_slice(&pcm); + if pcm.len() % 2 == 1 { + out.push(0); + } } - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, out.len() as u64); + let mut file = crate::out::create(job.sink, &path)?; file.write_all(&out).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) + Ok(Report { path, frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/tests/render_settings.rs b/crates/export/tests/render_settings.rs new file mode 100644 index 00000000..3347efe1 --- /dev/null +++ b/crates/export/tests/render_settings.rs @@ -0,0 +1,310 @@ +//! Render Settings and Output Module features end to end (through a [`Sink`], so nothing touches +//! the disk): field rendering and 3:2 pulldown, crop / region of interest / resize, alpha-only +//! and premultiplied output, the solo and guide-layer overrides, the render log, storage +//! overflow and PCM audio formats. + +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::*; +use effectcraft_export::{Job, JobOptions, Report, Sink, StorageQuota, export}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; + +const W: u32 = 64; +const H: u32 = 48; + +/// A comp at `rate`, black, with a white bar 8 px wide sweeping left to right (64 px/s) and a +/// static red block at the bottom left; `extra` adds layers. +fn project(rate: FrameRate, extra: impl FnOnce(&mut Project, &mut Comp)) -> (Project, ItemId) { + let mut p = Project::default(); + let mut comp = Comp::new(W, H, rate, Tick::from_seconds_f64(2.0)); + comp.background = [0.0, 0.0, 0.0]; + let bar = p.add_item("Bar", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 8, height: 32, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Bar", LayerSource::Solid { item: bar }, (8, 32), None); + l.props.prop_mut("transform/position").unwrap().keys = + vec![Keyframe::new(Tick::ZERO, Value::Vec3([0.0, 16.0, 0.0])), Keyframe::new(Tick::from_seconds_f64(1.0), Value::Vec3([64.0, 16.0, 0.0]))]; + comp.layers.push(l); + let red = p.add_item("Block", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 0.0, 0.0], width: 16, height: 8, pixel_aspect: 1.0 })); + let mut b = build::layer(&mut p, &comp, "Block", LayerSource::Solid { item: red }, (16, 8), None); + b.props.prop_mut("transform/position").unwrap().value = Value::Vec3([8.0, 44.0, 0.0]); + comp.layers.push(b); + extra(&mut p, &mut comp); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +type Files = Arc)>>>; + +fn custom(start: f64, end: f64) -> RenderSettings { + RenderSettings { time_span: TimeSpan::Custom { start: Tick::from_seconds_f64(start), end: Tick::from_seconds_f64(end) }, ..Default::default() } +} + +fn run_with(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str, options: JobOptions, cancel: bool) -> (Result, Files) { + let files: Files = Default::default(); + let f = files.clone(); + let sink: Box = Box::new(move |path: &str, data: Vec| f.lock().unwrap().push((path.to_string(), data))); + let job = Job { project: p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: s, output: om, path, sink: Some(&*sink), options }; + let r = export(&job, &mut |pr| !(cancel && pr.done > 0)).map_err(|e| e.to_string()); + (r, files) +} + +fn run(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str) -> Files { + let (r, files) = run_with(p, cid, s, om, path, JobOptions::default(), false); + r.expect("export"); + files +} + +fn png(files: &Files, i: usize) -> image::RgbaImage { + let f = files.lock().unwrap(); + let pngs: Vec<&(String, Vec)> = f.iter().filter(|(p, _)| p.ends_with(".png")).collect(); + let mut pngs = pngs; + pngs.sort_by(|a, b| a.0.cmp(&b.0)); + image::load_from_memory(&pngs[i].1).expect("png").to_rgba8() +} + +fn row(img: &image::RgbaImage, y: u32) -> Vec { + (0..img.width()).flat_map(|x| img.get_pixel(x, y).0).collect() +} + +#[test] +fn field_render_interleaves_two_times() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + // Frame 10 at 25 fps with fields: 0.40 s and 0.42 s. + for (fr, upper) in [(FieldRender::UpperFirst, true), (FieldRender::LowerFirst, false)] { + let s = RenderSettings { field_render: fr, ..custom(0.4, 0.44) }; + let fields = png(&run(&p, cid, &s, &om, "/f/field_[#####].png"), 0); + // Progressive references at 50 fps: frames 20 (0.40 s) and 21 (0.42 s). + let r = RenderSettings { frame_rate: Some(FrameRate::new(50, 1)), ..custom(0.4, 0.44) }; + let refs = run(&p, cid, &r, &om, "/f/ref_[#####].png"); + let (a, b) = (png(&refs, 0), png(&refs, 1)); + assert_ne!(row(&a, 10), row(&b, 10), "the bar moved between the fields"); + for y in 0..H { + let first = (y % 2 == 0) == upper; + assert_eq!(row(&fields, y), row(if first { &a } else { &b }, y), "{fr:?}: line {y}"); + } + } +} + +#[test] +fn pulldown_follows_the_cadence() { + let ntsc = FrameRate::new(30000, 1001); + let (p, cid) = project(ntsc, |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let five = 5.0 * 1001.0 / 30000.0; + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, ..custom(0.0, five) }; + let out = run(&p, cid, &s, &om, "/p/pd_[#####].png"); + assert_eq!(out.lock().unwrap().len(), 5); + // Film frames at 23.976 fps. + let r = RenderSettings { frame_rate: Some(FrameRate::new(24000, 1001)), ..custom(0.0, 0.2) }; + let film = run(&p, cid, &r, &om, "/p/film_[#####].png"); + let mut pattern = String::new(); + for k in 0..5 { + let (a, b) = Pulldown::Wssww.fields(k as u64).unwrap(); + pattern.push(if a == b { 'W' } else { 'S' }); + let img = png(&out, k); + let (fa, fb) = (png(&film, a as usize), png(&film, b as usize)); + for y in 0..H { + assert_eq!(row(&img, y), row(if y % 2 == 0 { &fa } else { &fb }, y), "frame {k} line {y}"); + } + } + assert_eq!(pattern, "WSSWW"); +} + +#[test] +fn crop_roi_and_resize() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.4, 0.44); + let full = png(&run(&p, cid, &s, &OutputModule::for_format(OutputFormat::PngSequence), "/c/full_[#####].png"), 0); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.crop = Crop { enabled: true, top: 4, left: 10, bottom: 6, right: 2, ..Default::default() }; + let c = png(&run(&p, cid, &s, &om, "/c/crop_[#####].png"), 0); + assert_eq!((c.width(), c.height()), (52, 38)); + for (x, y) in [(0, 0), (20, 11), (51, 37), (3, 30)] { + assert_eq!(c.get_pixel(x, y), full.get_pixel(x + 10, y + 4), "({x}, {y})"); + } + // Region of interest at half resolution. + om.crop = Crop { enabled: true, use_roi: true, roi: Some([16.0, 8.0, 32.0, 32.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..s.clone() }; + let r = png(&run(&p, cid, &half, &om, "/c/roi_[#####].png"), 0); + assert_eq!((r.width(), r.height()), (16, 16)); + // Resize (fixed size, low and high quality) after the crop. + om.crop = Crop::default(); + for q in [ResizeQuality::Low, ResizeQuality::High] { + om.resize = Resize { enabled: true, width: 32, height: 20, lock_aspect: false, quality: q }; + let z = png(&run(&p, cid, &s, &om, "/c/resize_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 20)); + // The red block (bottom left) survives the resize. + let px = z.get_pixel(4, 18).0; + assert!(px[0] > 150 && px[1] < 80, "{q:?}: {px:?}"); + } + om.resize.lock_aspect = true; + let z = png(&run(&p, cid, &s, &om, "/c/aspect_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 24)); +} + +#[test] +fn alpha_only_and_premultiplied() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + // A half-transparent green layer over the right half. + let g = p.add_item("Green", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 1.0, 0.0], width: 32, height: 48, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Green", LayerSource::Solid { item: g }, (32, 48), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([48.0, 24.0, 0.0]); + l.props.prop_mut("transform/opacity").unwrap().value = Value::Scalar(50.0); + comp.layers.insert(0, l); + }); + let s = custom(0.0, 0.04); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.channels = Channels::Alpha; + let a = png(&run(&p, cid, &s, &om, "/a/alpha_[#####].png"), 0); + assert_eq!(a.get_pixel(60, 2).0[0], 128, "half-opaque green"); + assert_eq!(a.get_pixel(20, 2).0[0], 0, "transparent"); + assert_eq!(a.get_pixel(8, 44).0[0], 255, "opaque block"); + om.channels = Channels::Rgba; + let straight = png(&run(&p, cid, &s, &om, "/a/s_[#####].png"), 0); + om.alpha_mode = AlphaMode::Premultiplied; + let premul = png(&run(&p, cid, &s, &om, "/a/p_[#####].png"), 0); + assert_eq!(straight.get_pixel(60, 2).0, [0, 255, 0, 128]); + assert_eq!(premul.get_pixel(60, 2).0, [0, 128, 0, 128]); +} + +#[test] +fn solo_and_guide_layer_overrides() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + let g = p.add_item("Guide", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 0.0, 1.0], width: 8, height: 8, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Guide", LayerSource::Solid { item: g }, (8, 8), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([56.0, 44.0, 0.0]); + l.switches.guide = true; + comp.layers.insert(0, l); + // Solo the moving bar. + comp.layers[1].switches.solo = true; + }); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = custom(0.4, 0.44); + let cur = png(&run(&p, cid, &s, &om, "/s/cur_[#####].png"), 0); + assert_eq!(cur.get_pixel(8, 44).0[0], 0, "solo: the red block is hidden"); + assert_eq!(cur.get_pixel(56, 44).0[2], 0, "guide layers off by default"); + let off = RenderSettings { solo: CurrentOrOff::AllOff, guide_layers: CurrentOrOff::Current, ..s.clone() }; + let all = png(&run(&p, cid, &off, &om, "/s/off_[#####].png"), 0); + assert_eq!(all.get_pixel(8, 44).0[0], 255, "Solo Switches: All Off"); + // (with solo off the guide renders too: Guide Layers: Current Settings) + assert_eq!(all.get_pixel(56, 44).0[2], 255); +} + +#[test] +fn render_logs() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { field_render: FieldRender::UpperFirst, name: "DV-ish".into(), ..custom(0.0, 0.2) }; + let log_of = |files: &Files| { + files.lock().unwrap().iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(p, d)| (p.clone(), String::from_utf8(d.clone()).unwrap())) + }; + // Errors Only + success: no log. + let (r, files) = run_with(&p, cid, &s, &om, "/l/a_[#####].png", JobOptions::default(), false); + assert!(r.unwrap().log.is_none()); + assert!(log_of(&files).is_none()); + // Plus Per Frame Info: settings and one line per frame. + let opts = JobOptions { log: RenderLog::PlusPerFrameInfo, label: "#1 Comp".into(), ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/l/b_[#####].png", opts, false); + let r = r.unwrap(); + let (path, text) = log_of(&files).expect("log"); + assert_eq!(r.log.as_deref(), Some(path.as_str())); + assert_eq!(path, "/l/b_RenderLog.txt"); + assert!(text.contains("Item: #1 Comp") && text.contains("Result: Done (5 frames"), "{text}"); + assert!(text.contains("Field Render: Upper Field First") && text.contains("Template: DV-ish") && text.contains("Format: PNG Sequence"), "{text}"); + assert_eq!(text.lines().filter(|l| l.contains("rendered in")).count(), 5, "{text}"); + assert!(text.contains("Frame 0 (0.000 s): rendered in") && text.contains("Frame 4 (0.160 s)"), "{text}"); + // Plus Settings: no per-frame lines. + let opts = JobOptions { log: RenderLog::PlusSettings, ..Default::default() }; + let (_, files) = run_with(&p, cid, &s, &om, "/l/c.gif", opts, false); + let (_, text) = log_of(&files).expect("log"); + assert!(text.contains("Render Settings:") && !text.contains("rendered in"), "{text}"); + // Errors Only + failure (the composition is gone): the error is logged. + let gif = OutputModule::for_format(OutputFormat::Gif); + let (r, files) = run_with(&p, ItemId(424_242), &s, &gif, "/l/d.gif", JobOptions::default(), false); + assert!(r.is_err()); + let (path, text) = log_of(&files).expect("error log"); + assert_eq!(path, "/l/d_RenderLog.txt"); + assert!(text.contains("Result: Failed") && text.contains("Error: composition not found") && !text.contains("Render Settings:"), "{text}"); + // A user stop is not an error: no log. + let (r, files) = run_with(&p, cid, &s, &gif, "/l/e.gif", JobOptions::default(), true); + assert!(r.is_err()); + assert!(log_of(&files).is_none()); +} + +/// The primary folder holds `limit` bytes; overflow folders are unlimited. +struct Quota { + limit: u64, + used: Mutex, +} + +impl StorageQuota for Quota { + fn has_room(&self, path: &str, bytes: u64) -> bool { + if !path.starts_with("/primary/") { + return true; + } + let mut u = self.used.lock().unwrap(); + if *u + bytes > self.limit { + return false; + } + *u += bytes; + true + } +} + +#[test] +fn storage_overflow_moves_files_to_the_overflow_folder() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { skip_existing: false, ..custom(0.0, 0.4) }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], log: RenderLog::PlusSettings, ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/primary/seq_[#####].png", opts, false); + let r = r.unwrap(); + assert_eq!(r.overflow.len(), 10, "{:?}", r.overflow); + let f = files.lock().unwrap(); + assert_eq!(f.iter().filter(|(p, _)| p.starts_with("/overflow/seq_") && p.ends_with(".png")).count(), 10); + let log = f.iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(_, d)| String::from_utf8_lossy(d).to_string()).unwrap(); + assert!(log.contains("Storage overflow: 10 file(s)"), "{log}"); + drop(f); + // Use Storage Overflow off: everything stays in place. + let off = RenderSettings { storage_overflow: false, ..s }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], ..Default::default() }; + let (r, files) = run_with(&p, cid, &off, &om, "/primary/seq_[#####].png", opts, false); + assert!(r.unwrap().overflow.is_empty()); + assert!(files.lock().unwrap().iter().all(|(p, _)| p.starts_with("/primary/"))); +} + +#[test] +fn wav_and_aiff_channels_and_formats() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.0, 0.5); + let mut om = OutputModule::for_format(OutputFormat::Wav); + om.audio_channels = 1; + om.audio_format = AudioFormat::S24; + om.audio_sample_rate = 44_100; + let files = run(&p, cid, &s, &om, "/w/a.wav"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[..4], b"RIFF"); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + let u32_at = |o: usize| u32::from_le_bytes([d[o], d[o + 1], d[o + 2], d[o + 3]]); + assert_eq!((u16_at(20), u16_at(22), u32_at(24), u16_at(32), u16_at(34)), (1, 1, 44_100, 3, 24)); + assert_eq!(u32_at(40), 22_050 * 3, "half a second of mono 24-bit"); + om.audio_channels = 2; + om.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &om, "/w/b.wav"); + let d = files.lock().unwrap()[0].1.clone(); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + assert_eq!((u16_at(20), u16_at(22), u16_at(32), u16_at(34)), (3, 2, 8, 32), "IEEE float stereo"); + // AIFF has no float: 24-bit. + let mut aiff = OutputModule::for_format(OutputFormat::Aiff); + aiff.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &aiff, "/w/c.aif"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[8..12], b"AIFF"); + assert_eq!(u16::from_be_bytes([d[26], d[27]]), 24, "sample size"); +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 1ade594f..c6810cb0 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -55,7 +55,7 @@ fn settings() -> RenderSettings { fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); - let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None }; + let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None, options: Default::default() }; let mut last = Progress::default(); let mut calls = 0; let r = export(&job, &mut |pr| { @@ -167,7 +167,18 @@ fn h264_odd_size_is_cropped_even() { let comp = p.comp(cid).expect("comp"); assert_eq!(s.output_size(comp).0 % 2, 1); let path = d.join("odd.mp4").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + }; let r = export(&job, &mut |_| true).expect("export"); assert_eq!((r.width % 2, r.height % 2), (0, 0)); } @@ -287,7 +298,18 @@ fn cancel_stops_early() { let om = OutputModule::for_format(OutputFormat::PngSequence); let s = settings(); let path = d.join("c_[#####].png").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + }; let r = export(&job, &mut |_| false); assert!(matches!(r, Err(effectcraft_export::ExportError::Cancelled))); let written = std::fs::read_dir(&d).map(|r| r.count()).unwrap_or(0); diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index b5de7714..4161abc7 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -40,9 +40,25 @@ impl Exporter for FileExporter { output: &job.item.output, path: job.path, sink: self.sink.as_deref(), + options: effectcraft_export::JobOptions { + log: job.item.log, + label: job.label.clone(), + storage: job.storage, + overflow: job.project.render_prefs.overflow_folders.clone(), + }, }; match effectcraft_export::export(&j, &mut |p| progress(p.done, p.total)) { - Ok(r) => Ok(ExportResult { path: r.path, frames: r.frames, width: r.width, height: r.height, bytes: r.bytes, seconds: r.seconds, audio: r.audio }), + Ok(r) => Ok(ExportResult { + path: r.path, + frames: r.frames, + width: r.width, + height: r.height, + bytes: r.bytes, + seconds: r.seconds, + audio: r.audio, + log: r.log, + overflow: r.overflow, + }), Err(effectcraft_export::ExportError::Cancelled) => Err(effectcraft_engine::render_queue::CANCELLED.into()), Err(e) => Err(e.to_string()), } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 470c4edc..bf8fcc2a 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -8,6 +8,7 @@ pub mod build; pub mod essential; pub mod props; pub mod render_queue; +pub mod render_templates; pub mod styles; pub mod tracking; @@ -931,11 +932,25 @@ pub struct Project { /// The Render Queue (Composition ▸ Add to Render Queue). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub render_queue: Vec, + /// Render Settings / Output Module templates and their defaults. + #[serde(default, skip_serializing_if = "render_templates::RenderTemplates::is_default")] + pub render_templates: render_templates::RenderTemplates, + /// Render Queue preferences (Notify, storage overflow folders). + #[serde(default, skip_serializing_if = "render_queue::RenderQueuePrefs::is_default")] + pub render_prefs: render_queue::RenderQueuePrefs, } impl Default for Project { fn default() -> Self { - Project { schema: SCHEMA_VERSION, settings: ProjectSettings::default(), items: BTreeMap::new(), next_id: 1, render_queue: Vec::new() } + Project { + schema: SCHEMA_VERSION, + settings: ProjectSettings::default(), + items: BTreeMap::new(), + next_id: 1, + render_queue: Vec::new(), + render_templates: Default::default(), + render_prefs: Default::default(), + } } } diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index d0f0c91f..35aed0be 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -84,38 +84,329 @@ impl ProxyUse { } } +/// Render Settings ▸ Field Render: interlaced output, each frame woven from two fields rendered +/// half a frame apart. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum FieldRender { + #[default] + Off, + /// The upper field (lines 0, 2, 4…) holds the earlier time. + UpperFirst, + LowerFirst, +} + +impl FieldRender { + pub const ALL: [FieldRender; 3] = [FieldRender::Off, FieldRender::UpperFirst, FieldRender::LowerFirst]; + pub fn label(self) -> &'static str { + match self { + FieldRender::Off => "Off", + FieldRender::UpperFirst => "Upper Field First", + FieldRender::LowerFirst => "Lower Field First", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "off" | "none" | "progressive" => Some(FieldRender::Off), + "upper" | "upperfirst" | "upperfieldfirst" => Some(FieldRender::UpperFirst), + "lower" | "lowerfirst" | "lowerfieldfirst" => Some(FieldRender::LowerFirst), + _ => None, + } + } +} + +fn norm(s: &str) -> String { + s.to_ascii_lowercase().replace([' ', '_', '-', '.', ':', '&'], "") +} + +/// Render Settings ▸ 3:2 Pulldown: with field rendering, four film frames (at 4/5 of the output +/// rate) spread over five interlaced frames in 2:3 cadence. The phase names which of the five +/// frames are Whole (both fields from one film frame) or Split. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum Pulldown { + #[default] + Off, + Wssww, + Sswww, + Swwws, + Wwwss, + Wwssw, +} + +/// Film frame (within a group of four) of each of the ten fields of the 2:3 cadence. +const CADENCE: [i64; 10] = [0, 0, 1, 1, 1, 2, 2, 3, 3, 3]; + +impl Pulldown { + pub const ALL: [Pulldown; 6] = [Pulldown::Off, Pulldown::Wssww, Pulldown::Sswww, Pulldown::Swwws, Pulldown::Wwwss, Pulldown::Wwssw]; + pub fn label(self) -> &'static str { + match self { + Pulldown::Off => "Off", + Pulldown::Wssww => "WSSWW", + Pulldown::Sswww => "SSWWW", + Pulldown::Swwws => "SWWWS", + Pulldown::Wwwss => "WWWSS", + Pulldown::Wwssw => "WWSSW", + } + } + pub fn parse(s: &str) -> Option { + let n = norm(s); + if matches!(n.as_str(), "off" | "none") { + return Some(Pulldown::Off); + } + Pulldown::ALL.into_iter().find(|p| p.label().eq_ignore_ascii_case(&n)) + } + /// Offset (in frames) into the WWSSW base cadence. + fn rotation(self) -> Option { + Some(match self { + Pulldown::Off => return None, + Pulldown::Wwssw => 0, + Pulldown::Wssww => 1, + Pulldown::Sswww => 2, + Pulldown::Swwws => 3, + Pulldown::Wwwss => 4, + }) + } + /// The film frames (counted from the first film frame of the render) of the first and + /// second field of output frame `k`. + pub fn fields(self, k: u64) -> Option<(i64, i64)> { + let r = self.rotation()?; + let film = |f: i64| f.div_euclid(10) * 4 + CADENCE[f.rem_euclid(10) as usize]; + let f0 = 2 * (k as i64 + r); + let base = film(2 * r); + Some((film(f0) - base, film(f0 + 1) - base)) + } + /// The W/S pattern of five consecutive output frames. + pub fn pattern(self) -> String { + (0..5).filter_map(|k| self.fields(k).map(|(a, b)| if a == b { 'W' } else { 'S' })).collect() + } +} + +/// A layer switch override (Render Settings ▸ Frame Blending / Motion Blur). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SwitchOverride { + /// The layer switches and the composition's enable switch. + #[default] + Current, + /// Every layer whose switch is on, whatever the composition's enable switch. + OnForChecked, + OffForAll, +} + +impl SwitchOverride { + pub const ALL: [SwitchOverride; 3] = [SwitchOverride::Current, SwitchOverride::OnForChecked, SwitchOverride::OffForAll]; + pub fn label(self) -> &'static str { + match self { + SwitchOverride::Current => "Current Settings", + SwitchOverride::OnForChecked => "On for Checked Layers", + SwitchOverride::OffForAll => "Off for All Layers", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(SwitchOverride::Current), + "on" | "onforchecked" | "onforcheckedlayers" | "true" => Some(SwitchOverride::OnForChecked), + "off" | "offforall" | "offforalllayers" | "false" => Some(SwitchOverride::OffForAll), + _ => None, + } + } +} + +/// Render Settings ▸ Effects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum EffectsMode { + #[default] + Current, + AllOn, + AllOff, +} + +impl EffectsMode { + pub const ALL: [EffectsMode; 3] = [EffectsMode::Current, EffectsMode::AllOn, EffectsMode::AllOff]; + pub fn label(self) -> &'static str { + match self { + EffectsMode::Current => "Current Settings", + EffectsMode::AllOn => "All On", + EffectsMode::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(EffectsMode::Current), + "allon" | "on" => Some(EffectsMode::AllOn), + "alloff" | "off" => Some(EffectsMode::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Solo Switches / Guide Layers: the current switches, or all off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum CurrentOrOff { + #[default] + Current, + AllOff, +} + +impl CurrentOrOff { + pub fn label(self) -> &'static str { + match self { + CurrentOrOff::Current => "Current Settings", + CurrentOrOff::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" | "on" => Some(CurrentOrOff::Current), + "alloff" | "off" => Some(CurrentOrOff::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Color Depth. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorDepth { + /// The project's bit depth. + #[default] + Current, + Bpc8, + Bpc16, + Bpc32, +} + +impl ColorDepth { + pub const ALL: [ColorDepth; 4] = [ColorDepth::Current, ColorDepth::Bpc8, ColorDepth::Bpc16, ColorDepth::Bpc32]; + pub fn label(self) -> &'static str { + match self { + ColorDepth::Current => "Current Settings", + ColorDepth::Bpc8 => "8 bits per channel", + ColorDepth::Bpc16 => "16 bits per channel", + ColorDepth::Bpc32 => "32 bits per channel", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(ColorDepth::Current), + "8" | "8bpc" | "8bitsperchannel" => Some(ColorDepth::Bpc8), + "16" | "16bpc" | "16bitsperchannel" => Some(ColorDepth::Bpc16), + "32" | "32bpc" | "32bitsperchannel" | "float" => Some(ColorDepth::Bpc32), + _ => None, + } + } + /// The depth renders use in a project of depth `project`. + pub fn resolve(self, project: crate::BitDepth) -> crate::BitDepth { + match self { + ColorDepth::Current => project, + ColorDepth::Bpc8 => crate::BitDepth::Bpc8, + ColorDepth::Bpc16 => crate::BitDepth::Bpc16, + ColorDepth::Bpc32 => crate::BitDepth::Bpc32, + } + } +} + +/// How one output frame is sampled in time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameSample { + Progressive(Tick), + /// Two fields woven together: `first` goes to the dominant field. + Fields { + first: Tick, + second: Tick, + upper_first: bool, + }, +} + +fn default_true() -> bool { + true +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct RenderSettings { + /// The template these settings came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub quality: RenderQuality, /// Output scale (1 = Full, 0.5 = Half, 1/3 = Third, 0.25 = Quarter). pub resolution: f64, pub time_span: TimeSpan, - /// `None` = use the comp's frame rate. + /// `None` = use the comp's frame rate (Time Sampling ▸ Use comp's frame rate). pub frame_rate: Option, - /// Motion blur for layers with the switch on (and the comp's Enable Motion Blur). + /// Motion blur allowed at all (false = Off for All Layers; see `motion_blur_mode`). pub motion_blur: bool, + /// Motion Blur: Current Settings / On for Checked Layers / Off for All Layers. + pub motion_blur_mode: SwitchOverride, + /// Frame Blending: Current Settings / On for Checked Layers / Off for All Layers. + pub frame_blending: SwitchOverride, + /// Field Render. + pub field_render: FieldRender, + /// 3:2 Pulldown phase (with field rendering). + pub pulldown: Pulldown, + /// Effects: Current Settings / All On / All Off. + pub effects: EffectsMode, + /// Solo Switches: Current Settings / All Off. + pub solo: CurrentOrOff, + /// Guide Layers: Current Settings / All Off. + pub guide_layers: CurrentOrOff, + /// Color Depth. + pub color_depth: ColorDepth, /// Image sequences: skip frames whose file already exists. pub skip_existing: bool, /// Proxy Use (Best Settings: Use No Proxies). pub proxy_use: ProxyUse, + /// Use Storage Overflow: when the output volume is full, continue in the project's overflow + /// folders. + #[serde(default = "default_true")] + pub storage_overflow: bool, } impl Default for RenderSettings { fn default() -> Self { RenderSettings { + name: String::new(), quality: RenderQuality::Best, resolution: 1.0, time_span: TimeSpan::WorkArea, frame_rate: None, motion_blur: true, + motion_blur_mode: SwitchOverride::Current, + frame_blending: SwitchOverride::Current, + field_render: FieldRender::Off, + pulldown: Pulldown::Off, + effects: EffectsMode::Current, + solo: CurrentOrOff::Current, + guide_layers: CurrentOrOff::AllOff, + color_depth: ColorDepth::Current, skip_existing: false, proxy_use: ProxyUse::UseNone, + storage_overflow: true, } } } impl RenderSettings { + /// Effective Motion Blur override. + pub fn motion_blur_override(&self) -> SwitchOverride { + if self.motion_blur { self.motion_blur_mode } else { SwitchOverride::OffForAll } + } + /// How output frame `i` is sampled (field rendering and 3:2 pulldown). + pub fn sample(&self, comp: &Comp, i: u64) -> FrameSample { + let upper_first = match self.field_render { + FieldRender::Off => return FrameSample::Progressive(self.frame_time(comp, i)), + FieldRender::UpperFirst => true, + FieldRender::LowerFirst => false, + }; + let r = self.rate(comp); + if let Some((a, b)) = self.pulldown.fields(i) { + // Film frames at 4/5 of the output rate, from the first one starting in the span. + let film = FrameRate::new(r.num * 4, r.den * 5); + let f0 = Self::ceil_frame(film, self.span(comp).0); + return FrameSample::Fields { first: film.tick_of(f0 + a), second: film.tick_of(f0 + b), upper_first }; + } + let f = self.first_frame(comp) + i as i64; + let t = r.tick_of(f); + let half = Tick((r.tick_of(f + 1).0 - t.0) / 2); + FrameSample::Fields { first: t, second: Tick(t.0 + half.0), upper_first } + } /// The comp-time span `[start, end)` to render. pub fn span(&self, comp: &Comp) -> (Tick, Tick) { let (a, b) = match self.time_span { @@ -162,11 +453,45 @@ impl RenderSettings { _ => "Custom", } } - /// One-line summary shown next to "Render Settings:" (AE shows the template name). + /// One-line summary shown next to "Render Settings:" (the template name, like AE). pub fn summary(&self) -> String { + if !self.name.is_empty() { + return self.name.clone(); + } let q = if self.quality == RenderQuality::Best { "Best Settings" } else { "Draft Settings" }; format!("{q} · {}", self.resolution_label()) } + /// Every setting as `(label, value)` lines (render logs, the settings dialog). + pub fn describe(&self, comp: Option<&Comp>) -> Vec<(&'static str, String)> { + let span = match self.time_span { + TimeSpan::WorkArea => "Work Area Only".to_string(), + TimeSpan::LengthOfComp => "Length of Comp".to_string(), + TimeSpan::Custom { start, end } => format!("Custom ({:.3}s – {:.3}s)", start.seconds(), end.seconds()), + }; + let rate = match (self.frame_rate, comp) { + (Some(r), _) => format!("{:.3} fps", r.as_f64()), + (None, Some(c)) => format!("Use comp's frame rate ({:.3})", c.frame_rate.as_f64()), + (None, None) => "Use comp's frame rate".into(), + }; + vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Quality", format!("{:?}", self.quality)), + ("Resolution", format!("{} ({:.3})", self.resolution_label(), self.resolution)), + ("Proxy Use", self.proxy_use.label().into()), + ("Effects", self.effects.label().into()), + ("Solo Switches", self.solo.label().into()), + ("Guide Layers", self.guide_layers.label().into()), + ("Color Depth", self.color_depth.label().into()), + ("Frame Blending", self.frame_blending.label().into()), + ("Field Render", self.field_render.label().into()), + ("3:2 Pulldown", self.pulldown.label().into()), + ("Motion Blur", self.motion_blur_override().label().into()), + ("Time Span", span), + ("Frame Rate", rate), + ("Skip Existing Files", if self.skip_existing { "On" } else { "Off" }.into()), + ("Use Storage Overflow", if self.storage_overflow { "On" } else { "Off" }.into()), + ] + } } /// Output Module ▸ Format. @@ -287,8 +612,30 @@ pub enum Channels { /// Opaque, composited over the comp background colour. #[default] Rgb, - /// With straight (unmatted) alpha. + /// With alpha (straight or premultiplied: [`AlphaMode`]). Rgba, + /// The alpha channel alone, as an opaque greyscale matte. + Alpha, +} + +impl Channels { + pub fn label(self) -> &'static str { + match self { + Channels::Rgb => "RGB", + Channels::Rgba => "RGB + Alpha", + Channels::Alpha => "Alpha", + } + } +} + +/// Output Module ▸ Color: how RGB is stored with alpha. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AlphaMode { + /// Unmatted colour. + #[default] + Straight, + /// Colour multiplied by alpha (matted with black). + Premultiplied, } /// Output Module ▸ Audio Output. @@ -301,6 +648,121 @@ pub enum AudioOutput { Off, } +/// Output Module ▸ Audio Output ▸ sample format (PCM outputs: WAV, AIFF, QuickTime). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AudioFormat { + #[default] + S16, + S24, + /// 32-bit float (WAV, QuickTime; AIFF writes 24-bit). + F32, +} + +impl AudioFormat { + pub fn label(self) -> &'static str { + match self { + AudioFormat::S16 => "16 Bit", + AudioFormat::S24 => "24 Bit", + AudioFormat::F32 => "32 Bit Float", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "16" | "16bit" | "s16" => Some(AudioFormat::S16), + "24" | "24bit" | "s24" => Some(AudioFormat::S24), + "32" | "32bit" | "32bitfloat" | "f32" | "float" => Some(AudioFormat::F32), + _ => None, + } + } +} + +/// Output Module ▸ Crop (pixels of the rendered frame; negative values add transparent pixels). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Crop { + pub enabled: bool, + /// Use Region of Interest: crop to `roi` (comp pixels, captured from the viewer's region of + /// interest when the option is turned on) instead of the edge values. + pub use_roi: bool, + pub roi: Option<[f64; 4]>, + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Crop { + /// Edges `(top, left, bottom, right)` in pixels of a `w`×`h` frame rendered at `scale`. + pub fn edges(&self, w: u32, h: u32, scale: f64) -> (i32, i32, i32, i32) { + if !self.enabled { + return (0, 0, 0, 0); + } + if self.use_roi + && let Some([x, y, rw, rh]) = self.roi + { + let l = (x * scale).round() as i32; + let t = (y * scale).round() as i32; + let r = w as i32 - ((x + rw) * scale).round() as i32; + let b = h as i32 - ((y + rh) * scale).round() as i32; + return (t, l, b, r); + } + (self.top, self.left, self.bottom, self.right) + } +} + +/// Output Module ▸ Resize quality. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ResizeQuality { + /// Bilinear. + Low, + /// Bicubic. + #[default] + High, +} + +/// Output Module ▸ Resize. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Resize { + pub enabled: bool, + pub width: u32, + pub height: u32, + /// Lock Aspect Ratio: setting one dimension derives the other from the rendered frame. + pub lock_aspect: bool, + pub quality: ResizeQuality, +} + +impl Default for Resize { + fn default() -> Self { + Resize { enabled: false, width: 1920, height: 1080, lock_aspect: true, quality: ResizeQuality::High } + } +} + +/// Output Module ▸ Resize ▸ Resize to: presets. +pub const RESIZE_PRESETS: [(&str, u32, u32); 10] = [ + ("HDTV 1080", 1920, 1080), + ("HDTV 720", 1280, 720), + ("UHD 4K", 3840, 2160), + ("DCI 2K", 2048, 1080), + ("DCI 4K", 4096, 2160), + ("NTSC DV", 720, 480), + ("PAL D1/DV", 720, 576), + ("Square 1080", 1080, 1080), + ("Vertical 1080×1920", 1080, 1920), + ("Web 640×360", 640, 360), +]; + +/// Output Module ▸ Post-Render Action. +pub fn post_render_parse(s: &str) -> Option { + match norm(s).as_str() { + "none" => Some(PostRenderAction::None), + "import" => Some(PostRenderAction::Import), + "importreplace" | "importandreplace" | "importandreplaceusage" | "importreplaceusage" | "replace" => Some(PostRenderAction::ImportAndReplace), + "setproxy" | "proxy" => Some(PostRenderAction::SetProxy), + _ => None, + } +} + /// ProRes flavour. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum ProResProfile { @@ -341,38 +803,133 @@ impl ProResProfile { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct OutputModule { + /// The template this module came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub format: OutputFormat, pub channels: Channels, + /// Color: straight or premultiplied RGB with alpha. + pub alpha_mode: AlphaMode, /// Output path or template (see the module docs). pub output: String, - /// JPEG quality 1–100. + /// JPEG / WebM quality 1–100. pub quality: u8, - /// H.264 target bitrate. + /// H.264 target bitrate (and WebM's when `webm_bitrate`). pub bitrate_kbps: u32, + /// WebM: rate control by `bitrate_kbps` instead of `quality`. + pub webm_bitrate: bool, + /// Movies: frames between key frames (0 = automatic, two seconds). + pub keyframe_interval: u32, pub prores_profile: ProResProfile, + pub crop: Crop, + pub resize: Resize, pub audio: AudioOutput, pub audio_sample_rate: u32, + /// 1 (mono) or 2 (stereo). + pub audio_channels: u8, + pub audio_format: AudioFormat, /// GIF: loop forever. pub gif_loop: bool, + /// Include Project Link: kept for parity; EffectCraft's writers have no place to store a + /// link back to the project, so it changes nothing in the file. + pub include_project_link: bool, } impl Default for OutputModule { fn default() -> Self { OutputModule { + name: String::new(), format: OutputFormat::H264, channels: Channels::Rgb, + alpha_mode: AlphaMode::Straight, output: DEFAULT_TEMPLATE.into(), quality: 90, bitrate_kbps: 10_000, + webm_bitrate: false, + keyframe_interval: 0, prores_profile: ProResProfile::Hq, + crop: Crop::default(), + resize: Resize::default(), audio: AudioOutput::Auto, audio_sample_rate: 48_000, + audio_channels: 2, + audio_format: AudioFormat::S16, gif_loop: true, + include_project_link: true, } } } impl OutputModule { + /// The size of the frames this module writes for frames rendered at `w`×`h` with render + /// scale `scale`: after Crop and Resize (before codec rounding). + pub fn frame_size(&self, w: u32, h: u32, scale: f64) -> (u32, u32) { + let (t, l, b, r) = self.crop.edges(w, h, scale); + let cw = (w as i32 - l - r).max(1) as u32; + let ch = (h as i32 - t - b).max(1) as u32; + if !self.resize.enabled { + return (cw, ch); + } + let (mut rw, mut rh) = (self.resize.width.max(1), self.resize.height.max(1)); + if self.resize.lock_aspect { + // Width rules; height follows the cropped frame's aspect. + rh = ((rw as f64 * ch as f64 / cw as f64).round() as u32).max(1); + } + rw = rw.min(30_000); + rh = rh.min(30_000); + (rw, rh) + } + /// The written frame size of an item (Render Settings resolution, Crop, Resize, codec + /// rounding). + pub fn output_size(&self, comp: &Comp, settings: &RenderSettings) -> (u32, u32) { + let (w, h) = settings.output_size(comp); + let (w, h) = self.frame_size(w, h, settings.resolution.clamp(0.01, 4.0)); + self.format.coded_size(w, h) + } + /// Every setting as `(label, value)` lines (render logs). + pub fn describe(&self) -> Vec<(&'static str, String)> { + let mut v = vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Format", self.format.label().into()), + ("Channels", self.channels.label().into()), + ("Output To", self.output.clone()), + ]; + if self.channels == Channels::Rgba { + v.insert(3, ("Color", format!("{:?}", self.alpha_mode))); + } + match self.format { + OutputFormat::H264 => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::ProRes => v.push(("Codec", self.prores_profile.label().into())), + OutputFormat::WebM if self.webm_bitrate => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::WebM | OutputFormat::JpegSequence => v.push(("Quality", self.quality.to_string())), + _ => {} + } + if self.crop.enabled { + let c = &self.crop; + v.push(( + "Crop", + if c.use_roi { format!("Region of Interest {:?}", c.roi) } else { format!("T {} L {} B {} R {}", c.top, c.left, c.bottom, c.right) }, + )); + } + if self.resize.enabled { + let r = &self.resize; + v.push(("Resize", format!("{}×{} ({:?} quality{})", r.width, r.height, r.quality, if r.lock_aspect { ", aspect locked" } else { "" }))); + } + if self.format.supports_audio() { + v.push(( + "Audio Output", + format!( + "{:?} · {} Hz · {} · {}", + self.audio, + self.audio_sample_rate, + if self.audio_channels == 1 { "Mono" } else { "Stereo" }, + self.audio_format.label() + ), + )); + } + v.push(("Include Project Link", if self.include_project_link { "On" } else { "Off" }.into())); + v + } pub fn for_format(format: OutputFormat) -> OutputModule { let mut m = OutputModule { format, ..Default::default() }; if format.is_sequence() { @@ -384,7 +941,7 @@ impl OutputModule { pub fn set_format(&mut self, format: OutputFormat) { let old = self.format; self.format = format; - if !format.supports_alpha() { + if !format.supports_alpha() && self.channels == Channels::Rgba { self.channels = Channels::Rgb; } if self.output.contains("[fileExtension]") { @@ -406,10 +963,10 @@ impl OutputModule { } /// One-line summary shown next to "Output Module:". pub fn summary(&self) -> String { - let ch = match self.channels { - Channels::Rgb => "RGB", - Channels::Rgba => "RGB + Alpha", - }; + if !self.name.is_empty() { + return self.name.clone(); + } + let ch = self.channels.label(); match self.format { OutputFormat::H264 => format!("H.264 · {} kbps", self.bitrate_kbps), OutputFormat::ProRes => format!( @@ -478,6 +1035,85 @@ pub struct RenderQueueItem { /// Post-Render Action of the first output module. #[serde(default, skip_serializing_if = "PostRenderAction::is_none")] pub post_render: PostRenderAction, + /// The Log menu: what the render log next to the output records. + #[serde(default, skip_serializing_if = "RenderLog::is_default")] + pub log: RenderLog, +} + +/// Render Queue item ▸ Log. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum RenderLog { + /// A log only when the render fails. + #[default] + ErrorsOnly, + /// Always: the result plus every Render Settings and Output Module setting. + PlusSettings, + /// Also one line per rendered frame. + PlusPerFrameInfo, +} + +impl RenderLog { + pub const ALL: [RenderLog; 3] = [RenderLog::ErrorsOnly, RenderLog::PlusSettings, RenderLog::PlusPerFrameInfo]; + pub fn is_default(&self) -> bool { + *self == RenderLog::ErrorsOnly + } + pub fn label(self) -> &'static str { + match self { + RenderLog::ErrorsOnly => "Errors Only", + RenderLog::PlusSettings => "Plus Settings", + RenderLog::PlusPerFrameInfo => "Plus Per Frame Info", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "errors" | "errorsonly" => Some(RenderLog::ErrorsOnly), + "settings" | "plussettings" => Some(RenderLog::PlusSettings), + "perframe" | "plusperframe" | "plusperframeinfo" | "frames" => Some(RenderLog::PlusPerFrameInfo), + _ => None, + } + } +} + +/// The render log's path for an output: next to it, `_RenderLog.txt` (a sequence's +/// frame-number run is dropped). +pub fn log_path(output: &str) -> String { + let p = std::path::Path::new(output); + let stem = p.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + let mut stem: String = stem.replace("_[#####]", "").replace("[#####]", ""); + if let Some(i) = stem.find('#') { + let n = stem[i..].chars().take_while(|c| *c == '#').count(); + stem.replace_range(i..i + n, ""); + } + let stem = stem.trim_end_matches(['_', '.', ' ']); + let name = format!("{}_RenderLog.txt", if stem.is_empty() { "Render" } else { stem }); + match p.parent().filter(|d| !d.as_os_str().is_empty()) { + Some(d) => d.join(name).to_string_lossy().to_string(), + None => name, + } +} + +/// Storage hook (Render Settings ▸ Use Storage Overflow): whether `bytes` more fit on the volume +/// holding `path`. The desktop app has no quota by default; tests and hosts with quotas supply +/// one. +pub trait StorageQuota: Send + Sync { + fn has_room(&self, path: &str, bytes: u64) -> bool; +} + +/// Project-wide Render Queue preferences. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderQueuePrefs { + /// Notify when the render finishes (a sound and a notification in the desktop app). + pub notify: bool, + /// Storage overflow folders, used in order when the output volume is full. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow_folders: Vec, +} + +impl RenderQueuePrefs { + pub fn is_default(&self) -> bool { + *self == RenderQueuePrefs::default() + } } /// What happens after an item renders (Output Module ▸ Post-Render Action). @@ -521,6 +1157,7 @@ impl RenderQueueItem { last_output: None, extra_outputs: vec![], post_render: PostRenderAction::None, + log: RenderLog::ErrorsOnly, } } /// Every output module, the first one first. @@ -612,6 +1249,90 @@ mod tests { assert_eq!(ProResProfile::from_name("hq"), Some(ProResProfile::Hq)); } + #[test] + fn pulldown_cadence() { + assert_eq!(Pulldown::Wwssw.pattern(), "WWSSW"); + for p in &Pulldown::ALL[1..] { + assert_eq!(p.pattern(), p.label(), "{p:?}"); + } + // 2:3 cadence: ten fields carry four film frames; every output frame starts on film 0. + for p in &Pulldown::ALL[1..] { + let mut fields = vec![]; + for k in 0..10 { + let (a, b) = p.fields(k).unwrap(); + fields.extend([a, b]); + } + assert_eq!(fields[0], 0, "{p:?}"); + assert!(fields.windows(2).all(|w| w[1] == w[0] || w[1] == w[0] + 1), "{p:?}: {fields:?}"); + // 20 fields = 8 film frames' worth of cadence (a phase may start or end mid-frame). + assert!((7..=8).contains(&(*fields.last().unwrap() - fields[0])), "{p:?} {fields:?}"); + // Each film frame appears in 2 or 3 consecutive fields, alternating. + let mut runs = vec![]; + let mut n = 1; + for w in fields.windows(2) { + if w[1] == w[0] { + n += 1; + } else { + runs.push(n); + n = 1; + } + } + assert!(runs[1..].iter().all(|r| *r == 2 || *r == 3), "{p:?}: {runs:?}"); + assert!(runs[1..].windows(2).all(|w| w[0] != w[1]), "2 and 3 alternate: {p:?}: {runs:?}"); + } + assert_eq!(Pulldown::parse("sswww"), Some(Pulldown::Sswww)); + // With field rendering at 29.97, film frames are 23.976 fps. + let c = Comp::new(64, 48, FrameRate::new(30000, 1001), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let film = FrameRate::new(24000, 1001); + match s.sample(&c, 1) { + FrameSample::Fields { first, second, upper_first } => { + assert!(upper_first); + assert_eq!((first, second), (film.tick_of(0), film.tick_of(1)), "WSSWW: frame 1 is split between film frames 0 and 1"); + } + f => panic!("{f:?}"), + } + } + + #[test] + fn field_samples_are_half_a_frame_apart() { + let c = Comp::new(64, 48, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::LowerFirst, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(s.sample(&c, 3), FrameSample::Fields { first: Tick::from_seconds_f64(0.12), second: Tick::from_seconds_f64(0.14), upper_first: false }); + let p = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(p.sample(&c, 3), FrameSample::Progressive(Tick::from_seconds_f64(0.12))); + } + + #[test] + fn crop_and_resize_sizes() { + let c = Comp::new(1920, 1080, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0)); + let rs = RenderSettings::default(); + let mut m = OutputModule::default(); + assert_eq!(m.output_size(&c, &rs), (1920, 1080)); + m.crop = Crop { enabled: true, top: 10, left: 20, bottom: 30, right: 41, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1858, 1040), "H.264 rounds the odd width down"); + m.format = OutputFormat::PngSequence; + assert_eq!(m.output_size(&c, &rs), (1859, 1040)); + // Region of interest, at half resolution. + m.crop = Crop { enabled: true, use_roi: true, roi: Some([100.0, 50.0, 800.0, 400.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..Default::default() }; + assert_eq!(m.output_size(&c, &half), (400, 200)); + // Negative crop pads. + m.crop = Crop { enabled: true, top: -10, bottom: -10, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1920, 1100)); + // Resize: fixed, and with the aspect locked to the (cropped) frame. + m.crop = Crop::default(); + m.resize = Resize { enabled: true, width: 1280, height: 999, lock_aspect: false, quality: ResizeQuality::High }; + assert_eq!(m.output_size(&c, &rs), (1280, 999)); + m.resize.lock_aspect = true; + assert_eq!(m.output_size(&c, &rs), (1280, 720)); + m.crop = Crop { enabled: true, left: 480, right: 480, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1280, 1440), "960×1080 cropped, then 1280 wide at 8:9"); + assert_eq!(log_path("/out/Main_[#####].png"), "/out/Main_RenderLog.txt"); + assert_eq!(log_path("/out/a.mov"), "/out/a_RenderLog.txt"); + assert_eq!(log_path("seq###.tif"), "seq_RenderLog.txt"); + } + #[test] fn spans() { let mut c = Comp::new(100, 50, FrameRate::new(25, 1), Tick::from_seconds_f64(4.0)); diff --git a/crates/project/src/render_templates.rs b/crates/project/src/render_templates.rs new file mode 100644 index 00000000..cf52b000 --- /dev/null +++ b/crates/project/src/render_templates.rs @@ -0,0 +1,323 @@ +//! Render Settings and Output Module templates (Edit ▸ Templates ▸ Render Settings… / Output +//! Module…): the built-in presets, templates saved with the project, and the defaults new +//! Render Queue items, pre-renders and proxies start from. +//! +//! Built-ins always exist; a saved template with a built-in's name overrides it. Templates +//! carry their name in [`RenderSettings::name`] / [`OutputModule::name`], which the Render Queue +//! shows like After Effects does. + +use serde::{Deserialize, Serialize}; + +use crate::render_queue::*; +use effectcraft_time::FrameRate; + +/// A default slot (the Defaults section of the Templates dialogs). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TemplateSlot { + /// New Render Queue items rendering a movie. + Movie, + /// Single-frame renders (Composition ▸ Save Frame As ▸ File…). + Still, + /// Composition ▸ Pre-render…. + PreRender, + /// File ▸ Create Proxy ▸ Movie…. + MovieProxy, + /// File ▸ Create Proxy ▸ Still…. + StillProxy, +} + +impl TemplateSlot { + pub const ALL: [TemplateSlot; 5] = [TemplateSlot::Movie, TemplateSlot::Still, TemplateSlot::PreRender, TemplateSlot::MovieProxy, TemplateSlot::StillProxy]; + pub fn label(self) -> &'static str { + match self { + TemplateSlot::Movie => "Movie Default", + TemplateSlot::Still => "Frame Default", + TemplateSlot::PreRender => "Pre-Render Default", + TemplateSlot::MovieProxy => "Movie Proxy Default", + TemplateSlot::StillProxy => "Still Proxy Default", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "movie" | "moviedefault" => Some(TemplateSlot::Movie), + "still" | "frame" | "framedefault" | "stilldefault" => Some(TemplateSlot::Still), + "prerender" | "prerenderdefault" => Some(TemplateSlot::PreRender), + "movieproxy" | "movieproxydefault" => Some(TemplateSlot::MovieProxy), + "stillproxy" | "stillproxydefault" => Some(TemplateSlot::StillProxy), + _ => None, + } + } +} + +/// Which template list. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum TemplateKind { + RenderSettings, + OutputModule, +} + +impl TemplateKind { + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "rendersettings" | "settings" | "render" => Some(TemplateKind::RenderSettings), + "outputmodule" | "output" | "module" => Some(TemplateKind::OutputModule), + _ => None, + } + } +} + +/// Defaults per slot (template names). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct TemplateDefaults { + pub render_settings: Vec<(TemplateSlot, String)>, + pub output_modules: Vec<(TemplateSlot, String)>, +} + +impl Default for TemplateDefaults { + fn default() -> Self { + use TemplateSlot::*; + let s = |v: &[(TemplateSlot, &str)]| v.iter().map(|(k, n)| (*k, n.to_string())).collect(); + TemplateDefaults { + render_settings: s(&[ + (Movie, "Best Settings"), + (Still, "Current Settings"), + (PreRender, "Best Settings"), + (MovieProxy, "Draft Settings"), + (StillProxy, "Best Settings"), + ]), + output_modules: s(&[ + (Movie, "H.264 - Match Render Settings - 15 Mbps"), + (Still, "PNG Sequence"), + (PreRender, "Lossless with Alpha"), + (MovieProxy, "Lossless with Alpha"), + (StillProxy, "PNG Sequence with Alpha"), + ]), + } + } +} + +/// Templates saved with the project, plus the defaults. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderTemplates { + #[serde(skip_serializing_if = "Vec::is_empty")] + pub render_settings: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub output_modules: Vec, + pub defaults: TemplateDefaults, +} + +/// The built-in Render Settings templates. +pub fn builtin_render_settings() -> Vec { + let best = RenderSettings { + name: "Best Settings".into(), + frame_blending: SwitchOverride::OnForChecked, + motion_blur_mode: SwitchOverride::OnForChecked, + ..RenderSettings::default() + }; + let draft = RenderSettings { + name: "Draft Settings".into(), + quality: RenderQuality::Draft, + resolution: 0.5, + proxy_use: ProxyUse::CurrentSettings, + frame_blending: SwitchOverride::Current, + motion_blur_mode: SwitchOverride::Current, + ..RenderSettings::default() + }; + let dv = + RenderSettings { name: "DV Settings".into(), field_render: FieldRender::LowerFirst, frame_rate: Some(FrameRate::new(30000, 1001)), ..best.clone() }; + let multi = RenderSettings { name: "Multi-Machine Settings".into(), skip_existing: true, ..best.clone() }; + let current = RenderSettings { + name: "Current Settings".into(), + proxy_use: ProxyUse::CurrentSettings, + guide_layers: CurrentOrOff::Current, + ..RenderSettings::default() + }; + vec![best, draft, dv, multi, current] +} + +fn om(name: &str, format: OutputFormat) -> OutputModule { + let mut m = OutputModule::for_format(format); + m.name = name.into(); + m +} + +/// The built-in Output Module templates. +pub fn builtin_output_modules() -> Vec { + let mut v = vec![]; + let mut hq = om("High Quality", OutputFormat::ProRes); + hq.prores_profile = ProResProfile::Hq; + v.push(hq); + let mut hqa = om("High Quality with Alpha", OutputFormat::ProRes); + hqa.prores_profile = ProResProfile::P4444; + hqa.channels = Channels::Rgba; + hqa.alpha_mode = AlphaMode::Premultiplied; + v.push(hqa); + // Lossless: the highest-fidelity movie codec this build writes (ProRes 4444 XQ). + let mut ll = om("Lossless", OutputFormat::ProRes); + ll.prores_profile = ProResProfile::P4444Xq; + v.push(ll); + let mut lla = om("Lossless with Alpha", OutputFormat::ProRes); + lla.prores_profile = ProResProfile::P4444Xq; + lla.channels = Channels::Rgba; + v.push(lla); + for mbps in [5u32, 15, 40] { + let mut h = om(&format!("H.264 - Match Render Settings - {mbps} Mbps"), OutputFormat::H264); + h.bitrate_kbps = mbps * 1000; + v.push(h); + } + let mut alpha = om("Alpha Only", OutputFormat::ProRes); + alpha.channels = Channels::Alpha; + v.push(alpha); + v.push(om("AIFF 48kHz", OutputFormat::Aiff)); + v.push(om("WAV 48kHz", OutputFormat::Wav)); + v.push(om("PNG Sequence", OutputFormat::PngSequence)); + let mut pa = om("PNG Sequence with Alpha", OutputFormat::PngSequence); + pa.channels = Channels::Rgba; + v.push(pa); + let mut ta = om("TIFF Sequence with Alpha", OutputFormat::TiffSequence); + ta.channels = Channels::Rgba; + v.push(ta); + v.push(om("JPEG Sequence", OutputFormat::JpegSequence)); + let mut exr = om("OpenEXR Sequence with Alpha", OutputFormat::ExrSequence); + exr.channels = Channels::Rgba; + v.push(exr); + v.push(om("Animated GIF", OutputFormat::Gif)); + v.push(om("WebM (VP9 + Opus)", OutputFormat::WebM)); + let mut wa = om("WebM with Alpha", OutputFormat::WebM); + wa.channels = Channels::Rgba; + v.push(wa); + v +} + +fn same(a: &str, b: &str) -> bool { + a.trim().eq_ignore_ascii_case(b.trim()) +} + +impl RenderTemplates { + pub fn is_default(&self) -> bool { + *self == RenderTemplates::default() + } + /// Every Render Settings template: built-ins (unless overridden), then saved ones. + pub fn render_settings_list(&self) -> Vec { + let mut v: Vec = + builtin_render_settings().into_iter().filter(|b| !self.render_settings.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.render_settings.iter().cloned()); + v + } + pub fn output_module_list(&self) -> Vec { + let mut v: Vec = builtin_output_modules().into_iter().filter(|b| !self.output_modules.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.output_modules.iter().cloned()); + v + } + pub fn render_settings(&self, name: &str) -> Option { + self.render_settings_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn output_module(&self, name: &str) -> Option { + self.output_module_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn is_builtin(kind: TemplateKind, name: &str) -> bool { + match kind { + TemplateKind::RenderSettings => builtin_render_settings().iter().any(|t| same(&t.name, name)), + TemplateKind::OutputModule => builtin_output_modules().iter().any(|t| same(&t.name, name)), + } + } + /// Save (or replace) a template under `name`. + pub fn save_render_settings(&mut self, name: &str, mut s: RenderSettings) { + s.name = name.trim().to_string(); + self.render_settings.retain(|t| !same(&t.name, name)); + self.render_settings.push(s); + } + pub fn save_output_module(&mut self, name: &str, mut m: OutputModule) { + m.name = name.trim().to_string(); + self.output_modules.retain(|t| !same(&t.name, name)); + self.output_modules.push(m); + } + /// Delete a saved template (a built-in it overrode comes back). Built-ins themselves can't + /// be deleted. + pub fn delete(&mut self, kind: TemplateKind, name: &str) -> Result<(), String> { + let before = (self.render_settings.len(), self.output_modules.len()); + match kind { + TemplateKind::RenderSettings => self.render_settings.retain(|t| !same(&t.name, name)), + TemplateKind::OutputModule => self.output_modules.retain(|t| !same(&t.name, name)), + } + if before != (self.render_settings.len(), self.output_modules.len()) { + return Ok(()); + } + if Self::is_builtin(kind, name) { Err(format!("“{name}” is a built-in template")) } else { Err(format!("no template “{name}”")) } + } + /// The default template name of a slot. + pub fn default_name(&self, kind: TemplateKind, slot: TemplateSlot) -> String { + let list = match kind { + TemplateKind::RenderSettings => &self.defaults.render_settings, + TemplateKind::OutputModule => &self.defaults.output_modules, + }; + list.iter().find(|(s, _)| *s == slot).map(|(_, n)| n.clone()).unwrap_or_else(|| { + let d = TemplateDefaults::default(); + let l = if kind == TemplateKind::RenderSettings { d.render_settings } else { d.output_modules }; + l.into_iter().find(|(s, _)| *s == slot).map(|(_, n)| n).unwrap_or_default() + }) + } + pub fn set_default(&mut self, kind: TemplateKind, slot: TemplateSlot, name: &str) -> Result<(), String> { + let exists = match kind { + TemplateKind::RenderSettings => self.render_settings(name).is_some(), + TemplateKind::OutputModule => self.output_module(name).is_some(), + }; + if !exists { + return Err(format!("no template “{name}”")); + } + let list = match kind { + TemplateKind::RenderSettings => &mut self.defaults.render_settings, + TemplateKind::OutputModule => &mut self.defaults.output_modules, + }; + list.retain(|(s, _)| *s != slot); + list.push((slot, name.trim().to_string())); + Ok(()) + } + /// The Render Settings a slot starts from (a missing template falls back to the built-in + /// default). + pub fn default_render_settings(&self, slot: TemplateSlot) -> RenderSettings { + self.render_settings(&self.default_name(TemplateKind::RenderSettings, slot)).unwrap_or_else(|| builtin_render_settings().remove(0)) + } + pub fn default_output_module(&self, slot: TemplateSlot) -> OutputModule { + self.output_module(&self.default_name(TemplateKind::OutputModule, slot)).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtins_defaults_and_round_trip() { + let mut t = RenderTemplates::default(); + let names: Vec = t.render_settings_list().into_iter().map(|s| s.name).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert_eq!(t.default_render_settings(TemplateSlot::Movie).name, "Best Settings"); + assert_eq!(t.default_output_module(TemplateSlot::PreRender).channels, Channels::Rgba); + let dv = t.render_settings("dv settings").unwrap(); + assert_eq!(dv.field_render, FieldRender::LowerFirst); + // Save a custom template, make it the movie default, round-trip through JSON. + let mut s = RenderSettings { resolution: 0.25, ..Default::default() }; + s.color_depth = ColorDepth::Bpc16; + t.save_render_settings("Quarter 16", s); + t.set_default(TemplateKind::RenderSettings, TemplateSlot::Movie, "Quarter 16").unwrap(); + let mut m = OutputModule::for_format(OutputFormat::WebM); + m.crop = Crop { enabled: true, top: 4, left: 2, ..Default::default() }; + t.save_output_module("Web Crop", m); + let json = serde_json::to_string(&t).unwrap(); + let back: RenderTemplates = serde_json::from_str(&json).unwrap(); + assert_eq!(back, t); + assert_eq!(back.default_render_settings(TemplateSlot::Movie).color_depth, ColorDepth::Bpc16); + assert_eq!(back.output_module("web crop").unwrap().crop.top, 4); + assert!(t.delete(TemplateKind::RenderSettings, "Best Settings").is_err()); + t.delete(TemplateKind::OutputModule, "Web Crop").unwrap(); + assert!(t.output_module("Web Crop").is_none()); + // Overriding a built-in, then deleting the override, restores it. + t.save_output_module("High Quality", OutputModule::for_format(OutputFormat::Gif)); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::Gif); + t.delete(TemplateKind::OutputModule, "High Quality").unwrap(); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::ProRes); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..77dfc39f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -73,6 +73,8 @@ pub enum Dialog { Precompose, /// Layer ▸ Layer Styles ▸ Layer Style Options… LayerStyles, + /// Edit ▸ Templates ▸ Render Settings… / Output Module…. + RenderTemplates, } /// Host hooks provided by the native app (file pickers etc.). diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index eed7e639..18add8dc 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -506,8 +506,18 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu } "app.templates" => { let kind = p.get("kind").and_then(Value::as_str).unwrap_or("renderSettings"); - let title = if kind == "outputModule" { "Output Module Templates" } else { "Render Settings Templates" }; - crate::panels::dialogs::info(app, title, "Render and output templates are managed from the Render Queue panel (Window ▸ Render Queue)."); + crate::panels::rq_templates::open(app, ctx, kind); + Value::Null + } + "renderQueue.notify" => { + // Render finished with Notify on: a toast, the dock/taskbar attention request, and the + // host's system sound (the desktop app handles `renderQueue.notify`). + let msg = p.get("message").and_then(Value::as_str).unwrap_or("Render finished").to_string(); + app.toast = Some((msg, now)); + ctx.send_viewport_cmd(egui::ViewportCommand::RequestUserAttention(egui::UserAttentionType::Informational)); + if let Some(f) = app.hooks.app_action.as_ref() { + f("renderQueue.notify"); + } Value::Null } "app.find" => { diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index cabc8703..8992b69a 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -129,6 +129,7 @@ pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context) { Dialog::LayerStyles => super::layer_styles_dialog::show(app, ctx, &t), Dialog::TrackTarget => super::tracker::target_dialog(app, ctx, &t), Dialog::TrackApply => super::tracker::apply_dialog(app, ctx, &t), + Dialog::RenderTemplates => super::rq_templates::show(app, ctx, &t), } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index d0fb3a3e..740ced10 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -28,6 +28,7 @@ pub mod properties; pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; +pub mod rq_templates; pub mod script_console; pub mod settings; pub mod shortcut_editor; diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index c02090bc..ec0853a1 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -4,10 +4,18 @@ //! Output Module and Output To rows, and a status footer. //! //! Every control dispatches an engine `renderQueue.*` command (so agents can do the same) and -//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, -//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath}`. +//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, `renderQueue.notify`, +//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath,log}`. +//! +//! The Render Settings and Output Module menus list the templates first (Edit ▸ Templates), then +//! every setting; entries needing numbers (custom time span and frame rate, crop, resize, bitrate, +//! key frame interval, Make Template) open a form that runs the same command. -use effectcraft_engine::project::render_queue::{AudioOutput, Channels, OutputFormat, ProResProfile, RenderQuality, RenderQueueItem, RenderStatus, TimeSpan}; +use effectcraft_engine::project::render_queue::{ + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, EffectsMode, FieldRender, OutputFormat, PostRenderAction, ProResProfile, Pulldown, + RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, +}; +use effectcraft_engine::project::render_templates::RenderTemplates; use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::{Value, json}; @@ -53,42 +61,82 @@ fn status_color(t: &Tokens, s: &RenderStatus) -> Color32 { } } -/// Render Settings menu: (label, params). -fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { +fn mark(on: bool, l: &str) -> String { + if on { format!("✓ {l}") } else { format!(" {l}") } +} + +/// Render Settings menu: (label, params). `{"form": …}` entries open a form. +fn settings_menu(it: &RenderQueueItem, templates: &RenderTemplates) -> Vec<(String, Value)> { let s = &it.settings; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; - let mut v = vec![ - (mark(s.quality == RenderQuality::Best, "Best Settings"), json!({"quality": "best"})), - (mark(s.quality == RenderQuality::Draft, "Draft Settings"), json!({"quality": "draft"})), - ("-".into(), Value::Null), - ]; + let mut v = vec![]; + for t in templates.render_settings_list() { + v.push((mark(s.name == t.name, &t.name), json!({"template": t.name}))); + } + v.push((" Make Template…".into(), json!({"form": "makeRenderTemplate"}))); + v.push(("-".into(), Value::Null)); + v.push((mark(s.quality == RenderQuality::Best, "Quality: Best"), json!({"quality": "best"}))); + v.push((mark(s.quality == RenderQuality::Draft, "Quality: Draft"), json!({"quality": "draft"}))); for (l, k) in [("Full", "full"), ("Half", "half"), ("Third", "third"), ("Quarter", "quarter")] { v.push((mark(s.resolution_label() == l, &format!("Resolution: {l}")), json!({"resolution": k}))); } v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } + for (e, k) in EffectsMode::ALL.iter().zip(["current", "allOn", "allOff"]) { + v.push((mark(s.effects == *e, &format!("Effects: {}", e.label())), json!({"effects": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.solo == e, &format!("Solo Switches: {}", e.label())), json!({"solo": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.guide_layers == e, &format!("Guide Layers: {}", e.label())), json!({"guideLayers": k}))); + } + for (d, k) in ColorDepth::ALL.iter().zip(["current", "8", "16", "32"]) { + v.push((mark(s.color_depth == *d, &format!("Color Depth: {}", d.label())), json!({"colorDepth": k}))); + } + v.push(("-".into(), Value::Null)); + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.frame_blending == *m, &format!("Frame Blending: {}", m.label())), json!({"frameBlending": k}))); + } + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.motion_blur_override() == *m, &format!("Motion Blur: {}", m.label())), json!({"motionBlur": k}))); + } + for (f, k) in FieldRender::ALL.iter().zip(["off", "upper", "lower"]) { + v.push((mark(s.field_render == *f, &format!("Field Render: {}", f.label())), json!({"fieldRender": k}))); + } + if s.field_render != FieldRender::Off { + for pd in Pulldown::ALL { + v.push((mark(s.pulldown == pd, &format!("3:2 Pulldown: {}", pd.label())), json!({"pulldown": pd.label()}))); + } + } + v.push(("-".into(), Value::Null)); v.push((mark(s.time_span == TimeSpan::WorkArea, "Time Span: Work Area Only"), json!({"timeSpan": "workArea"}))); v.push((mark(s.time_span == TimeSpan::LengthOfComp, "Time Span: Length of Comp"), json!({"timeSpan": "comp"}))); - v.push(("-".into(), Value::Null)); + v.push((mark(matches!(s.time_span, TimeSpan::Custom { .. }), "Time Span: Custom…"), json!({"form": "timeSpan"}))); v.push((mark(s.frame_rate.is_none(), "Frame Rate: Use comp's frame rate"), json!({"frameRate": null}))); for fps in [12.0, 15.0, 23.976, 24.0, 25.0, 29.97, 30.0, 50.0, 60.0] { let on = s.frame_rate.is_some_and(|r| (r.as_f64() - fps).abs() < 0.01); v.push((mark(on, &format!("Frame Rate: {fps}")), json!({"frameRate": fps}))); } + v.push((" Frame Rate: Custom…".into(), json!({"form": "frameRate"}))); v.push(("-".into(), Value::Null)); - v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); - v.push(("-".into(), Value::Null)); - for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { - v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); - } + v.push((mark(s.storage_overflow, "Use Storage Overflow"), json!({"storageOverflow": !s.storage_overflow}))); v } -/// Output Module menu: (label, params). -fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, Value)> { +/// Output Module menu: (label, params). `{"form": …}` entries open a form. +fn output_menu(it: &RenderQueueItem, available: &[OutputFormat], templates: &RenderTemplates) -> Vec<(String, Value)> { let o = &it.output; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; let mut v = Vec::new(); + for t in templates.output_module_list() { + if available.contains(&t.format) { + v.push((mark(o.name == t.name, &t.name), json!({"template": t.name}))); + } + } + v.push((" Make Template…".into(), json!({"form": "makeOutputTemplate"}))); + v.push(("-".into(), Value::Null)); for f in OutputFormat::ALL { if available.contains(&f) { v.push((mark(o.format == f, &format!("Format: {}", f.label())), json!({"format": format!("{f:?}")}))); @@ -99,6 +147,13 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, if o.format.supports_alpha() { v.push((mark(o.channels == Channels::Rgba, "Channels: RGB + Alpha"), json!({"channels": "rgba"}))); } + if !o.format.is_audio_only() { + v.push((mark(o.channels == Channels::Alpha, "Channels: Alpha"), json!({"channels": "alpha"}))); + } + if o.channels == Channels::Rgba { + v.push((mark(o.alpha_mode == AlphaMode::Straight, "Color: Straight (Unmatted)"), json!({"color": "straight"}))); + v.push((mark(o.alpha_mode == AlphaMode::Premultiplied, "Color: Premultiplied (Matted)"), json!({"color": "premultiplied"}))); + } match o.format { OutputFormat::ProRes => { v.push(("-".into(), Value::Null)); @@ -108,14 +163,18 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } OutputFormat::H264 => { v.push(("-".into(), Value::Null)); - for kbps in [2_000, 5_000, 10_000, 20_000, 40_000] { + for kbps in [2_000, 5_000, 10_000, 15_000, 20_000, 40_000] { v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { - v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); + v.push((mark(o.quality == q && !o.webm_bitrate, &format!("Quality: {q}")), json!({"quality": q, "webmBitrate": false}))); + } + if o.format == OutputFormat::WebM { + v.push((mark(o.webm_bitrate, &format!("Target Bitrate ({} kbps)…", o.bitrate_kbps)), json!({"form": "bitrate"}))); + v.push((mark(o.keyframe_interval > 0, "Key Frame Interval…"), json!({"form": "keyframes"}))); } } OutputFormat::Gif => { @@ -124,20 +183,131 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } _ => {} } + if !o.format.is_audio_only() { + v.push(("-".into(), Value::Null)); + v.push((mark(!o.crop.enabled, "Crop: Off"), json!({"crop": false}))); + v.push((mark(o.crop.enabled && o.crop.use_roi, "Crop: Use Region of Interest"), json!({"crop": {"useRoi": true}}))); + v.push((mark(o.crop.enabled && !o.crop.use_roi, "Crop: Edges…"), json!({"form": "crop"}))); + v.push((mark(!o.resize.enabled, "Resize: Off"), json!({"resize": false}))); + for (n, w, h) in RESIZE_PRESETS { + let on = o.resize.enabled && o.resize.width == w && o.resize.height == h && !o.resize.lock_aspect; + v.push((mark(on, &format!("Resize to: {n} ({w}×{h})")), json!({"resize": {"preset": n}}))); + } + v.push((mark(o.resize.enabled, "Resize to: Custom…"), json!({"form": "resize"}))); + v.push(( + mark(o.resize.lock_aspect, "Resize: Lock Aspect Ratio"), + json!({"resize": {"enabled": o.resize.enabled, "lockAspect": !o.resize.lock_aspect}}), + )); + for (q, k) in [(ResizeQuality::Low, "low"), (ResizeQuality::High, "high")] { + v.push((mark(o.resize.quality == q, &format!("Resize Quality: {q:?}")), json!({"resize": {"enabled": o.resize.enabled, "quality": k}}))); + } + } + v.push(("-".into(), Value::Null)); + for (a, k) in [ + (PostRenderAction::None, "none"), + (PostRenderAction::Import, "import"), + (PostRenderAction::ImportAndReplace, "importAndReplace"), + (PostRenderAction::SetProxy, "setProxy"), + ] { + v.push((mark(it.post_render == a, &format!("Post-Render Action: {}", a.label())), json!({"postRenderAction": k}))); + } + v.push((mark(o.include_project_link, "Include Project Link"), json!({"includeProjectLink": !o.include_project_link}))); if o.format.supports_audio() { v.push(("-".into(), Value::Null)); for (a, l) in [(AudioOutput::Auto, "auto"), (AudioOutput::On, "on"), (AudioOutput::Off, "off")] { let label = match a { AudioOutput::Auto => "Audio Output: Auto", - AudioOutput::On => "Audio Output: On (48 kHz stereo)", + AudioOutput::On => "Audio Output: On", AudioOutput::Off => "Audio Output: Off", }; v.push((mark(o.audio == a, label), json!({"audio": l}))); } + for sr in [44_100u32, 48_000, 96_000] { + v.push((mark(o.audio_sample_rate == sr, &format!("Sample Rate: {:.1} kHz", sr as f64 / 1000.0)), json!({"sampleRate": sr}))); + } + v.push((mark(o.audio_channels == 1, "Audio Channels: Mono"), json!({"audioChannels": "mono"}))); + v.push((mark(o.audio_channels != 1, "Audio Channels: Stereo"), json!({"audioChannels": "stereo"}))); + if matches!(o.format, OutputFormat::ProRes | OutputFormat::Wav | OutputFormat::Aiff) { + for f in [AudioFormat::S16, AudioFormat::S24, AudioFormat::F32] { + v.push((mark(o.audio_format == f, &format!("Sample Format: {}", f.label())), json!({"audioFormat": f.label()}))); + } + } } v } +/// Open the form behind a `{"form": …}` menu entry. +fn open_menu_form(app: &mut EffectcraftApp, it: &RenderQueueItem, form: &str) { + use crate::panels::forms::{Field, form as open}; + let id = it.id; + let o = &it.output; + match form { + "makeRenderTemplate" => open( + app, + "Make Render Settings Template", + "renderQueue.saveTemplate", + json!({"kind": "renderSettings", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "makeOutputTemplate" => open( + app, + "Make Output Module Template", + "renderQueue.saveTemplate", + json!({"kind": "outputModule", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "timeSpan" => { + let comp = app.session.project.comp(it.comp); + let (a, b) = comp.map(|c| it.settings.span(c)).unwrap_or_default(); + open( + app, + "Custom Time Span", + "renderQueue.setRenderSettings", + json!({"item": id, "timeSpan": "custom"}), + vec![Field::num("start", "Start (s)", a.seconds()), Field::num("duration", "Duration (s)", (b.seconds() - a.seconds()).max(0.04))], + ) + } + "frameRate" => { + let r = app.session.project.comp(it.comp).map(|c| it.settings.rate(c).as_f64()).unwrap_or(30.0); + open(app, "Use This Frame Rate", "renderQueue.setRenderSettings", json!({"item": id}), vec![Field::num("frameRate", "Frames per second", r)]) + } + "crop" => open( + app, + "Crop", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![ + Field::num("cropTop", "Top", o.crop.top as f64), + Field::num("cropLeft", "Left", o.crop.left as f64), + Field::num("cropBottom", "Bottom", o.crop.bottom as f64), + Field::num("cropRight", "Right", o.crop.right as f64), + ], + ), + "resize" => open( + app, + "Resize to", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("resizeWidth", "Width", o.resize.width as f64), Field::num("resizeHeight", "Height", o.resize.height as f64)], + ), + "bitrate" => open( + app, + "WebM Target Bitrate", + "renderQueue.setOutputModule", + json!({"item": id, "webmBitrate": true}), + vec![Field::num("bitrate", "Bitrate (kbps)", o.bitrate_kbps as f64)], + ), + "keyframes" => open( + app, + "Key Frame Interval", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("keyframeInterval", "Frames between key frames (0 = every 2 s)", o.keyframe_interval as f64)], + ), + _ => {} + } +} + const OUTPUT_TEMPLATES: [&str; 4] = [ "[compName].[fileExtension]", "[compName]_[width]x[height].[fileExtension]", @@ -166,6 +336,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let rendering = app.session.is_rendering(); let available = app.session.exporter.as_ref().map(|e| e.formats()).unwrap_or_default(); let mut actions: Vec<(&'static str, Value)> = Vec::new(); + let mut forms: Vec<(RenderQueueItem, String)> = Vec::new(); let state_id = egui::Id::new("rq-ui"); let (mut open, mut selected): (Vec, Option) = ui.data(|d| d.get_temp(state_id).unwrap_or_default()); @@ -194,6 +365,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("renderQueue.render", json!({"wait": false}))); } app.auto.add("renderQueue.render", render_b, "Render"); + // Notify (sound + notification when the queue finishes). + let notify = app.session.project.render_prefs.notify; + let nb = Rect::from_min_size(pos2(render_b.min.x - 8.0 - 2.0 * bw - 8.0, render_b.min.y + 4.0), vec2(18.0, 18.0)); + if widgets::checkbox(ui, nb, notify, &t, egui::Id::new("rq-notify")).clicked() { + actions.push(("renderQueue.setNotify", json!({"notify": !notify}))); + } + p.text(pos2(nb.max.x + 5.0, nb.center().y), Align2::LEFT_CENTER, "Notify", Tokens::ui(11.5), t.text_dim); + app.auto.add("renderQueue.notify", nb, if notify { "Notify: on" } else { "Notify: off" }); if rendering { if widgets::text_button(ui, stop_b, "Stop", false, &t, egui::Id::new("rq-stop")).clicked() { actions.push(("renderQueue.stop", json!({}))); @@ -201,8 +380,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add("renderQueue.stop", stop_b, "Stop"); } // Progress bar. - let bar = - Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(if rendering { stop_b.min.x - 16.0 } else { render_b.min.x - 16.0 }, render_b.max.y - 4.0)); + let bar = Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(nb.min.x - 16.0, render_b.max.y - 4.0)); p.rect_filled(bar, 3.0, t.field_bg); p.rect_stroke(bar, 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); let frac = progress.as_ref().filter(|p| p.total > 0).map(|p| (p.done as f32 / p.total as f32).clamp(0.0, 1.0)).unwrap_or(0.0); @@ -355,22 +533,28 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_size(pos2(list.min.x, y), vec2(list.width(), ROW_H)); y += ROW_H; lp.rect_filled(r, 0.0, t.panel_bg); + let templates = &app.session.project.render_templates; let (label, summary, key, menu): (&str, String, &str, Vec<(String, Value)>) = if sub == 0 { - ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it)) + ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it, templates)) } else { - ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available)) + ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available, templates)) }; lp.text(pos2(cx_num - 6.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(11.5), t.text_dim); let dd = Rect::from_min_size(pos2(ix + 70.0, r.min.y + 3.0), vec2(220.0, ROW_H - 6.0)); let did = egui::Id::new(("rq-dd", it.id, sub)); - if widgets::dropdown(ui, dd, &summary, &t, did).clicked() && !rendering { + let shown: String = if summary.chars().count() > 31 { summary.chars().take(30).chain(std::iter::once('…')).collect() } else { summary.clone() }; + if widgets::dropdown(ui, dd, &shown, &t, did).clicked() && !rendering { widgets::open_popup(ui, did); } let labels: Vec = menu.iter().map(|(l, _)| l.clone()).collect(); if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &labels, None) { let mut params = menu[i].1.clone(); - params["item"] = json!(it.id); - actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + if let Some(f) = params.get("form").and_then(Value::as_str) { + forms.push((it.clone(), f.to_string())); + } else { + params["item"] = json!(it.id); + actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + } } app.auto.add(&aid(key), dd, &summary); if sub == 0 { @@ -387,6 +571,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ); lp.text(pos2(dd.max.x + 14.0, r.center().y), Align2::LEFT_CENTER, info, Tokens::ui(11.0), t.text_faint); } + // Log: Errors Only / Plus Settings / Plus Per Frame Info. + let ld = Rect::from_min_size(pos2(r.max.x - 190.0, r.min.y + 3.0), vec2(150.0, ROW_H - 6.0)); + lp.text(pos2(ld.min.x - 6.0, r.center().y), Align2::RIGHT_CENTER, "Log:", Tokens::ui(11.5), t.text_dim); + let lid = egui::Id::new(("rq-log", it.id)); + if widgets::dropdown(ui, ld, it.log.label(), &t, lid).clicked() && !rendering { + widgets::open_popup(ui, lid); + } + let llabels: Vec = RenderLog::ALL.iter().map(|l| mark(it.log == *l, l.label())).collect(); + if let Some(i) = widgets::popup_menu(ui, lid, ld.left_bottom(), &llabels, None) { + actions.push(("renderQueue.setLog", json!({"item": it.id, "log": RenderLog::ALL[i].label()}))); + } + app.auto.add(&aid("log"), ld, it.log.label()); continue; } // Output To. @@ -490,6 +686,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { open.retain(|id| queue.iter().any(|i| i.id == *id)); ui.data_mut(|d| d.insert_temp(state_id, (open, selected))); let ctx = ui.ctx().clone(); + for (it, f) in forms { + open_menu_form(app, &it, &f); + } for (id, params) in actions { if let Err(e) = crate::menus::invoke(app, &ctx, id, params) { app.ui.status = e; diff --git a/crates/ui-egui/src/panels/rq_templates.rs b/crates/ui-egui/src/panels/rq_templates.rs new file mode 100644 index 00000000..c60c3009 --- /dev/null +++ b/crates/ui-egui/src/panels/rq_templates.rs @@ -0,0 +1,166 @@ +//! Edit ▸ Templates ▸ Render Settings… / Output Module…: the Defaults (which template new +//! Render Queue items, frames, pre-renders and proxies start from) and the template list +//! (settings of the selected template; Duplicate, Delete, New from a Render Queue item). +//! +//! Every action runs a `renderQueue.*Template*` engine command. Automation ids: +//! `templates.default.`, `templates.select`, `templates.duplicate`, `templates.delete`, +//! `templates.newName`, `templates.saveFromItem`, `templates.ok`. + +use effectcraft_engine::project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; +use egui::{Color32, vec2}; +use serde_json::{Value, json}; + +use crate::theme::Tokens; +use crate::{Dialog, EffectcraftApp}; + +#[derive(Clone, Default)] +struct State { + output: bool, + selected: String, + new_name: String, +} + +fn state_id() -> egui::Id { + egui::Id::new("rq-templates-dialog") +} + +/// Open the dialog for `kind` (`renderSettings` | `outputModule`). +pub fn open(app: &mut EffectcraftApp, ctx: &egui::Context, kind: &str) { + let output = TemplateKind::parse(kind) == Some(TemplateKind::OutputModule); + let t = &app.session.project.render_templates; + let k = if output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let selected = t.default_name(k, TemplateSlot::Movie); + ctx.data_mut(|d| d.insert_temp(state_id(), State { output, selected, new_name: String::new() })); + app.dialog = Some(Dialog::RenderTemplates); +} + +fn names(t: &RenderTemplates, output: bool) -> Vec { + if output { t.output_module_list().into_iter().map(|m| m.name).collect() } else { t.render_settings_list().into_iter().map(|r| r.name).collect() } +} + +pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut st: State = ctx.data(|d| d.get_temp(state_id())).unwrap_or_default(); + let kind = if st.output { "outputModule" } else { "renderSettings" }; + let k = if st.output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let templates = app.session.project.render_templates.clone(); + let list = names(&templates, st.output); + if !list.contains(&st.selected) { + st.selected = list.first().cloned().unwrap_or_default(); + } + let mut actions: Vec<(&'static str, Value)> = vec![]; + let mut close = false; + let title = if st.output { "Output Module Templates" } else { "Render Settings Templates" }; + super::dialogs::modal(ctx, title, vec2(600.0, 560.0), t, |ui| { + ui.label(egui::RichText::new("Defaults").font(Tokens::semibold(12.5)).color(t.text)); + egui::Grid::new("tpl-defaults").num_columns(2).spacing(vec2(14.0, 6.0)).show(ui, |ui| { + for slot in TemplateSlot::ALL { + ui.label(slot.label()); + let cur = templates.default_name(k, slot); + let mut pick = cur.clone(); + let r = egui::ComboBox::from_id_salt(("tpl-def", slot.label())).selected_text(&cur).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut pick, n.clone(), n); + } + }); + app.auto.add(&format!("templates.default.{slot:?}"), r.response.rect, &cur); + if pick != cur { + actions.push(("renderQueue.setTemplateDefault", json!({"kind": kind, "slot": format!("{slot:?}"), "name": pick}))); + } + ui.end_row(); + } + }); + ui.add_space(12.0); + ui.separator(); + ui.label(egui::RichText::new("Settings").font(Tokens::semibold(12.5)).color(t.text)); + ui.horizontal(|ui| { + ui.label("Settings Name:"); + let r = egui::ComboBox::from_id_salt("tpl-select").selected_text(&st.selected).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut st.selected, n.clone(), n); + } + }); + app.auto.add("templates.select", r.response.rect, &st.selected); + }); + let builtin = RenderTemplates::is_builtin(k, &st.selected) + && !match k { + TemplateKind::RenderSettings => templates.render_settings.iter().any(|r| r.name.eq_ignore_ascii_case(&st.selected)), + TemplateKind::OutputModule => templates.output_modules.iter().any(|m| m.name.eq_ignore_ascii_case(&st.selected)), + }; + // The selected template's settings. + let lines: Vec<(&'static str, String)> = if st.output { + templates.output_module(&st.selected).map(|m| m.describe()).unwrap_or_default() + } else { + templates.render_settings(&st.selected).map(|r| r.describe(None)).unwrap_or_default() + }; + egui::Frame::new().fill(t.field_bg).inner_margin(egui::Margin::same(8)).show(ui, |ui| { + ui.set_min_height(180.0); + ui.set_width(ui.available_width()); + egui::ScrollArea::vertical().max_height(200.0).show(ui, |ui| { + egui::Grid::new("tpl-lines").num_columns(2).spacing(vec2(12.0, 2.0)).show(ui, |ui| { + for (k, v) in &lines { + ui.label(egui::RichText::new(*k).color(t.text_dim).size(11.5)); + ui.label(egui::RichText::new(v).size(11.5)); + ui.end_row(); + } + }); + }); + }); + ui.horizontal(|ui| { + let r = ui.button("Duplicate"); + app.auto.add("templates.duplicate", r.rect, "Duplicate"); + if r.clicked() { + let mut n = format!("{} copy", st.selected); + let mut i = 2; + while list.iter().any(|x| x.eq_ignore_ascii_case(&n)) { + n = format!("{} copy {i}", st.selected); + i += 1; + } + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": n, "from": st.selected}))); + st.selected = n; + } + let r = ui.add_enabled(!builtin, egui::Button::new("Delete")); + app.auto.add("templates.delete", r.rect, "Delete"); + if r.clicked() { + actions.push(("renderQueue.deleteTemplate", json!({"kind": kind, "name": st.selected}))); + } + if builtin { + ui.label(egui::RichText::new("built-in").color(t.text_faint).size(11.0)); + } + }); + ui.add_space(8.0); + ui.separator(); + // New template from the selected (or only) Render Queue item. + let queue = &app.session.project.render_queue; + ui.horizontal(|ui| { + ui.label("New from Render Queue item:"); + let r = ui.add(egui::TextEdit::singleline(&mut st.new_name).hint_text("template name").desired_width(180.0)); + app.auto.add("templates.newName", r.rect, &st.new_name); + let item = queue.last().map(|i| i.id); + let ok = item.is_some() && !st.new_name.trim().is_empty(); + let r = ui.add_enabled(ok, egui::Button::new(format!("Save from #{}", queue.len().max(1)))); + app.auto.add("templates.saveFromItem", r.rect, "Save from item"); + if r.clicked() + && let Some(id) = item + { + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": st.new_name.trim(), "item": id}))); + st.selected = st.new_name.trim().to_string(); + st.new_name.clear(); + } + }); + ui.add_space(12.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let r = ui.add(egui::Button::new(egui::RichText::new(" OK ").color(Color32::WHITE)).fill(t.accent)); + app.auto.add("templates.ok", r.rect, "OK"); + close = r.clicked(); + }); + }); + for (id, p) in actions { + if let Err(e) = app.session.execute(id, p) { + app.ui.status = e.to_string(); + } + } + ctx.data_mut(|d| d.insert_temp(state_id(), st)); + if close || ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + app.dialog = None; + } +} diff --git a/crates/ui-egui/tests/render_queue_snapshot.rs b/crates/ui-egui/tests/render_queue_snapshot.rs index 5029a5f2..41ce13d2 100644 --- a/crates/ui-egui/tests/render_queue_snapshot.rs +++ b/crates/ui-egui/tests/render_queue_snapshot.rs @@ -41,3 +41,26 @@ fn render_queue_panel_snapshot() { } img.save(&out).unwrap(); } + +/// Edit ▸ Templates ▸ Output Module… (`RQ_TEMPLATES_SNAPSHOT=/abs/out.png`). +#[test] +#[ignore] +fn templates_dialog_snapshot() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("renderQueue.add", json!({"output": "/renders/[compName].[fileExtension]"})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Web Delivery", "item": s.project.render_queue[0].id, "params": {"format": "webm", "resize": {"preset": "HDTV 720"}}})) + .unwrap(); + let kind = std::env::var("RQ_TEMPLATES_KIND").unwrap_or_else(|_| "outputModule".into()); + s.execute("app.templates", json!({"kind": kind})).unwrap(); + let mut app = Some(EffectcraftApp::new(s)); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app.take().expect("app")); + h.run_steps(4); + assert!(h.state().dialog.is_some(), "the Templates dialog is open"); + let img = h.render().expect("render"); + let out = std::env::var("RQ_TEMPLATES_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out/rq_templates.png").into()); + if let Some(d) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(d).unwrap(); + } + img.save(&out).unwrap(); +} diff --git a/docs/parity.md b/docs/parity.md index 8fc67b9a..2b30c576 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -21,7 +21,7 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU -rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +rendering of 3D runs and adjustment layers; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and @@ -110,7 +110,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 key + inter frames with motion search, loop filter and rate control; Opus CELT-only) | +| Output | 97% | 0.8 | Render Settings complete (field render + 3:2 pulldown, effects/solo/guide/depth/blending/blur overrides, time sampling, storage overflow), Output Module crop/ROI/resize, alpha modes, post-render actions, PCM formats, templates with defaults, render logs, Notify (M10.2); WebM VP9 key + inter frames with motion search, loop filter and rate control. Left: Opus is CELT-only, Photoshop sequence output, overflow for movies only checks at file creation | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | diff --git a/docs/web.md b/docs/web.md index 041ef76c..72eb75ff 100644 --- a/docs/web.md +++ b/docs/web.md @@ -207,5 +207,3 @@ every request and reply, a render through an in-process offload). - Cancelling an analysis drops its partial result (the worker is terminated). - Stored media count against the origin's storage quota; `effectcraft.listStored()` shows usage, `removeStored` frees it (there is no storage manager in the UI yet). -- `engine.execute` rejects the spread Render Settings / Output Module keys of `renderQueue.add` - (`resolution`, `timeSpan`…) through its strict parameter check; use `ui.menu.invoke` as above. From baf4716246077d8f78303657d011d5fdfdd9ac18 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:14:12 -0400 Subject: [PATCH 32/36] M13.5: extended viewer (custom 3D views / Draft 3D render the pasteboard around the comp frame; view.extendedViewer) Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/view.rs | 13 ++++++ crates/engine/src/menus.rs | 1 + crates/render/src/lib.rs | 57 ++++++++++++++++++++--- crates/render/src/tests_roi.rs | 30 ++++++++++++ crates/render/src/three_d/compose.rs | 10 ++-- crates/ui-egui/src/icons.rs | 8 ++++ crates/ui-egui/src/lib.rs | 14 +++++- crates/ui-egui/src/panels/viewer.rs | 41 +++++++++++++++- crates/ui-egui/src/panels/viewer_tools.rs | 8 +++- crates/ui-egui/src/state.rs | 5 ++ crates/ui-egui/tests/ui_3d.rs | 48 +++++++++++++++++++ 11 files changed, 219 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/commands/view.rs b/crates/engine/src/commands/view.rs index b3d86565..4cb001f6 100644 --- a/crates/engine/src/commands/view.rs +++ b/crates/engine/src/commands/view.rs @@ -89,6 +89,18 @@ fn share_view_options(s: &mut Session, p: &Value) -> Result { Ok(json!(s.state.share_view_options)) } +/// Settings ▸ 3D ▸ Extended Viewer: 3D views (custom views, or Draft 3D) show the pasteboard +/// around the comp frame with the layers that reach onto it. +fn extended_viewer(s: &mut Session, p: &Value) -> Result { + let v = super::b_p(p, "value").unwrap_or(!s.prefs.three_d.extended_viewer); + if v != s.prefs.three_d.extended_viewer { + s.prefs.three_d.extended_viewer = v; + s.prefs_changed(); + s.save_prefs(); + } + Ok(json!(v)) +} + pub fn specs() -> Vec { vec![ cmd!("view.addGuide", "Add Guide...", ["View"], None, "{orientation?: vertical|horizontal, position? (comp px)}", has_comp, add_guide), @@ -97,6 +109,7 @@ pub fn specs() -> Vec { cmd!("view.exportGuides", "Export Guides...", ["View"], None, "{path}", has_guides, export_guides), cmd!("view.layout", "Switch View Layout", [], None, "{views: 1|2|4}", has_comp, view_layout), cmd!("view.shareViewOptions", "Share View Options", ["View", "Switch View Layout"], None, "{value?}", has_comp, share_view_options), + cmd!("view.extendedViewer", "Extended Viewer", [], None, "{value?}", always, extended_viewer), cmd!("view.setRegionOfInterest", "Region of Interest", [], None, "{rect?: [x, y, w, h] | null}", always, region_of_interest), ] } diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 9ab366db..fce44a90 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -301,6 +301,7 @@ pub fn checked(s: &Session, command: &str, params: &Value) -> Option { "layer.mask.hideLocked" => Some(s.state.hide_locked_masks), "view.layout" => Some(params.get("views").and_then(Value::as_u64) == Some(s.state.view_layout.max(1) as u64)), "view.shareViewOptions" => Some(s.state.share_view_options), + "view.extendedViewer" => Some(s.prefs.three_d.extended_viewer), "view.snapping" => Some(s.state.snapping), "view.displayColorManagement" => Some(s.state.viewer.display_color_management), "view.simulateOutput" => { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 6ede490b..2f468782 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -363,6 +363,9 @@ pub struct Renderer<'a> { /// Switches inherited from the precomp layers this comp is rendered through (see /// [`RenderOpts::nested_switches`]): the worst quality and whether motion blur is allowed. pub(crate) inherited: Option<(Quality, bool)>, + /// This renderer draws into the top-level frame's canvas (the top comp, or a collapsed + /// precomp drawn straight into it): the region of interest's offset applies. + top: bool, } /// How a collapsed precomp's 3D layers are placed in the parent (see `Renderer::collapse_into`). @@ -407,6 +410,7 @@ impl<'a> Renderer<'a> { opacity_mul: 1.0, collapse3d: None, inherited: None, + top: true, } } @@ -441,7 +445,7 @@ impl<'a> Renderer<'a> { /// A renderer for nested work (precomps, layers read by effects): one level deeper. fn nested(&self) -> Renderer<'a> { - Renderer { depth: self.depth + 1, outer: None, opacity_mul: 1.0, collapse3d: None, ..*self } + Renderer { depth: self.depth + 1, outer: None, opacity_mul: 1.0, collapse3d: None, top: false, ..*self } } /// The nested comp of a precomp layer whose transformations collapse into this comp: the @@ -604,18 +608,28 @@ impl<'a> Renderer<'a> { let s = self.opts.scale; let w = ((comp.width as f64 * s).round() as u32).max(1); let h = ((comp.height as f64 * s).round() as u32).max(1); - // Region of interest on a comp with 3D layers: render the frame and crop it. + // Region of interest on a comp with 3D layers: render the frame and crop it. A region + // reaching past the comp frame (the Extended Viewer) renders Classic 3D natively — the + // planes project through the offset camera — and Advanced 3D as the frame, padded. if self.depth == 0 && let Some(r) = self.opts.roi && comp.has_3d() + && (roi_inside(r, comp.width, comp.height) || comp.renderer == effectcraft_project::Renderer::Advanced3D) { let full = Renderer { opts: RenderOpts { roi: None, ..self.opts }, ..*self }.comp_frame(comp_id, t); - let (x0, y0) = ((r[0] * s).round().max(0.0) as u32, (r[1] * s).round().max(0.0) as u32); + let (x0, y0) = ((r[0] * s).round() as i64, (r[1] * s).round() as i64); let (rw, rh) = (((r[2] * s).round() as u32).max(1), ((r[3] * s).round() as u32).max(1)); let mut out = Image::new(rw, rh); - for y in 0..rh.min(full.height.saturating_sub(y0)) { - for x in 0..rw.min(full.width.saturating_sub(x0)) { - out.data[(y * rw + x) as usize] = full.data[((y + y0) * full.width + x + x0) as usize]; + for y in 0..rh as i64 { + let sy = y + y0; + if sy < 0 || sy >= full.height as i64 { + continue; + } + for x in 0..rw as i64 { + let sx = x + x0; + if sx >= 0 && sx < full.width as i64 { + out.data[(y * rw as i64 + x) as usize] = full.data[(sy * full.width as i64 + sx) as usize]; + } } } return out; @@ -1124,7 +1138,7 @@ impl<'a> Renderer<'a> { /// The region of interest of a top-level 2D frame: (x, y) offset of the output in output /// pixels and its size. `None` renders the whole comp (no ROI, or a nested comp). pub(crate) fn roi_offset(&self) -> Option<(f64, f64, u32, u32)> { - let r = self.opts.roi.filter(|_| self.depth == 0)?; + let r = self.opts.roi.filter(|_| self.top)?; let s = self.opts.scale; if r[2] <= 0.0 || r[3] <= 0.0 { return None; @@ -1132,6 +1146,11 @@ impl<'a> Renderer<'a> { Some(((r[0] * s).round(), (r[1] * s).round(), ((r[2] * s).round() as u32).max(1), ((r[3] * s).round() as u32).max(1))) } + /// The region of interest's offset in output pixels for 3D projection (0 without one). + pub(crate) fn out_offset(&self) -> (f64, f64) { + self.roi_offset().map_or((0.0, 0.0), |(x, y, _, _)| (x, y)) + } + /// Scaled comp pixel → output pixel: the region of interest's offset. fn out_matrix(&self) -> Mat3 { match self.roi_offset() { @@ -1645,6 +1664,30 @@ pub fn adaptive_samples(travel: f64, per_frame: u32, limit: u32) -> usize { travel.ceil().clamp(lo, hi) as usize } +/// A region of interest `[x, y, w, h]` lies within a `w`×`h` comp frame. +pub fn roi_inside(r: [f64; 4], w: u32, h: u32) -> bool { + r[0] >= 0.0 && r[1] >= 0.0 && r[0] + r[2] <= w as f64 + 1e-6 && r[1] + r[3] <= h as f64 + 1e-6 +} + +/// The Extended Viewer's region of interest: the comp frame grown by `margin` × its size on every +/// side (`margin` = 1 shows a pasteboard one comp wide around the frame), intersected with the +/// `visible` comp-space rectangle `[x0, y0, x1, y1]` and snapped outward to whole 16 px steps from the frame edges +/// so panning re-renders rarely. `None` when the visible area lies inside the frame. +pub fn extended_region(w: u32, h: u32, visible: [f64; 4], margin: f64) -> Option<[f64; 4]> { + let (w, h) = (w as f64, h as f64); + let (mx, my) = (w * margin, h * margin); + let snap_lo = |v: f64| (v / 16.0).floor() * 16.0; + let snap_hi = |v: f64| (v / 16.0).ceil() * 16.0; + let x0 = snap_lo(visible[0].max(-mx)).min(0.0); + let y0 = snap_lo(visible[1].max(-my)).min(0.0); + let x1 = w + snap_hi(visible[2].min(w + mx) - w).max(0.0); + let y1 = h + snap_hi(visible[3].min(h + my) - h).max(0.0); + if x0 >= 0.0 && y0 >= 0.0 && x1 <= w && y1 <= h { + return None; + } + Some([x0, y0, x1 - x0, y1 - y0]) +} + pub fn render_frame(project: &Project, comp: ItemId, t: Tick, scale: f64) -> Image { Renderer::new(project, &NoFootage, RenderOpts { scale, ..Default::default() }).comp_frame(comp, t) } diff --git a/crates/render/src/tests_roi.rs b/crates/render/src/tests_roi.rs index 9adc8a8e..9182817c 100644 --- a/crates/render/src/tests_roi.rs +++ b/crates/render/src/tests_roi.rs @@ -71,3 +71,33 @@ fn roi_at_half_resolution() { assert_eq!((roi.width, roi.height), (30, 20)); assert_same(&roi, &crop(&full, 20, 10, 30, 20)); } + +/// Extended Viewer: a region reaching past the comp frame shows the 3D layer's pixels on the +/// pasteboard, and its inner part equals the normal frame. +#[test] +fn extended_region_renders_3d_layers_beyond_the_frame() { + let (mut p, cid) = project(true); + // Push the 3D (blue) layer half out of the frame on the right. + let comp = p.comp_mut(cid).unwrap(); + comp.layers[1].props.prop_mut("transform/position").unwrap().value = Value::Vec3([150.0, 60.0, 0.0]); + comp.layers[1].props.prop_mut("transform/rotation").unwrap().value = Value::Scalar(0.0); + let full = Renderer::new(&p, &NoFootage, RenderOpts::default()).comp_frame(cid, Tick::ZERO); + let region = crate::extended_region(160, 120, [-100.0, -100.0, 260.0, 220.0], 0.5).unwrap(); + assert_eq!(region, [-80.0, -64.0, 320.0, 248.0]); + let ext = Renderer::new(&p, &NoFootage, RenderOpts { roi: Some(region), ..Default::default() }).comp_frame(cid, Tick::ZERO); + assert_eq!((ext.width, ext.height), (320, 248)); + // Comp x 175 (past the 160 px frame) at y 60: the layer covers x 115..185. + let out = ext.get(175 + 80, 60 + 64); + assert!(out[3] > 0.5 && out[2] > 0.5, "pasteboard pixel {out:?}"); + // Nothing is drawn far outside every layer. + assert_eq!(ext.get(5, 5)[3], 0.0); + // The frame part is the normal frame. + assert_same(&crop(&ext, 80, 64, 160, 120), &full); +} + +#[test] +fn extended_region_is_none_inside_the_frame() { + assert_eq!(crate::extended_region(160, 120, [10.0, 10.0, 150.0, 110.0], 1.0), None); + // Clamped to the margin. + assert_eq!(crate::extended_region(100, 100, [-1000.0, 0.0, 50.0, 100.0], 1.0), Some([-112.0, 0.0, 212.0, 100.0])); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index 25755a47..1f5470b3 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -74,8 +74,10 @@ impl Item<'_> { } /// Build the plane geometry of a layer for a camera. -fn geo(cam: &CameraState, world: Mat4, b: [f64; 4], comp: (f64, f64), scale: f64, out: (u32, u32)) -> Option { - let proj = Mat4::scale(vec3(scale, scale, 1.0)) * cam.projection(comp.0, comp.1); +fn geo(cam: &CameraState, world: Mat4, b: [f64; 4], comp: (f64, f64), scale: f64, out: (u32, u32), off: (f64, f64)) -> Option { + // The output canvas may start at `off` (scaled comp pixels): region of interest / Extended + // Viewer frames. + let proj = Mat4::translate(vec3(-off.0, -off.1, 0.0)) * Mat4::scale(vec3(scale, scale, 1.0)) * cam.projection(comp.0, comp.1); let full = proj * world; let h = full.plane_to_mat3(); // Edge-on planes project to a line (singular homography): nothing to draw. @@ -222,7 +224,7 @@ fn prepare_with<'a>( let bounds = buf_bounds(&buf); let n = if in_run { mb_samples(r, ctx, layer) } else { 1 }; let geos: Vec = if n <= 1 { - geo(&cam, world, bounds, comp, s, out).into_iter().collect() + geo(&cam, world, bounds, comp, s, out, r.out_offset()).into_iter().collect() } else { let fd = ctx.comp.frame_duration().seconds(); let angle = ctx.comp.shutter_angle / 360.0; @@ -232,7 +234,7 @@ fn prepare_with<'a>( let f = phase + angle * i as f64 / (n - 1) as f64; let sub = ctx.at(ctx.time + Tick::from_seconds_f64(f * fd)); let c = camera_for(r, &sub); - geo(&c, outer * sub.world_matrix(layer) * local, bounds, comp, s, out) + geo(&c, outer * sub.world_matrix(layer) * local, bounds, comp, s, out, r.out_offset()) }) .collect() }; diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index d2de8fc3..ac9a028e 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -89,6 +89,8 @@ pub enum Icon { MaskVis, Checker, Region, + /// Extended Viewer: a small frame with corner ticks reaching outward. + ExtendedViewer, Plus, Minus, Close, @@ -513,6 +515,12 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.rect(2.5, 2.5, 13.5, 13.5); pen.rect(5.5, 5.5, 10.5, 10.5); } + ExtendedViewer => { + pen.rect(5.0, 5.0, 11.0, 11.0); + for (x, y, dx, dy) in [(2.0, 2.0, 1.0, 1.0), (14.0, 2.0, -1.0, 1.0), (2.0, 14.0, 1.0, -1.0), (14.0, 14.0, -1.0, -1.0)] { + pen.line(&[(x + dx * 3.0, y), (x, y), (x, y + dy * 3.0)]); + } + } Plus => { pen.line(&[(8.0, 3.0), (8.0, 13.0)]); pen.line(&[(3.0, 8.0), (13.0, 8.0)]); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index c56100af..e345e9c3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -344,6 +344,7 @@ impl EffectcraftApp { && v.channel == effectcraft_engine::viewer::Channel::Rgb && v.exposure == 0.0 && self.session.state.region_of_interest.is_none() + && self.ui.viewer.extended.is_none() } /// Set up the GPU compositor on egui-wgpu's device (once). Without an adapter that runs @@ -439,7 +440,7 @@ impl EffectcraftApp { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); // The region of interest changes what is rendered too. - let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); + let roi = self.viewer_region(comp); if let Some(r) = roi { r.map(f64::to_bits).hash(&mut h); } @@ -454,6 +455,15 @@ impl EffectcraftApp { h.finish() | 1 } + /// The comp-space region viewer frames of `comp` cover: the region of interest, else the + /// Extended Viewer's area (frame plus pasteboard), else `None` (the whole frame). + pub fn viewer_region(&self, comp: ItemId) -> Option<[f64; 4]> { + if self.session.active_comp_id() != Some(comp) { + return None; + } + self.session.state.region_of_interest.or(self.ui.viewer.extended) + } + /// Request a prefetch frame render (no-op if cached/in flight). pub fn request_frame(&self, comp: ItemId, frame: i64, scale: f64) { self.request_frame_with(comp, frame, scale, false); @@ -467,7 +477,7 @@ impl EffectcraftApp { /// Render options of viewer frames of `comp` at `scale`. pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); - let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); + let roi = self.viewer_region(comp); RenderOpts { scale, motion_blur: true, diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index ccb963cf..3d1f8de4 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -461,6 +461,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ctx.data_mut(|d| d.insert_temp(map_id(), map)); let comp_rect = Rect::from_min_size(origin, vec2(cw * zoom, ch * zoom)); let painter = p.with_clip_rect(area); + // Settings ▸ 3D ▸ Extended Viewer: custom 3D views (and Draft 3D) render the visible + // pasteboard too, so 3D layers reaching past the comp frame stay visible there. + app.ui.viewer.extended = extended_region(app, &comp, cid, map); // Frame. let ppp = ctx.pixels_per_point(); @@ -491,7 +494,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; // The frame (or snapshot) through Show Channel and exposure; ROI frames cover the region. vt::draw_frame(app, &ctx, &painter, comp_rect, cid, &ectx); - painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); + if app.ui.viewer.extended.is_some() { + // The comp frame outlined over the extended render. + painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_gray(150)), StrokeKind::Outside); + app.auto.add("viewer.extendedArea", rect_of(&map, app.ui.viewer.extended), "Extended Viewer area"); + } else { + painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); + } app.auto.add("viewer.comp", comp_rect, &comp_name); app.auto.add("viewer.area", area, "Composition viewer"); @@ -1654,6 +1663,34 @@ pub(crate) fn hex_rgb(s: &str) -> Option<[u8; 3]> { Some([p(0)?, p(2)?, p(4)?]) } +/// The Extended Viewer's render region for this frame (see [`ViewerState::extended`]): +/// Settings ▸ 3D ▸ Extended Viewer on, the comp has 3D layers and the view is a custom 3D view +/// (or Draft 3D is on), and part of the visible viewer lies outside the comp frame. +/// +/// [`ViewerState::extended`]: crate::state::ViewerState::extended +pub(crate) fn extended_region( + app: &EffectcraftApp, + comp: &effectcraft_engine::project::Comp, + cid: effectcraft_engine::project::ItemId, + map: ViewerMap, +) -> Option<[f64; 4]> { + if !app.session.prefs.three_d.extended_viewer || !comp.has_3d() || app.session.state.region_of_interest.is_some() { + return None; + } + let view = app.session.state.views3d.get(&cid).map(|v| v.current).unwrap_or_default(); + if view == View3D::ActiveCamera && !comp.draft_3d { + return None; + } + let a = map.to_comp(map.area.min); + let b = map.to_comp(map.area.max); + effectcraft_engine::render::extended_region(comp.width, comp.height, [a[0], a[1], b[0], b[1]], 1.0) +} + +fn rect_of(map: &ViewerMap, r: Option<[f64; 4]>) -> Rect { + let [x, y, w, h] = r.unwrap_or_default(); + Rect::from_min_max(map.to_screen([x, y]), map.to_screen([x + w, y + h])) +} + /// Put a rendered frame on screen: CPU pixels go into an egui texture; GPU frames are drawn /// straight from their wgpu texture (registered with egui-wgpu, no readback). fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames::FrameKey, img: crate::frames::FrameImage) { @@ -1672,7 +1709,7 @@ fn show_frame(app: &mut EffectcraftApp, ctx: &egui::Context, key: crate::frames: } app.viewer_shown = app.viewer_tex.as_ref().map(|(t, k)| (t.id(), *k)); app.viewer_image = Some(img); - vt::set_texture_roi(ctx, app.session.state.region_of_interest); + vt::set_texture_roi(ctx, app.session.state.region_of_interest.or(app.ui.viewer.extended)); } FrameImage::Gpu(f) => { let Some(rs) = &app.wgpu else { return }; diff --git a/crates/ui-egui/src/panels/viewer_tools.rs b/crates/ui-egui/src/panels/viewer_tools.rs index 0ce1b29c..18f67973 100644 --- a/crates/ui-egui/src/panels/viewer_tools.rs +++ b/crates/ui-egui/src/panels/viewer_tools.rs @@ -694,7 +694,13 @@ pub(crate) fn bottom_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui, bar: Rect, if let Some(i) = popup(app, ui, "vw-cam-pop", r, &items, "view3dItem") { let _ = app.session.execute("view.set3DView", json!({"view": View3D::ALL[i].id()})); } - x = r.max.x + 6.0; + x = r.max.x + 4.0; + // Extended Viewer (Settings ▸ 3D): custom views and Draft 3D show past the comp frame. + let on = app.session.prefs.three_d.extended_viewer; + if tog(ui, &mut app.auto, &mut x, Icon::ExtendedViewer, on, "extendedViewer", "Extended Viewer").0 { + let _ = app.session.execute("view.extendedViewer", json!({})); + } + x += 2.0; } // Current time. let tc = crate::panels::timecode(&app.session, comp, time); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 07715d16..584cdd95 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -299,6 +299,10 @@ pub struct ViewerState { /// The pointer is dragging in the viewer this frame (Adaptive Resolution). #[serde(skip)] pub interacting: bool, + /// Extended Viewer: the comp-space region the viewer renders this frame (comp frame plus + /// the visible pasteboard), set while a 3D view shows past the frame. + #[serde(skip)] + pub extended: Option<[f64; 4]>, /// View ▸ Panel Background Color (`None` = theme default). pub pasteboard: Option<[u8; 3]>, pub custom_pasteboard: [u8; 3], @@ -326,6 +330,7 @@ impl Default for ViewerState { ruler_origin: [0.0, 0.0], roi_draw: false, interacting: false, + extended: None, pasteboard: None, custom_pasteboard: [0x80, 0x80, 0x80], } diff --git a/crates/ui-egui/tests/ui_3d.rs b/crates/ui-egui/tests/ui_3d.rs index 9cd7e04f..385bf7ac 100644 --- a/crates/ui-egui/tests/ui_3d.rs +++ b/crates/ui-egui/tests/ui_3d.rs @@ -116,3 +116,51 @@ fn snapshot() { img.save(format!("{d}/ui3d_reveal_rotation.png")).unwrap(); } } + +/// Settings ▸ 3D ▸ Extended Viewer: a custom 3D view renders the pasteboard around the comp +/// frame, and 3D layers reaching past the frame draw pixels there. +#[test] +fn extended_viewer_renders_past_the_comp_frame() { + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| app()); + h.state_mut().session.execute("view.3d.custom1", json!({})).unwrap(); + // Zoom out so the viewer shows plenty of pasteboard. + h.state_mut().ui.viewer.zoom = Some(0.15); + settle(&mut h); + assert!(h.state().ui.viewer.extended.is_none(), "off by default"); + let ids: Vec = h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect(); + assert!(ids.iter().any(|i| i == "viewer.extendedViewer"), "toggle registered"); + assert_eq!(h.state_mut().session.execute("view.extendedViewer", json!({"value": true})).unwrap(), json!(true)); + h.step(); + settle(&mut h); + h.step(); + settle(&mut h); + let region = h.state().ui.viewer.extended.expect("extended region in a custom view"); + let (cw, ch) = { + let c = h.state().session.active_comp().unwrap(); + (c.width as f64, c.height as f64) + }; + assert!(region[0] < 0.0 && region[1] < 0.0 && region[0] + region[2] > cw && region[1] + region[3] > ch, "{region:?}"); + let ids: Vec = h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect(); + assert!(ids.iter().any(|i| i == "viewer.extendedArea")); + let img = h.state_mut().viewer_pixels().expect("frame"); + let s = img.size[0] as f64 / region[2]; + assert!((img.size[1] as f64 / region[3] - s).abs() < 0.05, "frame covers the region"); + // Opaque pixels outside the comp rectangle. + let (fx0, fy0) = (-region[0] * s, -region[1] * s); + let (fx1, fy1) = (fx0 + cw * s, fy0 + ch * s); + let mut outside = 0; + for y in 0..img.size[1] { + for x in 0..img.size[0] { + let (xf, yf) = (x as f64 + 0.5, y as f64 + 0.5); + if (xf < fx0 - 1.0 || xf > fx1 + 1.0 || yf < fy0 - 1.0 || yf > fy1 + 1.0) && img.pixels[y * img.size[0] + x].a() > 0 { + outside += 1; + } + } + } + assert!(outside > 50, "pixels drawn on the pasteboard: {outside}"); + // The Active Camera view (Draft 3D off) shows just the frame. + h.state_mut().session.execute("view.3d.activeCamera", json!({})).unwrap(); + h.step(); + h.step(); + assert!(h.state().ui.viewer.extended.is_none()); +} From 1b823eb852b8d7ab00c41bfc6c795ede4dff1879 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:25:45 -0400 Subject: [PATCH 33/36] =?UTF-8?q?M13.5:=20Home=20=E2=96=B8=20Learn=20tab?= =?UTF-8?q?=20with=20interactive=20tutorials=20(learn.list/start/step/stop?= =?UTF-8?q?/state,=20coach=20card,=20target=20highlight)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/mod.rs | 1 + crates/engine/src/learn.rs | 525 +++++++++++++++++++++++++++++ crates/engine/src/lib.rs | 5 + crates/ui-egui/src/control.rs | 4 + crates/ui-egui/src/lib.rs | 4 +- crates/ui-egui/src/panels/home.rs | 71 +++- crates/ui-egui/src/panels/learn.rs | 199 +++++++++++ crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/state.rs | 4 + 9 files changed, 810 insertions(+), 4 deletions(-) create mode 100644 crates/engine/src/learn.rs create mode 100644 crates/ui-egui/src/panels/learn.rs diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index d9e4feec..d9ef2816 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -182,6 +182,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(align_data::specs()); v.extend(content_fill::specs()); v.extend(stubs::specs()); + v.extend(crate::learn::specs()); v }) } diff --git a/crates/engine/src/learn.rs b/crates/engine/src/learn.rs new file mode 100644 index 00000000..cea72936 --- /dev/null +++ b/crates/engine/src/learn.rs @@ -0,0 +1,525 @@ +//! Learn: interactive tutorials on the Home screen (our own content, licensed with the repo). +//! +//! A tutorial is a list of steps. Each step names the UI element to highlight (an automation id, +//! see `docs/control-protocol.md`), the commands that complete it when the user performs them, +//! and the commands "Show me" runs to perform it for the user. Progress lives on the +//! [`Session`] (not the project), so opening the demo project in a step keeps the tutorial going. +//! +//! Commands: `learn.list`, `learn.start {id}`, `learn.step {action: next|back|showMe|goto, +//! index?}`, `learn.stop`, `learn.state`. + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::commands::{CommandSpec, always, bad, str_p}; +use crate::{Result, Session, cmd, query}; + +/// One step of a tutorial. +#[derive(Clone, Debug, Serialize)] +pub struct Step { + pub title: String, + pub text: String, + /// Automation id of the element to highlight. `*` matches any run of characters and + /// `{layer}` stands for the first selected layer's id (see [`resolve_target`]). + pub target: Option, + /// Commands that complete the step when they run (empty: a reading step, advanced with Next). + pub accepts: Vec, + /// Parameters the completing command must carry (a subset match; `null` = any). + pub when: Value, + /// What "Show me" runs: (command, params), in order. + pub show: Vec<(String, Value)>, +} + +/// An interactive tutorial. +#[derive(Clone, Debug, Serialize)] +pub struct Tutorial { + pub id: String, + pub title: String, + pub summary: String, + /// Rough length in minutes. + pub minutes: u32, + pub steps: Vec, +} + +/// Where the user is in a tutorial. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Progress { + pub tutorial: String, + /// Index of the current step (`== steps.len()` once finished). + pub step: usize, + pub done: bool, +} + +struct S<'a> { + title: &'a str, + text: &'a str, + target: Option<&'a str>, + accepts: &'a [&'a str], + when: Value, + show: Vec<(&'a str, Value)>, +} + +fn step(s: S) -> Step { + Step { + title: s.title.into(), + text: s.text.into(), + target: s.target.map(str::to_string), + accepts: s.accepts.iter().map(|a| a.to_string()).collect(), + when: s.when, + show: s.show.into_iter().map(|(c, p)| (c.to_string(), p)).collect(), + } +} + +/// The built-in tutorials. +pub fn tutorials() -> Vec { + vec![ + Tutorial { + id: "animate-title".into(), + title: "Animate a title".into(), + summary: "Make a composition, type a title and fly it in with two position keyframes and an easy ease.".into(), + minutes: 4, + steps: vec![ + step(S { + title: "Make a composition", + text: "Everything you animate lives in a composition. Choose Composition ▸ New Composition (Cmd+N) and keep the HD 1920×1080 settings.", + target: Some("menu.Composition"), + accepts: &["comp.new"], + when: Value::Null, + show: vec![("comp.new", json!({"name": "Title Animation", "width": 1920, "height": 1080, "duration": 5}))], + }), + step(S { + title: "Type a title", + text: "Choose Layer ▸ New ▸ Text (or the Type tool) and type a short title. A new text layer appears at the top of the Timeline.", + target: Some("menu.Layer"), + accepts: &["layer.newText"], + when: Value::Null, + show: vec![("layer.newText", json!({"text": "Hello, motion", "size": 120}))], + }), + step(S { + title: "Start the animation", + text: "Twirl the text layer open, then Transform. Click the stopwatch beside Position: it turns blue and records a keyframe at the current time.", + target: Some("timeline.layer.{layer}.twirl"), + accepts: &["prop.toggleAnimation", "prop.addKey", "prop.toggleKey", "anim.addKeyframe"], + when: Value::Null, + show: vec![ + ("prop.set", json!({"path": "transform/position", "value": [-400.0, 540.0, 0.0]})), + ("prop.toggleAnimation", json!({"path": "transform/position", "value": true})), + ], + }), + step(S { + title: "Move in time", + text: "Drag the blue current-time indicator to about 0:00:01:00, or click in the time ruler above the layer bars.", + target: Some("timeline.ruler"), + accepts: &["time.set", "time.step", "time.go"], + when: Value::Null, + show: vec![("time.set", json!({"time": 1.0}))], + }), + step(S { + title: "Set the end position", + text: "Drag the title to the middle of the frame in the Composition viewer, or scrub the Position value. Because the stopwatch is on, a second keyframe appears by itself.", + target: Some("viewer.comp"), + accepts: &["prop.set"], + when: json!({"path": "transform/position"}), + show: vec![("prop.set", json!({"path": "transform/position", "value": [960.0, 540.0, 0.0]}))], + }), + step(S { + title: "Smooth it out", + text: "Select both keyframes (drag a box around them, or click the Position name) and press F9 for Easy Ease. The title now glides in and settles.", + target: Some("timeline.ruler"), + accepts: &["keys.easyEase"], + when: Value::Null, + show: vec![("keys.selectAll", json!({})), ("keys.easyEase", json!({}))], + }), + step(S { + title: "Watch it", + text: "Press the Spacebar (or Play in the Preview panel) to play your animation. Press it again to stop. That's it: you made your first animation.", + target: Some("panel.Preview"), + accepts: &["playback.toggle"], + when: Value::Null, + show: vec![("time.set", json!({"time": 0.0})), ("playback.toggle", json!({}))], + }), + ], + }, + Tutorial { + id: "track-attach".into(), + title: "Track and attach".into(), + summary: "Follow a moving element with the point tracker, then hang a null on the track and parent a layer to it.".into(), + minutes: 5, + steps: vec![ + step(S { + title: "Open the demo", + text: "We'll track the moving lower third in the demo project. Open it from Home ▸ Open Demo Project (or File ▸ Open Demo Project).", + target: Some("menu.File"), + accepts: &["file.openDemoProject"], + when: Value::Null, + show: vec![("file.openDemoProject", json!({}))], + }), + step(S { + title: "Pick what to track", + text: "Click the Lower Third layer in the Timeline, then press I to jump to where it starts (0:00:05:00). The tracker follows the pixels of the selected layer's source.", + target: Some("timeline.layer.*.row"), + accepts: &["layer.select"], + when: Value::Null, + show: vec![("layer.select", json!({"layers": ["Lower Third"]})), ("time.set", json!({"time": 5.5}))], + }), + step(S { + title: "Add a track point", + text: "Open Window ▸ Tracker and click Track Motion. A track point appears: the inner box is the feature to follow, the outer box is where to look for it on the next frame.", + target: Some("tracker.trackMotion"), + accepts: &["track.motion", "track.new"], + when: Value::Null, + show: vec![("track.motion", json!({}))], + }), + step(S { + title: "Analyze", + text: "Click Analyze forward ▶ in the Tracker panel. The point follows the feature frame by frame and leaves a path of keyframes behind.", + target: Some("tracker.analyze.forward"), + accepts: &["track.analyze"], + when: Value::Null, + show: vec![("track.analyze", json!({"direction": "forward", "end": 6.0, "wait": true}))], + }), + step(S { + title: "Make a null to carry the motion", + text: "Choose Layer ▸ New ▸ Null Object. A null is invisible; it only holds a transform, which makes it a handy anchor for tracked motion.", + target: Some("menu.Layer"), + accepts: &["layer.newNull"], + when: Value::Null, + show: vec![("layer.newNull", json!({"name": "Track Null"}))], + }), + step(S { + title: "Point the track at the null", + text: "Select Lower Third again, click Edit Target… in the Tracker panel and choose the null as the motion target.", + target: Some("tracker.editTarget"), + accepts: &["track.setTarget"], + when: Value::Null, + show: vec![("track.setTarget", json!({"layer": "Lower Third", "target": "Track Null"}))], + }), + step(S { + title: "Apply", + text: "Click Apply and keep X and Y. The null's Position now follows the track.", + target: Some("tracker.apply"), + accepts: &["track.apply"], + when: Value::Null, + show: vec![("track.apply", json!({"layer": "Lower Third", "dimensions": "xy"}))], + }), + step(S { + title: "Attach a layer", + text: "Drag a layer's pick whip (the spiral in Parent & Link) onto the null, or pick it from the menu. The child now rides along with the track.", + target: Some("timeline.layer.*.pickWhip"), + accepts: &["layer.setParent"], + when: Value::Null, + show: vec![("layer.setParent", json!({"layers": ["Tagline"], "parent": "Track Null"}))], + }), + ], + }, + Tutorial { + id: "3d-scene".into(), + title: "Make a 3D scene".into(), + summary: "Turn flat layers into 3D, light them, add a camera and look at the scene from outside the frame.".into(), + minutes: 5, + steps: vec![ + step(S { + title: "Make a composition", + text: "Choose Composition ▸ New Composition. Any size works; 1920×1080 is a good start.", + target: Some("menu.Composition"), + accepts: &["comp.new"], + when: Value::Null, + show: vec![("comp.new", json!({"name": "3D Scene", "width": 1920, "height": 1080, "duration": 5}))], + }), + step(S { + title: "Add a floor", + text: "Choose Layer ▸ New ▸ Solid and make a large solid in a calm colour. It will become the floor.", + target: Some("menu.Layer"), + accepts: &["layer.newSolid"], + when: Value::Null, + show: vec![("layer.newSolid", json!({"name": "Floor", "color": "#3c5a82", "width": 2400, "height": 2400}))], + }), + step(S { + title: "Make it 3D", + text: "Click the cube switch of the solid in the Timeline. The layer gains a Z position and X/Y/Z rotation.", + target: Some("timeline.layer.{layer}.switch.threeD"), + accepts: &["layer.setSwitch"], + when: json!({"switch": "threeD"}), + show: vec![("layer.setSwitch", json!({"switch": "threeD", "value": true}))], + }), + step(S { + title: "Lay it down", + text: "Set X Rotation to -80° (twirl open Transform and scrub the value) so the solid lies like a floor, and push its Position down a little.", + target: Some("timeline.layer.{layer}.twirl"), + accepts: &["prop.set"], + when: Value::Null, + show: vec![ + ("prop.set", json!({"path": "transform/rotationX", "value": -80.0})), + ("prop.set", json!({"path": "transform/position", "value": [960.0, 760.0, 0.0]})), + ], + }), + step(S { + title: "Light it", + text: "Choose Layer ▸ New ▸ Light and pick Spot. 3D layers now catch light and fall-off.", + target: Some("menu.Layer"), + accepts: &["layer.newLight"], + when: Value::Null, + show: vec![("layer.newLight", json!({"kind": "Spot", "name": "Key Light"}))], + }), + step(S { + title: "Add a camera", + text: "Choose Layer ▸ New ▸ Camera. The comp is now seen through this camera; orbit it with the camera tools (press C).", + target: Some("menu.Layer"), + accepts: &["layer.newCamera"], + when: Value::Null, + show: vec![("layer.newCamera", json!({"name": "Camera"}))], + }), + step(S { + title: "Step outside the camera", + text: "In the 3D View popup under the viewer choose Custom View 1. You see the scene, the camera and the light from a working view.", + target: Some("viewer.view3d"), + accepts: &[ + "view.set3DView", + "view.3d.custom1", + "view.3d.custom2", + "view.3d.custom3", + "view.3d.front", + "view.3d.top", + "view.3d.left", + "view.3d.right", + "view.3d.back", + "view.3d.bottom", + ], + when: Value::Null, + show: vec![("view.3d.custom1", json!({}))], + }), + step(S { + title: "See past the frame", + text: "Click Extended Viewer beside the 3D View popup. The viewer now shows the layers beyond the comp's edges, with the frame outlined.", + target: Some("viewer.extendedViewer"), + accepts: &["view.extendedViewer"], + when: Value::Null, + show: vec![("view.extendedViewer", json!({"value": true}))], + }), + ], + }, + ] +} + +/// The tutorial with this id. +pub fn find(id: &str) -> Option { + tutorials().into_iter().find(|t| t.id == id) +} + +/// The automation id pattern of a step's target with `{layer}` filled in (`*` stays a wildcard). +pub fn resolve_target(target: &str, layer: Option) -> String { + match layer { + Some(l) => target.replace("{layer}", &l.to_string()), + None => target.replace("{layer}", "*"), + } +} + +/// Whether an automation id matches a target pattern (`*` = any run of characters). +pub fn target_matches(pattern: &str, id: &str) -> bool { + let parts: Vec<&str> = pattern.split('*').collect(); + if parts.len() == 1 { + return pattern == id; + } + let mut rest = id; + for (i, part) in parts.iter().enumerate() { + if i == 0 { + let Some(r) = rest.strip_prefix(part) else { return false }; + rest = r; + } else if i == parts.len() - 1 { + return rest.ends_with(part); + } else { + let Some(at) = rest.find(part) else { return false }; + rest = &rest[at + part.len()..]; + } + } + true +} + +/// `want`'s keys all equal in `got` (`null` matches anything). +fn params_match(want: &Value, got: &Value) -> bool { + match want { + Value::Object(m) => m.iter().all(|(k, v)| got.get(k) == Some(v)), + _ => true, + } +} + +impl Session { + /// The current tutorial and step, if one is running. + pub fn learn_current(&self) -> Option<(Tutorial, usize)> { + let p = self.learn.as_ref()?; + Some((find(&p.tutorial)?, p.step)) + } + + /// A command ran: complete the current step when it is one the step accepts. + pub(crate) fn learn_observe(&mut self, id: &str, params: &Value) { + if id.starts_with("learn.") { + return; + } + let Some((t, i)) = self.learn_current() else { return }; + let Some(st) = t.steps.get(i) else { return }; + if st.accepts.iter().any(|a| a == id) && params_match(&st.when, params) { + self.learn_goto(&t, i + 1); + } + } + + fn learn_goto(&mut self, t: &Tutorial, i: usize) { + let n = t.steps.len(); + let i = i.min(n); + if let Some(p) = &mut self.learn { + p.step = i; + p.done = i >= n; + } + if i >= n { + self.toast(format!("Tutorial complete: {}", t.title)); + } + } +} + +fn state_json(s: &Session) -> Value { + match s.learn_current() { + Some((t, i)) => json!({ + "tutorial": t.id, + "title": t.title, + "step": i, + "steps": t.steps.len(), + "done": i >= t.steps.len(), + "current": t.steps.get(i).map(|st| json!({ + "title": st.title, + "text": st.text, + "target": st.target.as_deref().map(|x| resolve_target(x, s.state.selected_layers.first().map(|l| l.0))), + "accepts": st.accepts, + })), + }), + None => Value::Null, + } +} + +fn list(_: &mut Session, _: &Value) -> Result { + Ok(json!( + tutorials() + .iter() + .map(|t| json!({"id": t.id, "title": t.title, "summary": t.summary, "minutes": t.minutes, "steps": t.steps.iter().map(|s| s.title.clone()).collect::>()})) + .collect::>() + )) +} + +fn start(s: &mut Session, p: &Value) -> Result { + let id = str_p(p, "id").or_else(|| str_p(p, "tutorial")).ok_or_else(|| bad("learn.start", "missing `id`"))?; + let t = find(id).ok_or_else(|| bad("learn.start", format!("unknown tutorial `{id}` (see learn.list)")))?; + s.learn = Some(Progress { tutorial: t.id.clone(), step: 0, done: false }); + Ok(state_json(s)) +} + +fn step_cmd(s: &mut Session, p: &Value) -> Result { + let (t, i) = s.learn_current().ok_or_else(|| bad("learn.step", "no tutorial is running (learn.start)"))?; + match str_p(p, "action").unwrap_or("next") { + "next" => s.learn_goto(&t, i + 1), + "back" | "prev" | "previous" => s.learn_goto(&t, i.saturating_sub(1)), + "goto" => { + let n = p.get("index").and_then(Value::as_u64).ok_or_else(|| bad("learn.step", "goto needs `index`"))?; + s.learn_goto(&t, n as usize); + } + "showMe" | "show" => { + let st = t.steps.get(i).ok_or_else(|| bad("learn.step", "the tutorial is finished"))?; + for (c, params) in &st.show { + s.execute(c, params.clone()).map_err(|e| bad("learn.step", format!("Show me: {c}: {e}")))?; + } + // Steps whose commands don't complete them (or reading steps) still move on. + if s.learn.as_ref().is_some_and(|p| p.step == i) { + s.learn_goto(&t, i + 1); + } + } + a => return Err(bad("learn.step", format!("action must be next|back|showMe|goto, not `{a}`"))), + } + Ok(state_json(s)) +} + +fn stop(s: &mut Session, _: &Value) -> Result { + s.learn = None; + Ok(Value::Null) +} + +pub fn specs() -> Vec { + vec![ + query!("learn.list", "Learn Tutorials", "{} → [{id, title, summary, minutes, steps}]", list), + query!("learn.state", "Tutorial State", "{} → {tutorial, step, steps, done, current: {title, text, target, accepts}} | null", |s, _| Ok(state_json( + s + ))), + cmd!("learn.start", "Start Tutorial", [], None, "{id}", always, start), + cmd!("learn.step", "Tutorial Step", [], None, "{action?: next|back|showMe|goto, index?}", always, step_cmd), + cmd!("learn.stop", "Close Tutorial", [], None, "{}", always, stop), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn targets_match_with_wildcards() { + assert!(target_matches("timeline.layer.*.row", "timeline.layer.12.row")); + assert!(!target_matches("timeline.layer.*.row", "timeline.layer.12.bar")); + assert!(target_matches("menu.File", "menu.File")); + assert!(!target_matches("menu.File", "menu.File.2")); + assert_eq!(resolve_target("timeline.layer.{layer}.twirl", Some(7)), "timeline.layer.7.twirl"); + assert_eq!(resolve_target("timeline.layer.{layer}.twirl", None), "timeline.layer.*.twirl"); + } + + #[test] + fn every_step_names_real_commands() { + for t in tutorials() { + assert!(!t.steps.is_empty(), "{}", t.id); + for st in &t.steps { + for c in st.accepts.iter().chain(st.show.iter().map(|(c, _)| c)) { + assert!(crate::commands::find(c).is_some(), "{}: unknown command {c}", t.id); + } + assert!(!st.show.is_empty(), "{}: `{}` has no Show me", t.id, st.title); + } + } + } + + #[test] + fn tutorials_run_end_to_end_with_show_me() { + for id in ["animate-title", "3d-scene", "track-attach"] { + let mut s = Session::default(); + s.execute("learn.start", json!({"id": id})).unwrap(); + let n = find(id).unwrap().steps.len(); + for i in 0..n { + let st = s.execute("learn.step", json!({"action": "showMe"})).unwrap_or_else(|e| panic!("{id} step {i}: {e}")); + assert_eq!(st["step"], json!(i + 1), "{id}: {st}"); + } + assert_eq!(s.execute("learn.state", json!({})).unwrap()["done"], json!(true)); + } + } + + #[test] + fn performing_the_command_advances() { + let mut s = Session::default(); + let list = s.execute("learn.list", json!({})).unwrap(); + assert_eq!(list.as_array().unwrap().len(), 3); + s.execute("learn.start", json!({"id": "animate-title"})).unwrap(); + // Unrelated commands don't advance. + s.execute("file.openDemoProject", json!({})).unwrap(); + assert_eq!(s.learn.as_ref().unwrap().step, 0); + s.execute("comp.new", json!({"name": "Mine"})).unwrap(); + assert_eq!(s.learn.as_ref().unwrap().step, 1); + s.execute("layer.newText", json!({"text": "Hi"})).unwrap(); + assert_eq!(s.learn.as_ref().unwrap().step, 2); + let cur = s.execute("learn.state", json!({})).unwrap(); + let lid = s.state.selected_layers[0].0; + assert_eq!(cur["current"]["target"], json!(format!("timeline.layer.{lid}.twirl"))); + // Back and Next. + s.execute("learn.step", json!({"action": "back"})).unwrap(); + assert_eq!(s.learn.as_ref().unwrap().step, 1); + s.execute("learn.step", json!({})).unwrap(); + assert_eq!(s.learn.as_ref().unwrap().step, 2); + // A matching command with the wrong parameters doesn't complete a `when` step. + s.execute("learn.start", json!({"id": "3d-scene"})).unwrap(); + s.execute("learn.step", json!({"action": "goto", "index": 2})).unwrap(); + s.execute("layer.setSwitch", json!({"switch": "shy", "value": true})).ok(); + assert_eq!(s.learn.as_ref().unwrap().step, 2); + s.execute("learn.stop", json!({})).unwrap(); + assert!(s.learn.is_none()); + assert!(s.execute("learn.step", json!({})).is_err()); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 17d9b235..25c88881 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -14,6 +14,7 @@ pub mod commands; pub mod config; pub mod demo; pub mod jobs; +pub mod learn; pub mod links; pub mod logging; pub mod mask_track; @@ -351,6 +352,8 @@ pub struct Session { /// Finished tasks, newest last (Progress panel, `jobs.list`). pub job_log: Vec, pub next_task_id: u64, + /// The running Learn tutorial (Home ▸ Learn), see [`learn`]. + pub learn: Option, } /// A script to run (see [`Session::script`]). @@ -412,6 +415,7 @@ impl Default for Session { tasks: vec![], job_log: vec![], next_task_id: 1, + learn: None, } } } @@ -433,6 +437,7 @@ impl Session { } } let r = (spec.run)(self, ¶ms)?; + self.learn_observe(id, ¶ms); if spec.journal { self.journal.push((id.to_string(), params)); if self.journal.len() > 10_000 { diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index d60b2ed2..ff3db561 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -204,6 +204,10 @@ pub fn handle(app: &mut EffectcraftApp, ctx: &egui::Context, req: &ControlReques if let Some(b) = p.get("home").and_then(Value::as_bool) { app.ui.start_screen = b; } + // Home screen tab: "home" or "learn". + if let Some(tab) = p.get("homeTab").and_then(Value::as_str) { + app.ui.home_learn = tab == "learn"; + } ok(Value::Null) } "ui.panel.show" | "ui.panel.close" => { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e345e9c3..d873a333 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -287,9 +287,10 @@ impl EffectcraftApp { self.ui.dock = self.ui.saved_workspaces.get(name).cloned().unwrap_or_else(|| dock::workspace(name)); self.ui.floating = self.ui.saved_floating.get(name).cloned().unwrap_or_else(|| dock::workspace_floating(name)); self.ui.maximized = None; - // Learn: the Home screen (community links) in the Composition panel. + // Learn: the Home screen's Learn tab (tutorials) in the Composition panel. if name == "Learn" { self.ui.start_screen = true; + self.ui.home_learn = true; } } @@ -854,6 +855,7 @@ impl EffectcraftApp { panels::precomp::mini_flowchart(self, &ctx); panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); + panels::learn::coach(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/panels/home.rs b/crates/ui-egui/src/panels/home.rs index 42ded4b7..d53a0c08 100644 --- a/crates/ui-egui/src/panels/home.rs +++ b/crates/ui-egui/src/panels/home.rs @@ -173,7 +173,25 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { crate::header::paint_logo(&p, Rect::from_min_size(pos2(x0, y), vec2(40.0, 40.0))); p.text(pos2(x0 + 52.0, y + 12.0), Align2::LEFT_CENTER, "EffectCraft", Tokens::semibold(22.0), Color32::WHITE); p.text(pos2(x0 + 52.0, y + 32.0), Align2::LEFT_CENTER, format!("Version {}", env!("CARGO_PKG_VERSION")), Tokens::ui(11.0), t.text_faint); - y += 64.0; + y += 56.0; + // Home / Learn tabs. + for (k, (label, learn)) in [("Home", false), ("Learn", true)].into_iter().enumerate() { + let r = Rect::from_min_size(pos2(x0 + k as f32 * 84.0, y), vec2(78.0, 28.0)); + let resp = ui.interact(r, egui::Id::new(("home-tab", label)), Sense::click()); + let on = app.ui.home_learn == learn; + if on || resp.hovered() { + p.rect_filled(r, 6.0, if on { Color32::from_rgb(0x2e, 0x31, 0x3b) } else { t.hover }); + } + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::medium(13.0), if on { Color32::WHITE } else { t.text_dim }); + if on { + p.line_segment([pos2(r.min.x + 12.0, r.max.y - 1.0), pos2(r.max.x - 12.0, r.max.y - 1.0)], Stroke::new(2.0, t.accent)); + } + app.auto.add(&format!("home.tab.{}", label.to_ascii_lowercase()), r, label); + if resp.clicked() { + app.ui.home_learn = learn; + } + } + y += 40.0; for (label, id, primary) in [ ("New Project", "file.newProject", true), ("Open Project…", "file.open", false), @@ -239,13 +257,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let community_bottom = cy + sib.len().div_ceil(per_row) as f32 * 28.0; - // ---- recent projects. + // ---- recent projects (or the Learn tab's tutorials). let (rx, mut ry, rw) = if wide { let rx = x0 + left_w + 40.0; (rx, rect.min.y + pad + 6.0, rect.max.x - pad - rx) } else { (x0, community_bottom + 24.0, left_w) }; + if app.ui.home_learn { + super::learn::home_tab(app, ui, Rect::from_min_max(pos2(rx, ry), pos2(rx + rw, rect.max.y - 8.0))); + run_actions(app, &ctx, actions); + return; + } p.text(pos2(rx, ry + 6.0), Align2::LEFT_CENTER, "Recent", Tokens::semibold(16.0), Color32::WHITE); let entries = recent_entries(&app.session.prefs); if !entries.is_empty() { @@ -304,9 +327,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ry += row_h; } + run_actions(app, &ctx, actions); +} + +fn run_actions(app: &mut EffectcraftApp, ctx: &egui::Context, actions: Vec<(&str, serde_json::Value)>) { for (id, params) in actions { let before = (app.session.path.clone(), app.session.revision); - match crate::menus::invoke(app, &ctx, id, params) { + match crate::menus::invoke(app, ctx, id, params) { Err(e) => app.ui.status = e, // Leave Home unless only the list changed or a file dialog was cancelled. Ok(_) => { @@ -382,8 +409,46 @@ mod tests { "home.help.discord", "home.help.github", "home.recent.0", + "home.tab.home", + "home.tab.learn", ] { assert!(app.auto.find(id).is_some(), "{id}"); } } + + #[test] + fn learn_tab_lists_and_starts_tutorials() { + let mut app = EffectcraftApp::new(effectcraft_engine::Session::default()); + app.ui.start_screen = true; + app.ui.home_learn = true; + let ctx = egui::Context::default(); + crate::theme::install(&ctx, &app.tokens); + let frame = |app: &mut EffectcraftApp| { + ctx.run_ui(Default::default(), |ui| { + app.auto.begin_frame(); + show(app, ui, Rect::from_min_size(pos2(0.0, 0.0), vec2(1200.0, 800.0))); + crate::panels::learn::coach(app, ui.ctx()); + }) + .textures_delta + .clear(); + }; + frame(&mut app); + frame(&mut app); + for t in effectcraft_engine::learn::tutorials() { + assert!(app.auto.find(&format!("home.learn.{}.start", t.id)).is_some(), "{}", t.id); + } + assert!(app.auto.find("home.recent.0").is_none()); + // Start through the session (as the Start button does); the coach card appears. + app.session.execute("learn.start", json!({"id": "animate-title"})).unwrap(); + app.ui.start_screen = false; + frame(&mut app); + frame(&mut app); + for id in ["learn.coach", "learn.showMe", "learn.next", "learn.close"] { + assert!(app.auto.find(id).is_some(), "{id}"); + } + // Show me performs the step and the coach moves on. + app.session.execute("learn.step", json!({"action": "showMe"})).unwrap(); + assert_eq!(app.session.learn.as_ref().unwrap().step, 1); + assert!(app.session.active_comp().is_some()); + } } diff --git a/crates/ui-egui/src/panels/learn.rs b/crates/ui-egui/src/panels/learn.rs new file mode 100644 index 00000000..a184e230 --- /dev/null +++ b/crates/ui-egui/src/panels/learn.rs @@ -0,0 +1,199 @@ +//! Learn: the Home screen's tutorial list and the coach card that walks through a running +//! tutorial (engine model in `effectcraft_engine::learn`). The coach highlights the step's target +//! (an automation id) and moves on when the user performs the step's command; "Show me" runs it. + +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde_json::json; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The Learn tab of the Home screen: one card per tutorial with its steps and a Start button. +pub fn home_tab(app: &mut EffectcraftApp, ui: &mut egui::Ui, area: Rect) { + let t = app.tokens; + let p = ui.painter().with_clip_rect(area); + let (x, mut y, w) = (area.min.x, area.min.y, area.width()); + p.text(pos2(x, y + 6.0), Align2::LEFT_CENTER, "Learn", Tokens::semibold(16.0), Color32::WHITE); + p.text( + pos2(x, y + 26.0), + Align2::LEFT_CENTER, + "Hands-on tutorials that run inside EffectCraft. Do each step yourself, or press Show me.", + Tokens::ui(11.5), + t.text_faint, + ); + y += 44.0; + let tutorials = effectcraft_engine::learn::tutorials(); + let running = app.session.learn.as_ref().map(|l| l.tutorial.clone()); + let card_h = 128.0; + for tut in &tutorials { + let r = Rect::from_min_size(pos2(x, y), vec2(w, card_h - 10.0)); + if r.min.y > area.max.y { + break; + } + let resp = ui.interact(r, egui::Id::new(("learn-card", &tut.id)), Sense::click()); + p.rect_filled(r, 8.0, if resp.hovered() { Color32::from_rgb(0x2c, 0x2e, 0x37) } else { Color32::from_rgb(0x24, 0x26, 0x2e) }); + // A small original badge: the step count in a circle. + let badge = Rect::from_min_size(pos2(r.min.x + 16.0, r.min.y + 16.0), vec2(44.0, 44.0)); + p.circle_filled(badge.center(), 22.0, t.accent.gamma_multiply(0.25)); + p.circle_stroke(badge.center(), 22.0, Stroke::new(1.5, t.accent)); + p.text(badge.center() - vec2(0.0, 5.0), Align2::CENTER_CENTER, tut.steps.len().to_string(), Tokens::semibold(15.0), Color32::WHITE); + p.text(badge.center() + vec2(0.0, 10.0), Align2::CENTER_CENTER, "steps", Tokens::ui(9.0), t.text_dim); + let tx = badge.max.x + 16.0; + p.text(pos2(tx, r.min.y + 22.0), Align2::LEFT_CENTER, &tut.title, Tokens::semibold(14.0), Color32::WHITE); + p.text(pos2(tx, r.min.y + 40.0), Align2::LEFT_CENTER, format!("{} min", tut.minutes), Tokens::ui(11.0), t.text_faint); + let text_w = (r.max.x - tx - 120.0).max(120.0); + let g = p.layout(tut.summary.clone(), Tokens::ui(12.0), t.text_dim, text_w); + p.galley(pos2(tx, r.min.y + 52.0), g, t.text_dim); + let steps: Vec = tut.steps.iter().map(|s| s.title.clone()).collect(); + let g = p.layout(steps.join(" · "), Tokens::ui(10.5), t.text_faint, text_w); + p.galley(pos2(tx, r.min.y + 84.0), g, t.text_faint); + let br = Rect::from_min_size(pos2(r.max.x - 104.0, r.min.y + 16.0), vec2(88.0, 28.0)); + let label = if running.as_deref() == Some(tut.id.as_str()) { "Resume" } else { "Start" }; + let clicked = widgets::text_button(ui, br, label, true, &t, egui::Id::new(("learn-start", &tut.id))).clicked(); + app.auto.add(&format!("home.learn.{}", tut.id), r, &tut.title); + app.auto.add(&format!("home.learn.{}.start", tut.id), br, label); + if clicked || resp.clicked() { + if running.as_deref() != Some(tut.id.as_str()) + && let Err(e) = app.session.execute("learn.start", json!({"id": tut.id})) + { + app.ui.status = e.to_string(); + } + app.ui.start_screen = false; + } + y += card_h; + } +} + +/// The automation element a step's target pattern points at (last frame's registry). +fn target_rect(app: &EffectcraftApp, pattern: &str) -> Option { + let lid = app.session.state.selected_layers.first().map(|l| l.0); + let pat = effectcraft_engine::learn::resolve_target(pattern, lid); + app.auto + .previous + .iter() + .find(|e| effectcraft_engine::learn::target_matches(&pat, &e.id) && e.rect[2] > 0.0 && e.rect[3] > 0.0) + .map(|e| Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3]))) +} + +/// The coach card (bottom right) and the highlight around the current step's target. +pub fn coach(app: &mut EffectcraftApp, ctx: &egui::Context) { + let Some((tut, i)) = app.session.learn_current() else { return }; + let t = app.tokens; + let n = tut.steps.len(); + let screen = ctx.content_rect(); + let step = tut.steps.get(i).cloned(); + // Highlight the target (pulsing outline over everything but the card). + if app.ui.start_screen { + // The Home screen covers the panels: just the card. + } else if let Some(st) = &step + && let Some(tp) = &st.target + && let Some(r) = target_rect(app, tp) + { + let time = ctx.input(|i| i.time); + let pulse = (0.5 + 0.5 * (time * 4.0).sin()) as f32; + let p = ctx.layer_painter(egui::LayerId::new(egui::Order::Foreground, egui::Id::new("learn-highlight"))); + let hr = r.expand(3.0 + 2.0 * pulse); + p.rect_stroke(hr, 5.0, Stroke::new(2.0, t.accent.gamma_multiply(0.6 + 0.4 * pulse)), StrokeKind::Outside); + app.auto.add("learn.highlight", hr, tp); + ctx.request_repaint_after(std::time::Duration::from_millis(33)); + } + let w = 340.0; + let mut action: Option<&str> = None; + let area = egui::Area::new(egui::Id::new("learn-coach")) + .order(egui::Order::Foreground) + .fixed_pos(pos2(screen.max.x - w - 20.0, screen.max.y - 250.0)) + .show(ctx, |ui| { + egui::Frame::new() + .fill(Color32::from_rgb(0x26, 0x28, 0x30)) + .stroke(Stroke::new(1.0, t.accent.gamma_multiply(0.7))) + .corner_radius(8.0) + .inner_margin(egui::Margin::same(14)) + .shadow(egui::epaint::Shadow { offset: [0, 4], blur: 16, spread: 0, color: Color32::from_black_alpha(140) }) + .show(ui, |ui| { + ui.set_width(w - 28.0); + ui.horizontal(|ui| { + ui.label(egui::RichText::new("LEARN").font(Tokens::semibold(10.0)).color(t.accent)); + ui.label(egui::RichText::new(&tut.title).font(Tokens::ui(11.0)).color(t.text_dim)); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let (cr, r) = ui.allocate_exact_size(vec2(16.0, 16.0), Sense::click()); + let r = r.on_hover_text("Close tutorial"); + crate::icons::paint(ui.painter(), cr, crate::icons::Icon::Close, if r.hovered() { Color32::WHITE } else { t.text_dim }); + app.auto.add("learn.close", r.rect, "Close tutorial"); + if r.clicked() { + action = Some("close"); + } + }); + }); + // Progress dots. + let (dr, _) = ui.allocate_exact_size(vec2(w - 28.0, 10.0), Sense::hover()); + for k in 0..n { + let c = pos2(dr.min.x + 4.0 + k as f32 * 12.0, dr.center().y); + let col = if k < i { + t.accent + } else if k == i { + Color32::WHITE + } else { + Color32::from_gray(80) + }; + ui.painter().circle_filled(c, 3.0, col); + } + ui.add_space(4.0); + match &step { + Some(st) => { + ui.label(egui::RichText::new(format!("Step {} of {n}: {}", i + 1, st.title)).font(Tokens::semibold(13.0)).color(Color32::WHITE)); + ui.add_space(2.0); + ui.label(egui::RichText::new(&st.text).font(Tokens::ui(12.0)).color(t.text)); + } + None => { + ui.label(egui::RichText::new("Tutorial complete").font(Tokens::semibold(13.0)).color(Color32::WHITE)); + ui.label( + egui::RichText::new("Nice work. Pick another tutorial on the Home screen's Learn tab.").font(Tokens::ui(12.0)).color(t.text), + ); + } + } + ui.add_space(8.0); + ui.horizontal(|ui| { + let back = ui.add_enabled(i > 0, egui::Button::new("Back")); + app.auto.add("learn.back", back.rect, "Back"); + if back.clicked() { + action = Some("back"); + } + if step.is_some() { + let show = ui.add(egui::Button::new("Show me")); + app.auto.add("learn.showMe", show.rect, "Show me"); + if show.clicked() { + action = Some("showMe"); + } + let next = ui.add(egui::Button::new("Next")); + app.auto.add("learn.next", next.rect, "Next"); + if next.clicked() { + action = Some("next"); + } + } else { + let more = ui.add(egui::Button::new("More tutorials")); + app.auto.add("learn.more", more.rect, "More tutorials"); + if more.clicked() { + action = Some("more"); + } + } + }); + }); + }); + app.auto.add("learn.coach", area.response.rect, &tut.title); + match action { + Some("close") => { + let _ = app.session.execute("learn.stop", json!({})); + } + Some("more") => { + let _ = app.session.execute("learn.stop", json!({})); + app.ui.home_learn = true; + app.ui.start_screen = true; + } + Some(a) => { + if let Err(e) = app.session.execute("learn.step", json!({"action": a})) { + app.ui.status = e.to_string(); + } + } + None => {} + } +} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index d9a9864a..ed766f0d 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -22,6 +22,7 @@ pub mod info; pub mod key_dialogs; pub mod layer_panel; pub mod layer_styles_dialog; +pub mod learn; pub mod markers_ui; pub mod media_panels; pub mod misc; diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 584cdd95..50031da7 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -597,6 +597,9 @@ pub struct UiState { #[serde(default = "yes")] pub preview_audio: bool, pub start_screen: bool, + /// The Home screen shows its Learn tab (tutorials) instead of the recent projects. + #[serde(default)] + pub home_learn: bool, /// Composition ▸ Preview ▸ Cache Frames When Idle. pub cache_when_idle: bool, /// Tracker panel ▸ Motion Source chosen without a tracker yet (layer id). @@ -666,6 +669,7 @@ impl Default for UiState { preview_cache_first: false, preview_audio: true, start_screen: false, + home_learn: false, cache_when_idle: false, tracker_source: None, layer_panel: None, From da6cc851b40a090912e1015f296de914efd38f76 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:31:59 -0400 Subject: [PATCH 34/36] =?UTF-8?q?Parity=20audit=20at=20978e8d7:=20?= =?UTF-8?q?=E2=89=8894%=20weighted=20(98%=20with=20fractions),=2080/12/0,?= =?UTF-8?q?=20=E2=89=884=E2=80=935=20h=20wall=20clock=20left?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 14 ++++++++------ docs/parity.md | 28 +++++++++++++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 7c7b337b..c02dd556 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,12 +25,14 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (3 October 2026): -- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of - professional daily-use features, 65% of the long tail); all 298 After Effects effects. -- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude - Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. -- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. +- **≈ 94% of After Effects' features, weighted by importance**, counting partial features as half + done (≈ 98% with per-feature fractions): 80 of 92 features done, 12 partial, none missing; all + 298 After Effects effects (42 still simplified). +- **≈ 19–22 agent-hours of work remain**: about **4–5 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 3 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's threading, wider GPU effect coverage, Advanced 3D + completion, and learned-model quality for Roto Brush and face tracking. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/docs/parity.md b/docs/parity.md index fce24ea3..54be9809 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -2,18 +2,28 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -## Current status (audit at commit `fa26ad9`, 2 October 2026, late) +## Current status (audit at commit `978e8d7`, 3 October 2026) | Measure | Value | |---|---| -| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | -| Unweighted | ≈ 91% | -| P0 / P1 / P2 | ≈ 96% / 90% / 65% | -| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | -| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | -| Disabled menu entries left | 26 commands (31 menu items) | -| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | -| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 94%** counting every partial feature as half done; ≈ 98% using per-feature fractions | +| Unweighted | ≈ 94% (half-credit) / 98% (fractions) | +| P0 / P1 / P2 | ≈ 96% / 92% / 85% (half-credit); 99% / 97% / 96% (fractions) | +| Features done / partial / missing | 80 / 12 / 0 of 92 | +| **Effects** | **298 of 298** After Effects 2026 effects exist; 42 still simplified ([effects.md](effects.md)); 58 run on the GPU | +| Disabled menu entries left | 0 (the stub list is empty) | +| Remaining work | ≈ 19–22 agent-hours at the pace measured so far (≈ 58 on the conservative audit scale) | +| **Wall-clock estimate** | **≈ 4–5 hours** with five agents in parallel; ≈ 3 hours for 100% of P0 + P1 | + +Partial features: effect implementations (42 simplified effects), the Preview panel's settings +(range, play from, skip, frame rate, full screen), GPU coverage (Advanced 3D, more GPU effects), +the web app (no threads for viewer frames and Roto Brush), the Learn panel and Align to Selection / +Distribute, Advanced 3D (iris shapes, collapsed precomps, extruded strokes), ScriptUI resource +strings and `onDraw`, Essential Graphics font and scale controls, and the learned-model quality of +Roto Brush and face tracking (both are classical). Menu items still absent: Save Frame As ▸ +Photoshop Layers / ProEXR, Open in Essential Graphics, Vanishing Point import, Watch Folder. + +### Previous audit (commit `fa26ad9`, 2 October 2026, late): ≈ 93% By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, From 40a498d6a2c73d649d53e7b92d3d21c4f79f3e48 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:36:59 -0400 Subject: [PATCH 35/36] M13.5: UI fidelity pass (compact rows, flat rows with hairlines, switch wells, selected name cells, muted bars, AE ruler/work-area layout, twirl chevrons, timecode size, workspace bar order, single viewer breadcrumb row) Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/src/header.rs | 4 +- crates/ui-egui/src/panels/project.rs | 25 +++--- crates/ui-egui/src/panels/timeline.rs | 111 ++++++++++++++++++-------- crates/ui-egui/src/panels/viewer.rs | 9 ++- crates/ui-egui/src/theme.rs | 2 +- crates/ui-egui/src/widgets.rs | 10 ++- 6 files changed, 111 insertions(+), 50 deletions(-) diff --git a/crates/ui-egui/src/header.rs b/crates/ui-egui/src/header.rs index b2fa142e..d185b52c 100644 --- a/crates/ui-egui/src/header.rs +++ b/crates/ui-egui/src/header.rs @@ -168,7 +168,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.set_workspace(crate::dock::WORKSPACES[i]); } rx = more.min.x - 6.0; - let shown = ["Small Screen", "Standard", "Animation", "Default"]; + // After Effects 2026's workspace bar order (right to left here): Default, Review, Learn, + // Small Screen, Standard. + let shown = ["Standard", "Small Screen", "Learn", "Review", "Default"]; for name in shown { let g = p.layout_no_wrap(name.to_string(), Tokens::ui(12.0), t.text); let w = g.size().x + 16.0; diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index f3898954..8c89f25f 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -14,6 +14,9 @@ use crate::panels::DragPayload; use crate::theme::Tokens; use crate::{EffectcraftApp, widgets}; +/// Item row height (After Effects' Project panel rows are compact, about 19 pt). +const ROW_H: f32 = 19.0; + fn item_icon(it: &Item) -> Icon { match &it.kind { ItemKind::Folder => Icon::Folder, @@ -323,8 +326,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let mut actions: Vec<(String, serde_json::Value)> = vec![]; for (i, (id, depth)) in rows.iter().enumerate() { let Some(it) = app.session.project.item(*id).cloned() else { continue }; - let r = Rect::from_min_size(pos2(list.min.x, y), vec2(list.width(), 22.0)); - y += 22.0; + let r = Rect::from_min_size(pos2(list.min.x, y), vec2(list.width(), ROW_H)); + y += ROW_H; if r.min.y > list.max.y { break; } @@ -335,14 +338,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.rect_filled( r, 0.0, - if selected { - t.row_selected - } else if i % 2 == 0 { - t.row - } else { - t.row_alt - }, + // Flat rows like After Effects' Project panel (no stripes). + if selected { t.row_selected } else { t.row }, ); + let _ = i; let x0 = r.min.x + 8.0 + 14.0 * *depth as f32; if it.is_folder() { let open = app.ui.project_open_folders.contains(&id.0); @@ -400,12 +399,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { true }; if !cell_edit(app, ui, &mut actions, "name", name_rect) { + // The selected item's name sits in a light cell with dark text. + if selected { + let g = lp.layout_no_wrap(it.name.clone(), Tokens::ui(12.0), Color32::BLACK); + let cell = Rect::from_min_max(pos2(x0 + 27.0, r.min.y + 1.0), pos2((x0 + 33.0 + g.size().x).min(name_clip.max.x), r.max.y - 1.0)); + lp.with_clip_rect(name_clip.expand2(vec2(3.0, 0.0))).rect_filled(cell, 0.0, Color32::from_rgb(0xa6, 0xa6, 0xa6)); + } lp.with_clip_rect(name_clip).text( pos2(x0 + 30.0, r.center().y), Align2::LEFT_CENTER, &it.name, Tokens::ui(12.0), - if selected { Color32::WHITE } else { t.text }, + if selected { Color32::from_gray(0x16) } else { t.text }, ); } // Label swatch: click for the label colour menu. diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 80c1a36c..9b3d6aac 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -47,6 +47,22 @@ enum RowKind { /// Synthetic group uid for a layer's Audio > Waveform twirl (never a real property uid). const WAVE_BIT: u64 = 1 << 60; +/// Time navigator's visible-span bar and the work area bar. +const NAV_BAR: Color32 = Color32::from_rgb(0x55, 0x55, 0x55); +const WORK_AREA_BAR: Color32 = Color32::from_rgb(0x5c, 0x5c, 0x5c); +/// Fill of the timeline's switch / A/V boxes (dark wells, as in After Effects). +const SWITCH_BOX: Color32 = Color32::from_rgb(0x19, 0x19, 0x19); +/// Background of the selected layer's name cell. +const SELECTED_NAME_BG: Color32 = Color32::from_rgb(0xa6, 0xa6, 0xa6); + +/// A layer bar's fill: the label colour muted toward grey (a little brighter when selected). +pub(crate) fn bar_color(label: Color32, selected: bool) -> Color32 { + let k = if selected { 0.62 } else { 0.45 }; + let g = 0x3a as f32; + let mix = |c: u8| (c as f32 * k + g * (1.0 - k)).round() as u8; + Color32::from_rgb(mix(label.r()), mix(label.g()), mix(label.b())) +} + /// Row height (expression editors grow with their text). fn row_height(row: &Row, rh: f32) -> f32 { match row.kind { @@ -756,7 +772,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let tc = crate::panels::timecode(&app.session, &comp, time); let tc_rect = Rect::from_min_size(pos2(rect.min.x + 12.0, top + 6.0), vec2(150.0, 24.0)); let tc_resp = ui.interact(tc_rect, egui::Id::new("tl-timecode"), Sense::click_and_drag()); - p.text(tc_rect.left_center(), Align2::LEFT_CENTER, &tc, Tokens::semibold(20.0), t.timecode); + p.text(tc_rect.left_center(), Align2::LEFT_CENTER, &tc, Tokens::semibold(16.0), t.timecode); let sub = format!("{:05} ({:.2} fps)", fr.frame_at(time) + app.session.project.settings.frame_start, fr.as_f64()); p.text(pos2(tc_rect.min.x, tc_rect.max.y + 8.0), Align2::LEFT_CENTER, sub, Tokens::ui(10.5), t.text_faint); app.auto.add("timeline.timecode", tc_rect, &tc); @@ -770,7 +786,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } if let Some(mut buf) = ctx.data(|d| d.get_temp::(egui::Id::new("tl-tc-edit"))) { let mut child = ui.new_child(egui::UiBuilder::new().max_rect(tc_rect)); - let r = child.add(egui::TextEdit::singleline(&mut buf).font(Tokens::semibold(18.0)).desired_width(150.0)); + let r = child.add(egui::TextEdit::singleline(&mut buf).font(Tokens::semibold(16.0)).desired_width(150.0)); r.request_focus(); if r.lost_focus() { let _ = app.session.execute("time.set", json!({"timecode": buf})); @@ -810,14 +826,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // ---- ruler (right). let ruler = Rect::from_min_max(pos2(graph_x0, top + header_h - ruler_h), pos2(rect.max.x, top + header_h)); - let nav = Rect::from_min_max(pos2(graph_x0 + 6.0, top + 6.0), pos2(graph_x1, top + 12.0)); - // Time navigator. - p.rect_filled(nav, 3.0, t.field_bg); + let nav = Rect::from_min_max(pos2(graph_x0 + 6.0, top + 6.0), pos2(graph_x1, top + 14.0)); + // Time navigator: the visible span as a grey bar with blue end handles (as in After + // Effects), over a dark track. + p.rect_filled(nav, 1.0, t.field_bg); let vis0 = (tm.start / comp.duration.seconds()) as f32; let vis1 = (tm.t(graph_x1) / comp.duration.seconds()) as f32; let nav_vis = Rect::from_min_max(pos2(nav.min.x + nav.width() * vis0.clamp(0.0, 1.0), nav.min.y), pos2(nav.min.x + nav.width() * vis1.clamp(0.0, 1.0), nav.max.y)); - p.rect_filled(nav_vis, 3.0, t.work_area); + p.rect_filled(nav_vis, 0.0, NAV_BAR); + for x in [nav_vis.min.x, nav_vis.max.x] { + p.rect_filled(Rect::from_min_max(pos2(x - 3.0, nav.min.y - 1.0), pos2(x + 3.0, nav.max.y + 1.0)), 3.0, t.accent); + } let nresp = ui.interact(nav, egui::Id::new("tl-nav"), Sense::drag()); if nresp.dragged() { let d = nresp.drag_delta().x as f64 / nav.width() as f64 * comp.duration.seconds(); @@ -828,12 +848,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } p.rect_filled(ruler, 0.0, t.tl_ruler_bg); app.auto.add("timeline.ruler", ruler, &format!("{},{}", tm.start, tm.pps)); - // Work area bar. - let wa = Rect::from_min_max(pos2(tm.x(comp.work_area.0.seconds()), ruler.min.y + 2.0), pos2(tm.x(comp.work_area.1.seconds()), ruler.min.y + 10.0)); - p.rect_filled(wa, 2.0, t.work_area); + // Work area bar: below the ruler, beside the column headers (After Effects' layout), with + // blue begin / end handles; the cache bar runs under it. + let wa_row = Rect::from_min_max(pos2(graph_x0, top + header_h), pos2(rect.max.x, top + header_h + colhdr_h)); + let wa = Rect::from_min_max(pos2(tm.x(comp.work_area.0.seconds()), wa_row.min.y + 3.0), pos2(tm.x(comp.work_area.1.seconds()), wa_row.max.y - 6.0)); + p.with_clip_rect(wa_row).rect_filled(wa, 0.0, WORK_AREA_BAR); for (hx, set) in [(wa.min.x, "begin"), (wa.max.x, "end")] { - let hr = Rect::from_center_size(pos2(hx, wa.center().y), vec2(8.0, 12.0)); - p.rect_filled(Rect::from_center_size(pos2(hx, wa.center().y), vec2(3.0, 10.0)), 1.0, t.text_dim); + let hr = Rect::from_center_size(pos2(hx, wa.center().y), vec2(8.0, 14.0)); + let handle = if set == "begin" { + Rect::from_min_max(pos2(hx - 1.0, wa.min.y), pos2(hx + 4.0, wa.max.y)) + } else { + Rect::from_min_max(pos2(hx - 4.0, wa.min.y), pos2(hx + 1.0, wa.max.y)) + }; + p.rect_filled(handle, 2.0, t.accent); let resp = ui.interact(hr, egui::Id::new(("wa", set)), Sense::drag()); app.auto.add(&format!("timeline.workArea.{set}"), hr, set); if resp.dragged() @@ -848,7 +875,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let scale_key = app.viewer_shown.as_ref().map(|(_, k)| k.scale).unwrap_or(1000); let cached = app.frames.cached_frames(app.session.revision, cid.0, scale_key); let fd = comp.frame_duration().seconds(); - let cy0 = ruler.min.y + 11.0; + let cy0 = wa_row.max.y - 4.0; let mut run: Option<(i64, i64)> = None; let draw_run = |a: i64, b: i64| { let x0 = tm.x(a as f64 * fd); @@ -1100,19 +1127,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cy = r.min.y + rh / 2.0; match &row.kind { RowKind::Layer => { - lp.rect_filled( - left, - 0.0, - if is_sel { - t.row_selected - } else if ri % 2 == 0 { - t.row - } else { - t.row_alt - }, - ); - gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, if is_sel { Color32::from_rgb(0x26, 0x26, 0x26) } else { t.tl_bg }); - lp.line_segment([pos2(rect.min.x, r.max.y), pos2(graph_x0, r.max.y)], Stroke::new(1.0, Color32::from_black_alpha(80))); + // After Effects draws layer rows as one flat tone separated by dark hairlines + // (outline and time graph alike), not as alternating stripes. + let _ = ri; + lp.rect_filled(left, 0.0, if is_sel { t.row_selected } else { t.row_alt }); + gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, if is_sel { Color32::from_rgb(0x2a, 0x2a, 0x2a) } else { t.tl_bg }); + lp.line_segment([pos2(rect.min.x, r.max.y - 0.5), pos2(graph_x0, r.max.y - 0.5)], Stroke::new(1.0, t.app_bg)); + gp.line_segment([pos2(graph_x0, r.max.y - 0.5), pos2(r.max.x, r.max.y - 0.5)], Stroke::new(1.0, t.app_bg)); // A/V features. let sw = &layer.switches; let av_items: [(Icon, bool, &str, bool); 4] = [ @@ -1122,11 +1143,11 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { (Icon::Lock, sw.locked, "lock", true), ]; for (i, (icon, on, name, applicable)) in av_items.into_iter().enumerate() { - let br = Rect::from_center_size(pos2(cw.av + 10.0 + 18.0 * i as f32, cy), vec2(17.0, 17.0)); + let br = Rect::from_center_size(pos2(cw.av + 10.0 + 18.0 * i as f32, cy), vec2(15.0, 15.0)); if !applicable || !vis.av { continue; } - lp.rect_stroke(br.shrink(1.5), 2.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); + lp.rect_filled(br.shrink(0.5), 1.0, SWITCH_BOX); let resp = widgets::icon_toggle(ui, br, icon, on, &t, egui::Id::new(("av", layer.id.0, i)), None); app.auto.add(&format!("timeline.layer.{}.{name}", layer.id.0), br, name); if resp.clicked() { @@ -1176,7 +1197,16 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ctx.data_mut(|d| d.insert_temp(rename_id, (rl, buf))); } } else { - let name_col = if is_sel { Color32::WHITE } else { t.text }; + // The selected layer's name sits in a light cell with dark text. + let name_col = if is_sel { Color32::from_gray(0x16) } else { t.text }; + if is_sel { + let g = lp.layout_no_wrap(display_name(app, layer), Tokens::ui(12.0), name_col); + let cell = Rect::from_min_max( + pos2(name_rect.min.x - 3.0, r.min.y + 1.0), + pos2((name_rect.min.x + g.size().x + 4.0).max(cw.name_end - 4.0).min(cw.name_end - 2.0), r.max.y - 1.0), + ); + lp.rect_filled(cell, 0.0, SELECTED_NAME_BG); + } let lpn = lp.with_clip_rect(name_rect.intersect(lp.clip_rect())); lpn.text(pos2(name_rect.min.x, cy), Align2::LEFT_CENTER, display_name(app, layer), Tokens::ui(12.0), name_col); } @@ -1223,7 +1253,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { continue; } let br = Rect::from_center_size(pos2(cw.switches + SW * i as f32 + SW / 2.0, cy), vec2(16.0, 16.0)); - lp.rect_stroke(br.shrink(1.5), 2.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); + lp.rect_filled(br.shrink(1.0), 1.0, SWITCH_BOX); let resp = widgets::icon_toggle(ui, br, icon, on, &t, egui::Id::new(("sw", layer.id.0, i)), None); app.auto.add(&format!("timeline.layer.{}.switch.{name}", layer.id.0), br, name); if resp.clicked() { @@ -1321,12 +1351,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if !graph_on { let x_in = tm.x(layer.in_point.seconds()); let x_out = tm.x(layer.out_point.seconds()); - let bar = Rect::from_min_max(pos2(x_in, r.min.y + 3.0), pos2(x_out, r.max.y - 3.0)); + let bar = Rect::from_min_max(pos2(x_in, r.min.y + 1.0), pos2(x_out, r.max.y - 1.0)); let lc = t.label(layer.label); - let fill = if is_sel { lc.gamma_multiply(0.85) } else { lc.gamma_multiply(0.55) }; - gp.rect_filled(bar, 2.0, fill); + let fill = bar_color(lc, is_sel); + gp.rect_filled(bar, 0.0, fill); if is_sel { - gp.rect_stroke(bar, 2.0, Stroke::new(1.0, Color32::from_white_alpha(120)), StrokeKind::Inside); + gp.rect_stroke(bar, 0.0, Stroke::new(1.0, Color32::from_white_alpha(90)), StrokeKind::Inside); } // Source extent (e.g. trimmed footage) shown faint. if let LayerSource::Footage { .. } | LayerSource::Comp { .. } = layer.source @@ -1437,7 +1467,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } if let Some(en) = eye { // Layer style eye switch in the A/V column, like AE. - let er = Rect::from_center_size(pos2(cw.av + 10.0, cy), vec2(17.0, 17.0)); + let er = Rect::from_center_size(pos2(cw.av + 10.0, cy), vec2(15.0, 15.0)); lp.rect_stroke(er.shrink(1.5), 2.0, Stroke::new(1.0, t.separator), StrokeKind::Inside); if widgets::icon_toggle(ui, er, Icon::Eye, *en, &t, egui::Id::new(("geye", uid)), None).clicked() { actions.push(("layer.style.toggle".into(), json!({"layer": layer.id.0, "style": uid}))); @@ -2610,4 +2640,17 @@ mod tests { let c = snap_candidates(&comp, &build_rows(&app, &comp)); assert!(c.contains(&comp.duration.seconds()) && c.contains(&0.0)); } + + /// UI fidelity (M13.5): compact After Effects-like rows and muted label-coloured bars. + #[test] + fn rows_and_bars_follow_after_effects_proportions() { + let t = crate::theme::Tokens::for_kind(crate::theme::ThemeKind::Dark); + assert_eq!(t.row_h, 19.0); + let red = t.label(effectcraft_engine::color::Label::Red); + let bar = bar_color(red, false); + // Muted: darker and less saturated than the label, still reddish. + assert!(bar.r() < red.r() && bar.r() > bar.g() + 30 && bar.g() >= red.g().min(0x3a) - 1); + let sel = bar_color(red, true); + assert!(sel.r() > bar.r()); + } } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 3d1f8de4..b8ec8d41 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -404,11 +404,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let p = ui.painter().clone(); VIEW_CAM.with(|c| c.set(app.session.view_camera(cid))); - // Navigator bar (open comps as breadcrumbs). - let nav = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); + // Navigator bar (open comps as breadcrumbs). After Effects shows a single breadcrumb row: + // when the Composition Navigator (nested comp flow) is above the viewer, this row is left + // out. + let nav_h = if super::precomp::has_flow(app) { 0.0 } else { 24.0 }; + let nav = Rect::from_min_size(rect.min, vec2(rect.width(), nav_h)); p.rect_filled(nav, 0.0, t.panel_bg); let mut x = nav.min.x + 10.0; - let open = app.session.state.open_comps.clone(); + let open = if nav_h > 0.0 { app.session.state.open_comps.clone() } else { vec![] }; for oc in open { let name = app.session.project.item(oc).map(|i| i.name.clone()).unwrap_or_default(); let g = p.layout_no_wrap(name.clone(), Tokens::ui(11.5), t.text); diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index eb02bf48..288a04b5 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -145,7 +145,7 @@ impl Tokens { radius_sm: 3.0, gap: 4.0, tab_h: 30.0, - row_h: 22.0, + row_h: 19.0, gradients: true, }; match kind { diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index 2af57f21..9703d6ba 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -87,7 +87,15 @@ pub fn swatch(ui: &mut Ui, rect: Rect, c: [f32; 4], id: egui::Id, t: &Tokens) -> pub fn twirl(ui: &mut Ui, rect: Rect, open: bool, id: egui::Id, t: &Tokens) -> Response { let resp = ui.interact(rect, id, Sense::click()); let col = if resp.hovered() { t.tab_text_active } else { t.text_dim }; - icons::paint(ui.painter(), rect.shrink(rect.width() * 0.22), if open { Icon::ChevronDown } else { Icon::ChevronRight }, col); + // An open chevron (> / v), as After Effects draws its twirl-downs. + let c = rect.center(); + let k = rect.width().min(rect.height()) * 0.2; + let pts = if open { + vec![c + vec2(-1.6 * k, -0.8 * k), c + vec2(0.0, 0.8 * k), c + vec2(1.6 * k, -0.8 * k)] + } else { + vec![c + vec2(-0.8 * k, -1.6 * k), c + vec2(0.8 * k, 0.0), c + vec2(-0.8 * k, 1.6 * k)] + }; + ui.painter().add(egui::Shape::line(pts, Stroke::new(1.4, col))); resp } From 93aa098a29f945cf165d80af6a2aa387397ac56f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:37:24 -0400 Subject: [PATCH 36/36] M13.5: docs (parity: extended viewer, Learn tutorials, UI fidelity) Co-Authored-By: Claude Opus 5.5 --- docs/parity.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index 3974630f..1cb71322 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -116,9 +116,9 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 94% | 0.5 | no extruded strokes, variable-axis animation changes outlines but not advances (OpenType features — stylistic sets, discretionary ligatures, contextual / stylistic alternates, swash, titling, ordinals, fractions, figure styles, true small caps / all small caps and superior / inferior glyphs with faux fallback — per character with the Character panel's OpenType popup and `text.fontFeatures` landed in M13.2; vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | -| 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer; collapsed precomps of another size seen through the parent's camera render (fixed in M13.2); stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | +| 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer for Advanced 3D comps (Classic 3D Extended Viewer landed in M13.5 UI completion); collapsed precomps of another size seen through the parent's camera render (fixed in M13.2); stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | | Effects | ≈ 80% | 4.0 | GPU versions of the remaining effects (58 run on the GPU since M12.7) and the missing controls listed as partial in [effects.md](effects.md) (every After Effects effect exists since M9.11, M12.5 and M12.6; parameter names, order, twirl-downs, popups, units and defaults were aligned in M9.12) | -| Interface | 74% | 5.0 | a richer Learn area (Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | +| Interface | 74% | 5.0 | more Learn tutorials and pixel-level fidelity of dialogs (the Home ▸ Learn tab with interactive tutorials and a UI fidelity pass landed in M13.5 UI completion; Timeline outline and Project panel columns scroll horizontally, the Layer Style dialog, ROI resize handles, Pan Behind snapping and 3D Reference Axes landed in M13.5; native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | @@ -163,6 +163,24 @@ style runs; the Layer Style dialog (LYR-9); Graph Editor snapping to markers and Reference Axes; Animate Text ▸ Variable Font Axes. Remaining in these rows: Lottie can't carry taper/wave, Advanced 3D's DOF has no iris shapes, and variable axes don't change advances. +## Update: M13.5 UI completion & fidelity + +**Extended Viewer** (Settings ▸ 3D, the viewer's Extended Viewer button, `view.extendedViewer`): +custom 3D views (and the Active Camera view with Draft 3D on) render the visible pasteboard +around the comp frame (the frame is outlined) so 3D layers reaching past the frame stay +visible; Classic 3D planes project through the offset canvas natively, Advanced 3D comps show +the frame only. **Home ▸ Learn**: three original interactive tutorials (Animate a title, Track +and attach, Make a 3D scene) whose steps highlight their UI target by automation id, advance when +the user runs the step's command, and can be performed with Show me (`learn.list`, +`learn.start`, `learn.step`, `learn.stop`, `learn.state`). **UI fidelity pass** against After +Effects 2026 captures: compact 19 pt Timeline / Project rows, flat rows with dark hairlines, +dark switch wells, the selected layer / item name in a light cell, label-coloured bars muted +toward grey, the time navigator with blue end handles and the work area bar (with the cache +bar under it) below the ruler, open twirl chevrons, a 16 pt current-time display, the +workspace bar order (Default, Review, Learn, Small Screen, Standard) and a single breadcrumb row +above the viewer. Not yet: Composition Settings / Render Queue / Settings dialogs compared +pixel by pixel (no reference captures of them yet). + ## Update: M13.6 panels and content tools The last placeholder panels became real panels with automation ids: **Lumetri Scopes**