//! Mask tracking, Mask Interpolation and the Warp Stabilizer end to end through the commands, on //! synthetic footage generated in-test (a rotating/scaling textured patch, a shaky camera). use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; use effectcraft_path::interp::paths_cross; use effectcraft_project::{LayerId, Uid}; use effectcraft_raster::Image; use effectcraft_render::RenderOpts; use effectcraft_time::Tick; use effectcraft_track::Homography; use serde_json::json; use crate::Session; use crate::tests_track::{FPS, H, W, frame_time, patch_frames, rotating_pose, setup, texture}; fn pose_h(f: u32) -> Homography { let (c, rot, sc) = rotating_pose(f); let (sn, cs) = rot.to_radians().sin_cos(); Homography([[sc * cs, -sc * sn, c[0]], [sc * sn, sc * cs, c[1]], [0.0, 0.0, 1.0]]) } fn mask_path(s: &Session, layer: LayerId, mask: Uid) -> effectcraft_project::Property { let l = s.active_comp().unwrap().layer(layer).unwrap(); l.props.find_group(mask).unwrap().get("path").unwrap().clone() } fn new_mask(s: &mut Session, layer: LayerId, path: &ShapePath) -> Uid { let ins: Vec<[f64; 2]> = path.in_tangents.clone(); let outs: Vec<[f64; 2]> = path.out_tangents.clone(); let r = s.execute("mask.new", json!({"layer": layer.0, "vertices": path.vertices, "inTangents": ins, "outTangents": outs, "closed": true})).unwrap(); r["mask"].as_u64().unwrap() } #[test] fn track_mask_follows_a_rotating_patch() { let (mut s, clip, _) = setup(patch_frames(rotating_pose)); let (c0, _, _) = rotating_pose(0); // An ellipse inside the textured patch. let shape = ShapePath::ellipse(c0, 64.0, 52.0); let uid = new_mask(&mut s, clip, &shape); // Selecting the mask enables Track Mask (the Tracker panel's mask mode). s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); s.state.selected_props = vec![(clip, uid)]; assert!(s.is_enabled("track.mask")); assert_eq!(crate::commands::mask_interp::selected_mask(&s), Some((clip, uid))); s.execute("time.set", json!({"time": 0})).unwrap(); let r = s.execute_checked("track.mask", json!({"method": "positionScaleRotation", "direction": "forward", "wait": true})).unwrap(); assert_eq!(r["frames"], 24, "{r}"); assert!(s.mask_job.is_none()); let pr = mask_path(&s, clip, uid); assert_eq!(pr.keys.len(), 25); let inv0 = pose_h(0).inverse().unwrap(); let mut worst: f64 = 0.0; for f in 0..25 { let KV::Path(p) = pr.value_at(frame_time(f)) else { panic!() }; let m = pose_h(f).then_after(&inv0); for (v, o) in p.vertices.iter().zip(&shape.vertices) { let want = m.apply(*o); worst = worst.max((v[0] - want[0]).hypot(v[1] - want[1])); } // Tangents rotate and scale with the patch. let want_t = m.apply_tangent(shape.vertices[0], shape.out_tangents[0]); assert!((p.out_tangents[0][0] - want_t[0]).hypot(p.out_tangents[0][1] - want_t[1]) < 1.5); } assert!(worst < 1.5, "worst vertex error {worst} px"); assert_eq!(s.time(), frame_time(24)); // One undo step; redo restores the keys. assert!(s.undo()); assert!(mask_path(&s, clip, uid).keys.is_empty()); assert!(s.redo()); assert_eq!(mask_path(&s, clip, uid).keys.len(), 25); // Track backward one frame from the end, with another method. let r = s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "position", "direction": "frameBackward", "wait": true})).unwrap(); assert_eq!(r["frames"], 1); assert_eq!(s.state.mask_track_method, crate::mask_track::MaskMethod::Position); // Serde keeps the keys. let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap(); assert_eq!(l.props.find_group(uid).unwrap().get("path").unwrap().keys.len(), 25); // Bad parameters are reported. assert!(s.execute_checked("track.mask", json!({"layer": clip.0, "mask": uid, "method": "warp"})).is_err()); assert!(s.execute_checked("track.mask", json!({"layer": clip.0, "mask": uid, "bogus": 1})).is_err()); } #[test] fn track_mask_in_background_and_cancel() { let (mut s, clip, _) = setup(patch_frames(rotating_pose)); let uid = new_mask(&mut s, clip, &ShapePath::rect(rotating_pose(0).0, 60.0, 60.0)); s.execute("time.set", json!({"time": 0})).unwrap(); let r = s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "perspective"})).unwrap(); assert_eq!(r["running"], true); // The analysis keeps running in the background; Stop keeps what was tracked. assert!(s.is_enabled("track.stop")); s.execute("track.stop", json!({})).unwrap(); let t0 = std::time::Instant::now(); while s.mask_job.is_some() { assert!(t0.elapsed().as_secs() < 60); std::thread::sleep(std::time::Duration::from_millis(2)); s.poll_mask_track(); } let n = mask_path(&s, clip, uid).keys.len(); assert!((1..25).contains(&n), "{n} keys"); // A full background run. s.execute("time.set", json!({"time": 0})).unwrap(); s.execute("track.mask", json!({"layer": clip.0, "mask": uid, "method": "perspective"})).unwrap(); while s.mask_job.is_some() { assert!(t0.elapsed().as_secs() < 120); std::thread::sleep(std::time::Duration::from_millis(5)); s.poll_mask_track(); } let pr = mask_path(&s, clip, uid); assert_eq!(pr.keys.len(), 25); let KV::Path(p) = pr.value_at(frame_time(20)) else { panic!() }; let m = pose_h(20).then_after(&pose_h(0).inverse().unwrap()); let want = m.apply(ShapePath::rect(rotating_pose(0).0, 60.0, 60.0).vertices[2]); assert!((p.vertices[2][0] - want[0]).hypot(p.vertices[2][1] - want[1]) < 2.0); } /// A five-pointed star (10 vertices) around `c`, rotated by `rot` degrees. fn star(c: [f64; 2], r: f64, rot: f64) -> ShapePath { let pts: Vec<[f64; 2]> = (0..10) .map(|i| { let a = (rot + i as f64 * 36.0 - 90.0).to_radians(); let rr = if i % 2 == 0 { r } else { r * 0.45 }; [c[0] + rr * a.cos(), c[1] + rr * a.sin()] }) .collect(); ShapePath::polygon(&pts, true) } #[test] fn mask_interpolation_creates_matching_keys() { let (mut s, clip, _) = setup(patch_frames(rotating_pose)); let a = ShapePath::rect([100.0, 100.0], 80.0, 60.0); let b = star([200.0, 120.0], 50.0, 30.0); let uid = new_mask(&mut s, clip, &a); let cid = s.active_comp_id().unwrap(); let (t0, t1) = (frame_time(0), frame_time(20)); s.edit("keys", None, |p, _| { let pr = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().props.find_group_mut(uid).unwrap().get_mut("path").unwrap(); pr.keys = vec![Keyframe::new(t0, KV::Path(a.clone())), Keyframe::new(t1, KV::Path(b.clone()))]; Ok(()) }) .unwrap(); let prop = mask_path(&s, clip, uid).uid; s.execute("keys.select", json!({"keys": [{"layer": clip.0, "prop": prop, "time": 0.0}, {"layer": clip.0, "prop": prop, "time": 0.8}]})).unwrap(); assert_eq!(s.state.selected_keys.len(), 2); let r = s .execute_checked( "mask.interpolate", json!({"keyframeRate": "auto", "bendingResistance": 50, "quality": 50, "addVertices": 10, "addVerticesUnit": "pixels", "matchingMethod": "auto", "firstVerticesMatch": false}), ) .unwrap(); // 19 in-betweens at the comp rate (frames 1…19). assert_eq!(r["keys"], 19, "{r}"); let pr = mask_path(&s, clip, uid); assert_eq!(pr.keys.len(), 21); let shapes: Vec = pr.keys.iter().map(|k| k.value.as_path().unwrap().clone()).collect(); let n = shapes[0].len(); assert!(n >= 28, "{n} vertices"); assert!(shapes.iter().all(|p| p.len() == n)); // Ends keep their outlines; vertex paths between consecutive keys don't cross. for (i, w) in shapes.windows(2).enumerate() { assert_eq!(paths_cross(&w[0], &w[1]), 0, "keys {i}/{}", i + 1); } assert!( shapes[0] .vertices .iter() .all(|v| (v[0] - 60.0).abs() < 1e-6 || (v[0] - 140.0).abs() < 1e-6 || (v[1] - 70.0).abs() < 1e-6 || (v[1] - 130.0).abs() < 1e-6) ); // Options persist; Keyframe Fields doubles the rate; undo restores. assert_eq!(s.state.mask_interp.add_vertices, Some(10.0)); assert!(s.undo()); assert_eq!(mask_path(&s, clip, uid).keys.len(), 2); let r = s .execute( "mask.interpolate", json!({"layer": clip.0, "mask": uid, "times": [0.0, 0.8], "keyframeRate": 10, "keyframeFields": true, "linearVertexPaths": true}), ) .unwrap(); assert_eq!(r["keys"], 15, "{r}"); assert!(s.execute("mask.interpolate", json!({"layer": clip.0, "mask": uid, "times": [0.0]})).is_err()); assert!(s.execute_checked("mask.interpolate", json!({"matchingMethod": "magic", "times": [0, 0.8], "layer": clip.0})).is_err()); } // ---------------------------------------------------------------- Warp Stabilizer /// Camera offset on frame `f`: a slow pan plus jitter. fn cam(f: u32) -> [f64; 2] { let j = [((f * 7919) % 9) as f64 - 4.0, ((f * 104_729) % 7) as f64 - 3.0]; [1.2 * f as f64 + j[0], 0.5 * f as f64 + j[1]] } fn shaky(f: u32) -> Image { let c = cam(f); let mut img = Image::new(W, H); for y in 0..H { for x in 0..W { let v = texture(x as f64 + 0.5 + c[0], y as f64 + 0.5 + c[1], 11); img.set(x, y, [v, v * 0.9, v * 0.8, 1.0]); } } img } fn jitter(p: &[[f64; 2]]) -> f64 { let mut s = 0.0; for k in 1..p.len() - 1 { s += (p[k + 1][0] - 2.0 * p[k][0] + p[k - 1][0]).hypot(p[k + 1][1] - 2.0 * p[k][1] + p[k - 1][1]); } s / (p.len() - 2) as f64 } fn warp_at(s: &mut Session, f: u32) -> Homography { s.execute("time.set", json!({"time": frame_time(f).seconds()})).unwrap(); let st = s.execute("warp.status", json!({})).unwrap(); let m: [[f64; 3]; 3] = serde_json::from_value(st["warp"].clone()).unwrap_or_else(|_| panic!("{st}")); Homography(m) } /// Output positions of the scene point that sits at the frame centre on frame 0. fn scene_track(s: &mut Session) -> (Vec<[f64; 2]>, Vec<[f64; 2]>) { let x0 = [160.0 + cam(0)[0], 120.0 + cam(0)[1]]; let before: Vec<[f64; 2]> = (0..25).map(|f| [x0[0] - cam(f)[0], x0[1] - cam(f)[1]]).collect(); let after = (0..25).map(|f| warp_at(s, f).apply(before[f as usize])).collect(); (before, after) } fn render(s: &Session, f: u32) -> Image { s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts::default()) } fn transparent(img: &Image) -> usize { img.data.iter().filter(|p| p[3] < 0.5).count() } fn set_param(s: &mut Session, layer: LayerId, path: &str, v: serde_json::Value) { s.execute("prop.set", json!({"layer": layer.0, "path": format!("effects/#1/{path}"), "value": v})).unwrap(); } #[test] fn warp_stabilizer_analyzes_and_stabilizes() { let (mut s, clip, solid) = setup(shaky); s.execute("edit.clear", json!({"layers": [solid.0]})).unwrap(); s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); // Effect ▸ Distort ▸ Warp Stabilizer, then analyse (blocking). let r = s.execute("effect.apply", json!({"effect": "Warp Stabilizer"})).unwrap(); let fx = r["effects"][0].as_u64().unwrap(); assert!(s.warp_pending.contains(&(s.active_comp_id().unwrap(), clip, fx)), "applying queues an analysis"); let st = s.execute_checked("warp.status", json!({})).unwrap(); assert_eq!(st["analyzed"], false); let r = s.execute_checked("warp.analyze", json!({"layer": clip.0, "wait": true})).unwrap(); assert_eq!(r["frames"], 25, "{r}"); assert_eq!(r["status"]["analyzed"], true, "{r}"); assert!(s.warp_pending.is_empty() && s.warp_job.is_none()); // Groups and AE parameter names. let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone(); let names: Vec<&str> = g.groups().map(|g| g.name.as_str()).collect(); assert_eq!(names, ["Stabilization", "Borders", "Advanced"]); assert!(g.prop("borders/autoScale/maximumScale").is_some()); assert_eq!(g.prop("stabilization/smoothness").unwrap().name, "Smoothness"); // Smooth Motion: the jitter of a scene point drops by more than 80 %. set_param(&mut s, clip, "borders/framing", json!(0)); let (before, after) = scene_track(&mut s); let (j0, j1) = (jitter(&before), jitter(&after)); assert!(j1 < 0.2 * j0, "jitter {j0} → {j1}"); // Rendering applies the frame's correction: output(p) = source(W⁻¹ p). let w = warp_at(&mut s, 7); let out = render(&s, 7); let src = shaky(7); let inv = w.inverse().unwrap(); for p in [[160.0, 120.0], [100.0, 80.0], [230.0, 170.0]] { let q = inv.apply([p[0] + 0.5, p[1] + 0.5]); let want = src.sample_bilinear(q[0], q[1]); let got = out.get(p[0] as i64, p[1] as i64); assert!((want[0] - got[0]).abs() < 0.03, "{p:?}: {want:?} vs {got:?}"); } // Stabilize Only shows the moving borders. let shown: usize = (0..25).map(|f| transparent(&render(&s, f))).sum(); assert!(shown > 500, "{shown}"); // No Motion keeps a feature fixed. set_param(&mut s, clip, "stabilization/result", json!(1)); let (_, fixed) = scene_track(&mut s); for q in &fixed { assert!((q[0] - fixed[12][0]).hypot(q[1] - fixed[12][1]) < 0.6, "{q:?} vs {:?}", fixed[12]); } set_param(&mut s, clip, "stabilization/result", json!(0)); // Stabilize, Crop, Auto-scale (the default) hides the borders; Crop leaves an opaque centre. set_param(&mut s, clip, "borders/framing", json!(2)); let st = s.execute("warp.status", json!({})).unwrap(); assert!(st["autoScale"].as_f64().unwrap() > 100.5, "{st}"); for f in 0..25 { assert_eq!(transparent(&render(&s, f)), 0, "frame {f}"); } set_param(&mut s, clip, "borders/framing", json!(1)); let st = s.execute("warp.status", json!({})).unwrap(); let crop: [f64; 4] = serde_json::from_value(st["crop"].clone()).unwrap(); for f in [0, 9, 24] { let img = render(&s, f); for y in (crop[1].ceil() as u32 + 1)..(crop[3].floor() as u32 - 1) { for x in (crop[0].ceil() as u32 + 1)..(crop[2].floor() as u32 - 1) { assert!(img.get(x as i64, y as i64)[3] > 0.99, "frame {f} ({x}, {y})"); } } assert!(img.get(0, 0)[3] < 0.01); } // Synthesize Edges fills most of the borders from neighbouring frames. set_param(&mut s, clip, "borders/framing", json!(3)); let synth: usize = (0..25).map(|f| transparent(&render(&s, f))).sum(); assert!(synth * 10 < shown, "synthesized {synth} vs {shown}"); // Show Track Points draws the features. set_param(&mut s, clip, "borders/framing", json!(0)); let plain = render(&s, 5); set_param(&mut s, clip, "advanced/showTrackPoints", json!(true)); assert_ne!(render(&s, 5).data, plain.data); // Serde keeps the analysis; undo / redo of the analysis. let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); let bg = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone(); assert_eq!(bg, s.active_comp().unwrap().layer(clip).unwrap().props.find_group(fx).unwrap().clone()); while s.history.undo.last().is_some_and(|(l, _)| l != "Warp Stabilizer Analysis") { s.undo(); } assert!(s.undo()); assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], false); assert!(s.redo()); assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); } #[test] fn warp_analysis_invalidates_and_cancels() { let (mut s, clip, solid) = setup(shaky); s.execute("edit.clear", json!({"layers": [solid.0]})).unwrap(); s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); // Animation ▸ Warp Stabilizer VFX applies and analyses. assert!(s.is_enabled("track.warpStabilizer")); let r = s.execute_checked("track.warpStabilizer", json!({"wait": true})).unwrap(); let fx = r["effect"].as_u64().unwrap(); let cid = s.active_comp_id().unwrap(); assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); // Trimming the layer clears the analysis and queues it again. s.execute("layer.timing", json!({"layers": [clip.0], "out": 0.8})).unwrap(); let st = s.execute("warp.status", json!({})).unwrap(); assert_eq!(st["analyzed"], false, "{st}"); assert_eq!(st["pending"], true); // Without an analysis the layer renders unstabilized. let img = s.render(cid, frame_time(3), RenderOpts::default()); assert_eq!(transparent(&img), 0); // Frontends start queued analyses in the background (poll_warp with auto). assert!(s.poll_warp(true)); assert!(s.warp_job.is_some()); let t0 = std::time::Instant::now(); while s.warp_job.is_some() { assert!(t0.elapsed().as_secs() < 120); let p = s.warp_progress().unwrap(); assert!(p.banner().starts_with("Analyzing in background (step 1 of 2)") || p.banner() == "Stabilizing..."); std::thread::sleep(std::time::Duration::from_millis(5)); s.poll_warp(true); } assert_eq!(s.execute("warp.status", json!({})).unwrap()["analyzed"], true); // Undoing the slip brings back the old analysis (the key matches again). // Cancel: nothing is written. s.execute("effect.apply", json!({"effect": "ec.distort.warpstabilizer", "layers": [clip.0]})).unwrap(); let fx2 = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().nth(1).unwrap().uid; let r = s.execute("warp.analyze", json!({"layer": clip.0, "effect": fx2})).unwrap(); assert_eq!(r["running"], true); assert!(s.execute("warp.analyze", json!({"layer": clip.0, "effect": fx})).is_err(), "one analysis at a time"); s.execute("warp.cancel", json!({})).unwrap(); while s.warp_job.is_some() { std::thread::sleep(std::time::Duration::from_millis(2)); s.poll_warp(false); } let st = s.execute("warp.status", json!({"layer": clip.0, "effect": fx2})).unwrap(); assert_eq!(st["analyzed"], false, "{st}"); // Time Remap changes invalidate too. s.execute("layer.enableTimeRemap", json!({"layers": [clip.0]})).unwrap(); assert_eq!(s.execute("warp.status", json!({"layer": clip.0, "effect": fx})).unwrap()["analyzed"], false); let _ = FPS; let _ = Tick::ZERO; }