mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-31 05:27:08 -06:00
Brandon ThomasandClaude Opus 5.5
e57e6a5120
Tracker panel (Window > Tracker): Track Camera / Warp Stabilizer (disabled), Track Motion / Stabilize Motion, Motion Source, Current Track, Track Type, Position/Rotation/Scale, Motion Target with Edit Target..., Options... (Motion Tracker Options dialog), Analyze (1 frame back, back, forward, 1 frame forward; stop while running) with progress, Reset and Apply (Apply Dimensions dialog). Track point widgets in the viewer: feature and search regions with corner handles, attach point crosshair and track path, all draggable through track.setPoint (one undo step per drag). Automation ids for every widget; headless UI test. Tracker: featureless regions no longer drift (ties keep the prediction, only confident matches predict motion). docs/agents.md: tracking example. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
257 lines
11 KiB
Rust
257 lines
11 KiB
Rust
//! Synthetic footage: a textured patch moving along a known path (with rotation and scale) over a
|
||
//! textured background, with noise. Ground truth is exact, so recovered positions are compared
|
||
//! against it directly.
|
||
|
||
use super::*;
|
||
|
||
/// Smooth, band-limited texture (periods of 9–40 px).
|
||
fn texture(x: f64, y: f64, seed: u32) -> [f32; 3] {
|
||
let mut v = [0.0f64; 3];
|
||
let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345);
|
||
let mut rnd = || {
|
||
s ^= s << 13;
|
||
s ^= s >> 17;
|
||
s ^= s << 5;
|
||
(s % 10_000) as f64 / 10_000.0
|
||
};
|
||
for _ in 0..7 {
|
||
let ang = rnd() * std::f64::consts::TAU;
|
||
let period = 9.0 + rnd() * 31.0;
|
||
let phase = rnd() * std::f64::consts::TAU;
|
||
let k = std::f64::consts::TAU / period;
|
||
let w = (k * (x * ang.cos() + y * ang.sin()) + phase).sin();
|
||
let c = [rnd(), rnd(), rnd()];
|
||
for i in 0..3 {
|
||
v[i] += w * (0.3 + c[i]) * 0.12;
|
||
}
|
||
}
|
||
v.map(|q| (0.5 + q).clamp(0.0, 1.0) as f32)
|
||
}
|
||
|
||
fn noise(x: u32, y: u32, f: u32) -> f32 {
|
||
effectcraft_raster::hash_noise(x, y.wrapping_add(f.wrapping_mul(7919)), 99) - 0.5
|
||
}
|
||
|
||
/// Patch pose on a frame: centre, rotation (degrees), scale.
|
||
#[derive(Clone, Copy)]
|
||
struct Pose {
|
||
c: [f64; 2],
|
||
rot: f64,
|
||
scale: f64,
|
||
}
|
||
|
||
impl Pose {
|
||
/// Where the patch-local point `u` lands in the frame.
|
||
fn map(&self, u: [f64; 2]) -> [f64; 2] {
|
||
let (s, c) = self.rot.to_radians().sin_cos();
|
||
[self.c[0] + self.scale * (c * u[0] - s * u[1]), self.c[1] + self.scale * (s * u[0] + c * u[1])]
|
||
}
|
||
fn local(&self, p: [f64; 2]) -> [f64; 2] {
|
||
let (s, c) = self.rot.to_radians().sin_cos();
|
||
let d = [(p[0] - self.c[0]) / self.scale, (p[1] - self.c[1]) / self.scale];
|
||
[c * d[0] + s * d[1], -s * d[0] + c * d[1]]
|
||
}
|
||
}
|
||
|
||
/// Render a frame: background texture, the patch (half size `half`) on top, Gaussian-ish noise
|
||
/// of amplitude `amp`, and an optional flat occluder rectangle.
|
||
fn frame(w: u32, h: u32, pose: Pose, half: f64, amp: f32, f: u32, occluder: Option<[f64; 4]>) -> Image {
|
||
let mut img = Image::new(w, h);
|
||
let ss = [0.25, 0.75];
|
||
for y in 0..h {
|
||
for x in 0..w {
|
||
let mut acc = [0.0f32; 3];
|
||
for sy in ss {
|
||
for sx in ss {
|
||
let p = [x as f64 + sx, y as f64 + sy];
|
||
let l = pose.local(p);
|
||
let t = if l[0].abs() <= half && l[1].abs() <= half { texture(l[0], l[1], 7) } else { texture(p[0] * 0.7, p[1] * 0.7, 3).map(|v| v * 0.6) };
|
||
for i in 0..3 {
|
||
acc[i] += t[i] * 0.25;
|
||
}
|
||
}
|
||
}
|
||
let n = noise(x, y, f) * amp;
|
||
let mut px = [acc[0] + n, acc[1] + n, acc[2] + n, 1.0];
|
||
if let Some([ox, oy, ow, oh]) = occluder {
|
||
let (cx, cy) = (x as f64 + 0.5, y as f64 + 0.5);
|
||
if cx >= ox && cx < ox + ow && cy >= oy && cy < oy + oh {
|
||
px = [0.4, 0.4, 0.4, 1.0];
|
||
}
|
||
}
|
||
img.set(x, y, px);
|
||
}
|
||
}
|
||
img
|
||
}
|
||
|
||
fn spec(c: [f64; 2], feature: f64, search: f64) -> PointSpec {
|
||
PointSpec { center: c, feature_size: [feature, feature], search_offset: [0.0, 0.0], search_size: [search, search] }
|
||
}
|
||
|
||
fn dist(a: [f64; 2], b: [f64; 2]) -> f64 {
|
||
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt()
|
||
}
|
||
|
||
#[test]
|
||
fn recovers_translation_path_subpixel() {
|
||
let path = |f: usize| {
|
||
let t = f as f64;
|
||
Pose { c: [100.0 + 4.37 * t + 6.0 * (t * 0.3).sin(), 90.0 + 2.71 * t - 3.0 * (t * 0.21).cos()], rot: 0.0, scale: 1.0 }
|
||
};
|
||
let f0 = frame(320, 240, path(0), 40.0, 0.02, 0, None);
|
||
let mut tr = Tracker::new(TrackOptions::default(), &[spec(path(0).c, 31.0, 71.0)], &Frame::new(&f0));
|
||
let mut worst: f64 = 0.0;
|
||
for f in 1..24 {
|
||
let img = frame(320, 240, path(f), 40.0, 0.02, f as u32, None);
|
||
let r = tr.step(&Frame::new(&img))[0];
|
||
let e = dist(r.center, path(f).c);
|
||
worst = worst.max(e);
|
||
assert!(r.confidence > 90.0, "frame {f}: confidence {}", r.confidence);
|
||
assert_eq!(r.status, Status::Tracked);
|
||
}
|
||
eprintln!("worst error {worst:.4} px");
|
||
assert!(worst < 0.25, "worst error {worst} px");
|
||
}
|
||
|
||
#[test]
|
||
fn layer_offset_and_channels() {
|
||
let pose = |f: usize| Pose { c: [120.0 + 3.3 * f as f64, 100.0 - 1.7 * f as f64], rot: 0.0, scale: 1.0 };
|
||
for channel in Channel::ALL {
|
||
for (blur, enhance) in [(0.0, false), (3.0, false), (0.0, true)] {
|
||
let opts = TrackOptions { channel, blur, enhance, ..Default::default() };
|
||
let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None);
|
||
// Layer coordinates are image pixels minus the offset.
|
||
let off = [10.0, -5.0];
|
||
let c0 = [pose(0).c[0] - off[0], pose(0).c[1] - off[1]];
|
||
let mut tr = Tracker::new(opts, &[spec(c0, 29.0, 61.0)], &Frame { img: &f0, offset: off });
|
||
for f in 1..6 {
|
||
let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, None);
|
||
let r = tr.step(&Frame { img: &img, offset: off })[0];
|
||
let want = [pose(f).c[0] - off[0], pose(f).c[1] - off[1]];
|
||
assert!(dist(r.center, want) < 0.3, "{channel:?} blur {blur} enhance {enhance} frame {f}: {:?} vs {want:?}", r.center);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn two_points_recover_rotation_and_scale() {
|
||
let pose = |f: usize| Pose { c: [160.0 + 1.5 * f as f64, 120.0 + 0.8 * f as f64], rot: 1.8 * f as f64, scale: 1.0 + 0.01 * f as f64 };
|
||
let u = [[-26.0, 0.0], [26.0, 0.0]];
|
||
let p0 = pose(0);
|
||
let f0 = frame(320, 240, p0, 52.0, 0.01, 0, None);
|
||
let mut tr = Tracker::new(TrackOptions::default(), &u.map(|q| spec(p0.map(q), 23.0, 51.0)), &Frame::new(&f0));
|
||
let (a0, l0) = angle_and_length(p0.map(u[0]), p0.map(u[1]));
|
||
let mut worst: f64 = 0.0;
|
||
for f in 1..20 {
|
||
let p = pose(f);
|
||
let img = frame(320, 240, p, 52.0, 0.01, f as u32, None);
|
||
let r = tr.step(&Frame::new(&img));
|
||
for k in 0..2 {
|
||
worst = worst.max(dist(r[k].center, p.map(u[k])));
|
||
}
|
||
let (a, l) = angle_and_length(r[0].center, r[1].center);
|
||
assert!((unwrap_degrees(0.0, a - a0) - p.rot).abs() < 0.5, "frame {f}: rotation {} vs {}", a - a0, p.rot);
|
||
assert!((l / l0 - p.scale).abs() < 0.01, "frame {f}: scale {} vs {}", l / l0, p.scale);
|
||
// The per-feature shape estimate follows too.
|
||
assert!((r[0].rotation - p.rot).abs() < 1.0, "frame {f}: feature rotation {}", r[0].rotation);
|
||
assert!((r[0].scale - p.scale).abs() < 0.02, "frame {f}: feature scale {}", r[0].scale);
|
||
}
|
||
eprintln!("worst error {worst:.4} px");
|
||
assert!(worst < 0.25, "worst error {worst} px");
|
||
}
|
||
|
||
#[test]
|
||
fn confidence_drops_on_occlusion_and_actions() {
|
||
let pose = |f: usize| Pose { c: [110.0 + 2.0 * f as f64, 100.0], rot: 0.0, scale: 1.0 };
|
||
let occ = |f: usize| (f == 5).then(|| [pose(f).c[0] - 12.0, pose(f).c[1] - 30.0, 40.0, 60.0]);
|
||
let run = |action: ConfidenceAction| {
|
||
let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None);
|
||
let opts = TrackOptions { action, ..Default::default() };
|
||
let mut tr = Tracker::new(opts, &[spec(pose(0).c, 31.0, 61.0)], &Frame::new(&f0));
|
||
(1..9)
|
||
.map(|f| {
|
||
let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, occ(f));
|
||
tr.step(&Frame::new(&img))[0]
|
||
})
|
||
.collect::<Vec<_>>()
|
||
};
|
||
let r = run(ConfidenceAction::Continue);
|
||
assert!(r[3].confidence > 95.0, "{}", r[3].confidence);
|
||
assert!(r[4].confidence < 70.0, "occluded frame confidence {}", r[4].confidence);
|
||
let r = run(ConfidenceAction::Extrapolate);
|
||
assert_eq!(r[4].status, Status::Extrapolated);
|
||
assert!(dist(r[4].center, pose(5).c) < 0.5, "extrapolated {:?}", r[4].center);
|
||
// Back on track after the occlusion.
|
||
assert!(dist(r[6].center, pose(7).c) < 0.25 && r[6].status == Status::Tracked);
|
||
let r = run(ConfidenceAction::Stop);
|
||
assert_eq!(r[4].status, Status::Stopped);
|
||
let r = run(ConfidenceAction::Adapt);
|
||
assert_eq!(r[4].status, Status::Adapted);
|
||
}
|
||
|
||
#[test]
|
||
fn backward_and_whole_pixel() {
|
||
let pose = |f: usize| Pose { c: [100.0 + 3.25 * f as f64, 100.0 + 1.5 * f as f64], rot: 0.0, scale: 1.0 };
|
||
let last = 6;
|
||
let fl = frame(240, 200, pose(last), 36.0, 0.0, 0, None);
|
||
let opts = TrackOptions { subpixel: false, ..Default::default() };
|
||
let mut tr = Tracker::new(opts, &[spec(pose(last).c, 31.0, 61.0)], &Frame::new(&fl));
|
||
for f in (0..last).rev() {
|
||
let img = frame(240, 200, pose(f), 36.0, 0.0, 0, None);
|
||
let r = tr.step(&Frame::new(&img))[0];
|
||
let want = pose(f).c;
|
||
// Whole-pixel displacements of the start position.
|
||
let d = [r.center[0] - pose(last).c[0], r.center[1] - pose(last).c[1]];
|
||
assert!((d[0] - d[0].round()).abs() < 1e-9 && (d[1] - d[1].round()).abs() < 1e-9);
|
||
assert!(dist(r.center, want) < 0.75, "frame {f}: {:?} vs {want:?}", r.center);
|
||
}
|
||
}
|
||
|
||
/// 1080p, one point: tracking cost per frame (frames are generated outside the timing).
|
||
#[test]
|
||
fn performance_1080p_single_point() {
|
||
let w = 1920;
|
||
let h = 1080;
|
||
// A cheap textured frame (the full synthetic renderer is slow at 1080p in debug builds).
|
||
let make = |dx: f64, dy: f64| {
|
||
let mut img = Image::new(w, h);
|
||
for y in 0..h {
|
||
for x in 0..w {
|
||
let (fx, fy) = (x as f64 + 0.5 - dx, y as f64 + 0.5 - dy);
|
||
let v = 0.5 + 0.2 * (fx * 0.31).sin() * (fy * 0.17).cos() + 0.15 * ((fx + fy) * 0.11).sin() + 0.1 * ((fx * 0.05).sin() * (fy * 0.23).sin());
|
||
img.set(x, y, [v as f32, v as f32, v as f32, 1.0]);
|
||
}
|
||
}
|
||
img
|
||
};
|
||
let frames: Vec<Image> = (0..6).map(|f| make(2.3 * f as f64, 1.1 * f as f64)).collect();
|
||
let c0 = [960.0, 540.0];
|
||
let s = PointSpec { center: c0, feature_size: [48.0, 48.0], search_offset: [0.0; 2], search_size: [128.0, 128.0] };
|
||
let t0 = std::time::Instant::now();
|
||
let mut tr = Tracker::new(TrackOptions::default(), &[s], &Frame::new(&frames[0]));
|
||
let mut last = PointResult { center: c0, confidence: 0.0, rotation: 0.0, scale: 1.0, status: Status::Tracked };
|
||
for f in frames.iter().skip(1) {
|
||
last = tr.step(&Frame::new(f))[0];
|
||
}
|
||
let per = t0.elapsed().as_secs_f64() / 5.0;
|
||
eprintln!("1080p single-point track: {:.2} ms/frame ({:.0} fps)", per * 1e3, 1.0 / per);
|
||
assert!(dist(last.center, [c0[0] + 2.3 * 5.0, c0[1] + 1.1 * 5.0]) < 0.25, "{:?}", last.center);
|
||
assert!(per < 0.1, "{per} s per frame");
|
||
}
|
||
|
||
#[test]
|
||
fn featureless_regions_stay_put() {
|
||
let flat = Image::filled(200, 160, [0.3, 0.3, 0.3, 1.0]);
|
||
for action in ConfidenceAction::ALL {
|
||
let opts = TrackOptions { action, ..Default::default() };
|
||
let mut tr = Tracker::new(opts, &[spec([100.0, 80.0], 21.0, 61.0)], &Frame::new(&flat));
|
||
for _ in 0..10 {
|
||
let r = tr.step(&Frame::new(&flat))[0];
|
||
assert!(dist(r.center, [100.0, 80.0]) < 1e-9, "{action:?}: drifted to {:?}", r.center);
|
||
assert!(r.confidence < 1.0);
|
||
}
|
||
}
|
||
}
|