From a39cee8104248d368ca1146060d120b946cdae6a Mon Sep 17 00:00:00 2001 From: Kristian Myram Date: Thu, 15 Oct 2026 22:03:29 +0200 Subject: [PATCH] S2.51: bench --roto --fixes counts the corrections each tracker needs Simulated user paints a foreground stroke through the true object and a background stroke on the biggest false positive on the first frame under IoU 0.9, up to 6 rounds. SAM 2 needs none on every clip; classical and MobileSAM need more than 6 on occlusion and re-entry. All trackers now take strokes on any frame. Co-Authored-By: Claude Sonnet 5.5 --- apps/effectcraft-cli/src/main.rs | 16 +++-- crates/track/src/roto/bench.rs | 108 ++++++++++++++++++++++++++++--- docs/roto-brush-sam2.md | 15 +++++ 3 files changed, 124 insertions(+), 15 deletions(-) diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 27710a4a2..be066ecd1 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -27,7 +27,7 @@ //! (--gpu: CPU vs GPU ms/frame for every comp at Full and Half) //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings //! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel) -//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--json] Roto Brush trackers on synthetic clips with known mattes +//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--json] Roto Brush trackers on synthetic clips with known mattes //! (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame) //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script //! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio @@ -126,7 +126,7 @@ const USAGE: &str = "usage: effectcraft-cli Result<(), Failure> { if sam2.is_none() { note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column"); } + // `--fixes`: also count the correction strokes each needs before every frame reaches IoU 0.9 (up to 6). + let fixes = args.flag("--fixes"); + let measure = |sc: &bench::Scenario, run: &dyn Fn(&bench::Scenario) -> Option| if fixes { bench::with_fixes(sc, 0.9, 6, run) } else { run(sc) }; let mut runs = vec![]; for sc in bench::scenarios(frames) { - runs.push(bench::run_flow(&sc, "classical", None)); + runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "classical", None)))); if let Some(m) = &mobilesam { - runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref()))); + runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref()))))); } - if let Some(run) = sam2.as_ref().and_then(|m| bench::run_tracker(&sc, "sam2-tiny", m.as_ref())) { - runs.push(run); + if let Some(m) = &sam2 { + runs.extend(measure(&sc, &|s| bench::run_tracker(s, "sam2-tiny", m.as_ref()))); } } if args.flag("--json") { diff --git a/crates/track/src/roto/bench.rs b/crates/track/src/roto/bench.rs index 2ce0663df..58aea2385 100644 --- a/crates/track/src/roto/bench.rs +++ b/crates/track/src/roto/bench.rs @@ -11,13 +11,14 @@ use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest}; use serde::Serialize; use web_time::Instant; -use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt}; +use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt_boxed}; pub const W: u32 = 200; pub const H: u32 = 150; const R: f64 = 32.0; /// A clip with the true matte of the visible object on every frame. +#[derive(Clone)] pub struct Scenario { pub name: &'static str, pub what: &'static str, @@ -42,6 +43,12 @@ pub struct Run { /// Mean IoU over the frames after the object's last absence (re-appearance), if it had one. pub after_return: Option, pub per_frame: Vec, + /// Correction strokes needed before every frame reaches the target IoU (`>N`: not within N); set by + /// [`with_fixes`], `None` when not measured. + pub fixes: Option, + /// The matte of every frame (not serialised). + #[serde(skip)] + pub mattes: Vec>, } fn noise(x: f64, y: f64, seed: u32) -> f64 { @@ -182,6 +189,8 @@ fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec], secs: f64) -> Run { secs_per_frame: secs / n, after_return, per_frame, + fixes: None, + mattes: mattes.to_vec(), } } @@ -190,7 +199,7 @@ fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec], secs: f64) -> Run { pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> Run { let opts = SegOpts::default(); let start = Instant::now(); - let refs: Vec<&Stroke> = sc.strokes.iter().collect(); + let refs: Vec<&Stroke> = sc.strokes.iter().filter(|s| s.frame == sc.base as i64).collect(); let mut segs: Vec> = vec![None; sc.frames.len()]; if let Some(first) = sc.frames.get(sc.base) { let base = segment_with(first, &refs, None, &opts, 0.0, model); @@ -201,7 +210,8 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> for f in sc.base + 1..sc.frames.len() { let (Some(prev_img), Some(img)) = (sc.frames.get(f - 1), sc.frames.get(f)) else { break }; let Some(Some(prev)) = segs.get(f - 1) else { break }; - let next = propagate_with(prev_img, prev, img, &[], &opts, 0.0, model); + let here: Vec<&Stroke> = sc.strokes.iter().filter(|s| s.frame == f as i64).collect(); + let next = propagate_with(prev_img, prev, img, &here, &opts, 0.0, model); if let Some(slot) = segs.get_mut(f) { *slot = Some(next); } @@ -218,16 +228,23 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Option { let t = model.tracker()?; let opts = SegOpts::default(); - let refs: Vec<&Stroke> = sc.strokes.iter().collect(); + let at = |f: usize| -> Vec<&Stroke> { sc.strokes.iter().filter(|s| s.frame == f as i64).collect() }; let clip: Vec> = sc.frames.iter().map(|i| Arc::new(ClipFrame { rgb: rgb_of(i), w: i.width as usize, h: i.height as usize })).collect(); - let req = TrackRequest::single(0, sc.frames.len(), vec![(sc.base as i64, tracker_prompt(&refs, 1.0))]); + // a prompt per stroke frame: a few clicks and the box of the classic matte (as Chain::track does) + let mut prompts = vec![]; + for f in 0..sc.frames.len() { + let strokes = at(f); + let Some(img) = sc.frames.get(f).filter(|_| strokes.iter().any(|s| s.kind == StrokeKind::Fg)) else { continue }; + prompts.push((f as i64, tracker_prompt_boxed(&strokes, 1.0, &super::segment(img, &strokes, None, &opts, 0.0)))); + } + let req = TrackRequest::single(0, sc.frames.len(), prompts); let start = Instant::now(); let mut mattes: Vec> = vec![vec![0u8; (W * H) as usize]; sc.frames.len()]; let mut get = |f: i64| usize::try_from(f).ok().and_then(|i| clip.get(i).cloned()); let r = t.track(&req, &mut get, &|| false, &mut |_, f, tf| { let (Ok(i), true) = (usize::try_from(f), tf.prob.len() == (W * H) as usize) else { return }; let Some(img) = sc.frames.get(i) else { return }; - let strokes: Vec<&Stroke> = if i == sc.base { refs.clone() } else { vec![] }; + let strokes = at(i); if let Some(slot) = mattes.get_mut(i) { *slot = segment_tracked(img, &strokes, tf.prob, tf.occluded, &opts, 0.0).matte; } @@ -236,14 +253,74 @@ pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Optio Some(metrics(sc, tracker, &mattes, start.elapsed().as_secs_f64())) } +/// The correction a user would paint on frame `f` where `pred` is wrong: a short foreground stroke through +/// the true object's centre, and a background stroke on the biggest false-positive blob's centre. +fn correction(sc: &Scenario, f: usize, pred: &[u8]) -> Vec { + let truth = sc.truth.get(f).map(Vec::as_slice).unwrap_or(&[]); + let centre = |keep: &dyn Fn(usize) -> bool| -> Option<[f64; 2]> { + let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0); + for i in (0..truth.len().min(pred.len())).filter(|i| keep(*i)) { + sx += (i % W as usize) as f64 + 0.5; + sy += (i / W as usize) as f64 + 0.5; + n += 1.0; + } + (n > 0.0).then(|| [sx / n, sy / n]) + }; + let mut out = vec![]; + if let Some(c) = centre(&|i| truth.get(i).is_some_and(|v| *v != 0)) { + out.push(line(StrokeKind::Fg, f as i64, [c[0] - 12.0, c[1]], [c[0] + 12.0, c[1]], 5.0)); + } + if let Some(c) = centre(&|i| pred.get(i).is_some_and(|v| *v != 0) && truth.get(i).is_some_and(|v| *v == 0)) { + out.push(line(StrokeKind::Bg, f as i64, [c[0] - 6.0, c[1]], [c[0] + 6.0, c[1]], 6.0)); + } + out +} + +/// Repeat `run` on `sc`, each time painting the correction a user would on the first frame under `target` IoU, +/// until every frame reaches it or `max` corrections were needed. Sets `fixes` on the first run's result: the +/// number of corrections, or `>max`. +pub fn with_fixes(sc: &Scenario, target: f64, max: usize, run: &dyn Fn(&Scenario) -> Option) -> Option { + let mut cur = sc.clone(); + let mut first: Option = None; + for k in 0..=max { + let r = run(&cur)?; + let bad = r.per_frame.iter().position(|v| *v < target); + let first_run = first.get_or_insert_with(|| r.clone()); + let Some(f) = bad else { + first_run.fixes = Some(k.to_string()); + break; + }; + if k == max { + first_run.fixes = Some(format!(">{max}")); + break; + } + let pred = r.mattes.get(f).cloned().unwrap_or_default(); + let fix = correction(&cur, f, &pred); + if fix.is_empty() { + first_run.fixes = Some(format!(">{max}")); + break; + } + cur.strokes.extend(fix); + } + first +} + /// A fixed-width table of the runs, grouped by scenario. pub fn table(runs: &[Run]) -> String { - let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame"); + let mut out = + format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9} {:>6}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame", "fixes"); for r in runs { let back = r.after_return.map_or_else(|| "-".to_string(), |v| format!("{v:.3}")); out.push_str(&format!( - "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3}\n", - r.scenario, r.tracker, r.mean_iou, r.min_iou, r.flicker, back, r.secs_per_frame + "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3} {:>6}\n", + r.scenario, + r.tracker, + r.mean_iou, + r.min_iou, + r.flicker, + back, + r.secs_per_frame, + r.fixes.as_deref().unwrap_or("-") )); } out @@ -283,6 +360,19 @@ mod tests { assert!(table(&[run]).contains("classical")); } + #[test] + fn corrections_are_counted_until_every_frame_reaches_the_target() { + let all = scenarios(24); + let classical = |s: &Scenario| Some(run_flow(s, "classical", None)); + // plain motion needs none + let easy = all.iter().find(|s| s.name == "move").and_then(|s| with_fixes(s, 0.9, 6, &classical)).unwrap(); + assert_eq!(easy.fixes.as_deref(), Some("0")); + // after the disk has left and returned the classic engine has lost it: it needs a correction (or never gets there) + let hard = all.iter().find(|s| s.name == "reenter").and_then(|s| with_fixes(s, 0.9, 6, &classical)).unwrap(); + assert_ne!(hard.fixes.as_deref(), Some("0"), "{:?}", hard.fixes); + assert!(table(&[easy, hard]).contains("fixes")); + } + #[test] fn tiny_clip_requests_are_clamped() { assert!(scenarios(0).iter().all(|s| s.frames.len() == 12)); diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index d36d2086c..a08c8a5e7 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -103,3 +103,18 @@ per frame: MobileSAM prompts every frame independently from the flow-warped matte, so it follows the object while the flow does and loses it when it is hidden; SAM 2's streaming memory is what lets it re-acquire the object. The synthetic clips are easy (one textured disc), so the "move" and "light" rows mostly show all three working; the last two rows are the difference. + +### Corrections needed (`--fixes`) +`bench --roto --fixes` also counts the correction strokes each tracker needs before every frame reaches IoU 0.9: after each +run it paints what a user would on the first frame under the target (a short foreground stroke through the true object's +centre, and a background stroke on the biggest false-positive blob), up to 6. 16 frames, debug build: + +| Clip | Classical | MobileSAM | SAM 2.1 Tiny | +|---|---|---|---| +| move | 0 | 0 | 0 | +| light | 0 | 0 | 0 | +| occlusion | more than 6 | more than 6 | 0 | +| reenter | more than 6 | more than 6 | 0 | + +The simulated user is simple (one stroke pair per round, always on the first bad frame), so "more than 6" means this +correction model never recovers the flow-based engines on those clips, not that no human could. Peak memory is not measured.