S2.51: bench --roto --fixes counts the corrections each tracker needs

Simulated user paints a foreground stroke through the true object and a background
stroke on the biggest false positive on the first frame under IoU 0.9, up to 6
rounds. SAM 2 needs none on every clip; classical and MobileSAM need more than 6 on
occlusion and re-entry. All trackers now take strokes on any frame.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

apps/effectcraft-cli/src/main.rs

@@ -27,7 +27,7 @@
 //!     (--gpu: CPU vs GPU ms/frame for every comp at Full and Half)
 //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N]   everyday-operation timings
 //!     on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
-//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--json]   Roto Brush trackers on synthetic clips with known mattes
+//! effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt] [--fixes] [--json]   Roto Brush trackers on synthetic clips with known mattes
 //!     (classical engine, MobileSAM and SAM 2.1 Tiny when installed; IoU, flicker, re-appearance, seconds per frame)
 //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE    run an After Effects-style script
 //! effectcraft-cli mcp [--autosave | --bridge PORT]            MCP server on stdio
@@ -126,7 +126,7 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
   bench --ops [--small] [--layers N] [--comps N] [--footage N]
                                            everyday operations on a large generated project: startup, open,
                                            save, auto-save, edits + undo/redo, timeline, Project panel
-  bench --roto [--frames N] [--sam2 FILE] [--mobilesam FILE] [--json]
+  bench --roto [--frames N] [--sam2 FILE] [--mobilesam FILE] [--fixes] [--json]
                                            Roto Brush trackers on synthetic clips with known mattes: movement,
                                            lighting change, occlusion, leaving and re-entering the frame
   script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
@@ -195,6 +195,7 @@ const VALUED: &[&str] = &[
 /// Options without a value. Any other `--option` is a usage error.
 const FLAGS: &[&str] = &[
     "--json",
+    "--fixes",
     "--save",
     "--demo",
     "--empty",
@@ -1033,14 +1034,17 @@ fn bench_roto(args: &Args) -> Result<(), Failure> {
     if sam2.is_none() {
         note!("bench --roto: SAM 2.1 Tiny is not installed; no SAM 2 column");
     }
+    // `--fixes`: also count the correction strokes each needs before every frame reaches IoU 0.9 (up to 6).
+    let fixes = args.flag("--fixes");
+    let measure = |sc: &bench::Scenario, run: &dyn Fn(&bench::Scenario) -> Option<bench::Run>| if fixes { bench::with_fixes(sc, 0.9, 6, run) } else { run(sc) };
     let mut runs = vec![];
     for sc in bench::scenarios(frames) {
-        runs.push(bench::run_flow(&sc, "classical", None));
+        runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "classical", None))));
         if let Some(m) = &mobilesam {
-            runs.push(bench::run_flow(&sc, "mobilesam", Some(m.as_ref())));
+            runs.extend(measure(&sc, &|s| Some(bench::run_flow(s, "mobilesam", Some(m.as_ref())))));
         }
-        if let Some(run) = sam2.as_ref().and_then(|m| bench::run_tracker(&sc, "sam2-tiny", m.as_ref())) {
-            runs.push(run);
+        if let Some(m) = &sam2 {
+            runs.extend(measure(&sc, &|s| bench::run_tracker(s, "sam2-tiny", m.as_ref())));
         }
     }
     if args.flag("--json") {

crates/track/src/roto/bench.rs

@@ -11,13 +11,14 @@ use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest};
 use serde::Serialize;
 use web_time::Instant;
 
-use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt};
+use super::{FrameSeg, SegOpts, Stroke, StrokeKind, iou, propagate_with, rgb_of, segment_tracked, segment_with, tracker_prompt_boxed};
 
 pub const W: u32 = 200;
 pub const H: u32 = 150;
 const R: f64 = 32.0;
 
 /// A clip with the true matte of the visible object on every frame.
+#[derive(Clone)]
 pub struct Scenario {
     pub name: &'static str,
     pub what: &'static str,
@@ -42,6 +43,12 @@ pub struct Run {
     /// Mean IoU over the frames after the object's last absence (re-appearance), if it had one.
     pub after_return: Option<f64>,
     pub per_frame: Vec<f64>,
+    /// Correction strokes needed before every frame reaches the target IoU (`>N`: not within N); set by
+    /// [`with_fixes`], `None` when not measured.
+    pub fixes: Option<String>,
+    /// The matte of every frame (not serialised).
+    #[serde(skip)]
+    pub mattes: Vec<Vec<u8>>,
 }
 
 fn noise(x: f64, y: f64, seed: u32) -> f64 {
@@ -182,6 +189,8 @@ fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec<u8>], secs: f64) -> Run {
         secs_per_frame: secs / n,
         after_return,
         per_frame,
+        fixes: None,
+        mattes: mattes.to_vec(),
     }
 }
 
@@ -190,7 +199,7 @@ fn metrics(sc: &Scenario, tracker: &str, mattes: &[Vec<u8>], secs: f64) -> Run {
 pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) -> Run {
     let opts = SegOpts::default();
     let start = Instant::now();
-    let refs: Vec<&Stroke> = sc.strokes.iter().collect();
+    let refs: Vec<&Stroke> = sc.strokes.iter().filter(|s| s.frame == sc.base as i64).collect();
     let mut segs: Vec<Option<FrameSeg>> = vec![None; sc.frames.len()];
     if let Some(first) = sc.frames.get(sc.base) {
         let base = segment_with(first, &refs, None, &opts, 0.0, model);
@@ -201,7 +210,8 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) ->
     for f in sc.base + 1..sc.frames.len() {
         let (Some(prev_img), Some(img)) = (sc.frames.get(f - 1), sc.frames.get(f)) else { break };
         let Some(Some(prev)) = segs.get(f - 1) else { break };
-        let next = propagate_with(prev_img, prev, img, &[], &opts, 0.0, model);
+        let here: Vec<&Stroke> = sc.strokes.iter().filter(|s| s.frame == f as i64).collect();
+        let next = propagate_with(prev_img, prev, img, &here, &opts, 0.0, model);
         if let Some(slot) = segs.get_mut(f) {
             *slot = Some(next);
         }
@@ -218,16 +228,23 @@ pub fn run_flow(sc: &Scenario, tracker: &str, model: Option<&dyn MaskModel>) ->
 pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Option<Run> {
     let t = model.tracker()?;
     let opts = SegOpts::default();
-    let refs: Vec<&Stroke> = sc.strokes.iter().collect();
+    let at = |f: usize| -> Vec<&Stroke> { sc.strokes.iter().filter(|s| s.frame == f as i64).collect() };
     let clip: Vec<Arc<ClipFrame>> = sc.frames.iter().map(|i| Arc::new(ClipFrame { rgb: rgb_of(i), w: i.width as usize, h: i.height as usize })).collect();
-    let req = TrackRequest::single(0, sc.frames.len(), vec![(sc.base as i64, tracker_prompt(&refs, 1.0))]);
+    // a prompt per stroke frame: a few clicks and the box of the classic matte (as Chain::track does)
+    let mut prompts = vec![];
+    for f in 0..sc.frames.len() {
+        let strokes = at(f);
+        let Some(img) = sc.frames.get(f).filter(|_| strokes.iter().any(|s| s.kind == StrokeKind::Fg)) else { continue };
+        prompts.push((f as i64, tracker_prompt_boxed(&strokes, 1.0, &super::segment(img, &strokes, None, &opts, 0.0))));
+    }
+    let req = TrackRequest::single(0, sc.frames.len(), prompts);
     let start = Instant::now();
     let mut mattes: Vec<Vec<u8>> = vec![vec![0u8; (W * H) as usize]; sc.frames.len()];
     let mut get = |f: i64| usize::try_from(f).ok().and_then(|i| clip.get(i).cloned());
     let r = t.track(&req, &mut get, &|| false, &mut |_, f, tf| {
         let (Ok(i), true) = (usize::try_from(f), tf.prob.len() == (W * H) as usize) else { return };
         let Some(img) = sc.frames.get(i) else { return };
-        let strokes: Vec<&Stroke> = if i == sc.base { refs.clone() } else { vec![] };
+        let strokes = at(i);
         if let Some(slot) = mattes.get_mut(i) {
             *slot = segment_tracked(img, &strokes, tf.prob, tf.occluded, &opts, 0.0).matte;
         }
@@ -236,14 +253,74 @@ pub fn run_tracker(sc: &Scenario, tracker: &str, model: &dyn MaskModel) -> Optio
     Some(metrics(sc, tracker, &mattes, start.elapsed().as_secs_f64()))
 }
 
+/// The correction a user would paint on frame `f` where `pred` is wrong: a short foreground stroke through
+/// the true object's centre, and a background stroke on the biggest false-positive blob's centre.
+fn correction(sc: &Scenario, f: usize, pred: &[u8]) -> Vec<Stroke> {
+    let truth = sc.truth.get(f).map(Vec::as_slice).unwrap_or(&[]);
+    let centre = |keep: &dyn Fn(usize) -> bool| -> Option<[f64; 2]> {
+        let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0);
+        for i in (0..truth.len().min(pred.len())).filter(|i| keep(*i)) {
+            sx += (i % W as usize) as f64 + 0.5;
+            sy += (i / W as usize) as f64 + 0.5;
+            n += 1.0;
+        }
+        (n > 0.0).then(|| [sx / n, sy / n])
+    };
+    let mut out = vec![];
+    if let Some(c) = centre(&|i| truth.get(i).is_some_and(|v| *v != 0)) {
+        out.push(line(StrokeKind::Fg, f as i64, [c[0] - 12.0, c[1]], [c[0] + 12.0, c[1]], 5.0));
+    }
+    if let Some(c) = centre(&|i| pred.get(i).is_some_and(|v| *v != 0) && truth.get(i).is_some_and(|v| *v == 0)) {
+        out.push(line(StrokeKind::Bg, f as i64, [c[0] - 6.0, c[1]], [c[0] + 6.0, c[1]], 6.0));
+    }
+    out
+}
+
+/// Repeat `run` on `sc`, each time painting the correction a user would on the first frame under `target` IoU,
+/// until every frame reaches it or `max` corrections were needed. Sets `fixes` on the first run's result: the
+/// number of corrections, or `>max`.
+pub fn with_fixes(sc: &Scenario, target: f64, max: usize, run: &dyn Fn(&Scenario) -> Option<Run>) -> Option<Run> {
+    let mut cur = sc.clone();
+    let mut first: Option<Run> = None;
+    for k in 0..=max {
+        let r = run(&cur)?;
+        let bad = r.per_frame.iter().position(|v| *v < target);
+        let first_run = first.get_or_insert_with(|| r.clone());
+        let Some(f) = bad else {
+            first_run.fixes = Some(k.to_string());
+            break;
+        };
+        if k == max {
+            first_run.fixes = Some(format!(">{max}"));
+            break;
+        }
+        let pred = r.mattes.get(f).cloned().unwrap_or_default();
+        let fix = correction(&cur, f, &pred);
+        if fix.is_empty() {
+            first_run.fixes = Some(format!(">{max}"));
+            break;
+        }
+        cur.strokes.extend(fix);
+    }
+    first
+}
+
 /// A fixed-width table of the runs, grouped by scenario.
 pub fn table(runs: &[Run]) -> String {
-    let mut out = format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame");
+    let mut out =
+        format!("{:<10} {:<12} {:>8} {:>8} {:>8} {:>10} {:>9} {:>6}\n", "clip", "tracker", "meanIoU", "minIoU", "flicker", "after ret.", "s/frame", "fixes");
     for r in runs {
         let back = r.after_return.map_or_else(|| "-".to_string(), |v| format!("{v:.3}"));
         out.push_str(&format!(
-            "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3}\n",
-            r.scenario, r.tracker, r.mean_iou, r.min_iou, r.flicker, back, r.secs_per_frame
+            "{:<10} {:<12} {:>8.3} {:>8.3} {:>8.3} {:>10} {:>9.3} {:>6}\n",
+            r.scenario,
+            r.tracker,
+            r.mean_iou,
+            r.min_iou,
+            r.flicker,
+            back,
+            r.secs_per_frame,
+            r.fixes.as_deref().unwrap_or("-")
         ));
     }
     out
@@ -283,6 +360,19 @@ mod tests {
         assert!(table(&[run]).contains("classical"));
     }
 
+    #[test]
+    fn corrections_are_counted_until_every_frame_reaches_the_target() {
+        let all = scenarios(24);
+        let classical = |s: &Scenario| Some(run_flow(s, "classical", None));
+        // plain motion needs none
+        let easy = all.iter().find(|s| s.name == "move").and_then(|s| with_fixes(s, 0.9, 6, &classical)).unwrap();
+        assert_eq!(easy.fixes.as_deref(), Some("0"));
+        // after the disk has left and returned the classic engine has lost it: it needs a correction (or never gets there)
+        let hard = all.iter().find(|s| s.name == "reenter").and_then(|s| with_fixes(s, 0.9, 6, &classical)).unwrap();
+        assert_ne!(hard.fixes.as_deref(), Some("0"), "{:?}", hard.fixes);
+        assert!(table(&[easy, hard]).contains("fixes"));
+    }
+
     #[test]
     fn tiny_clip_requests_are_clamped() {
         assert!(scenarios(0).iter().all(|s| s.frames.len() == 12));

docs/roto-brush-sam2.md

@@ -103,3 +103,18 @@ per frame:
 MobileSAM prompts every frame independently from the flow-warped matte, so it follows the object while the flow does and
 loses it when it is hidden; SAM 2's streaming memory is what lets it re-acquire the object. The synthetic clips are easy (one
 textured disc), so the "move" and "light" rows mostly show all three working; the last two rows are the difference.
+
+### Corrections needed (`--fixes`)
+`bench --roto --fixes` also counts the correction strokes each tracker needs before every frame reaches IoU 0.9: after each
+run it paints what a user would on the first frame under the target (a short foreground stroke through the true object's
+centre, and a background stroke on the biggest false-positive blob), up to 6. 16 frames, debug build:
+
+| Clip | Classical | MobileSAM | SAM 2.1 Tiny |
+|---|---|---|---|
+| move | 0 | 0 | 0 |
+| light | 0 | 0 | 0 |
+| occlusion | more than 6 | more than 6 | 0 |
+| reenter | more than 6 | more than 6 | 0 |
+
+The simulated user is simple (one stroke pair per round, always on the first bad frame), so "more than 6" means this
+correction model never recovers the flow-based engines on those clips, not that no human could. Peak memory is not measured.