S2.41: Roto Brush follows the object with SAM 2

When the chosen model has a tracker, roto.propagate runs one tracking pass over
the span: prompts from the strokes (a handful of points, which SAM 2 trusts),
the clip tracked both ways, each frame's probability map refined by the same
graph cut with the strokes as hard constraints. Results are cached under their
own keys with a per-frame confidence and an occluded flag; roto.status reports
them (source: tracker|classical). Without a tracker nothing changes.
Real weights: IoU 0.98 on all 21 frames, confidence 0.99.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/engine/src/analysis/models.rs

@@ -188,6 +188,11 @@ impl Session {
         });
     }
 
+    /// The Roto Brush model in use (`None`: the classic engine).
+    pub fn models_roto_active(&self) -> Option<String> {
+        self.models.status(Task::Mask).active
+    }
+
     /// Finish what the model workers did: a toast, and loading a model that just arrived.
     pub fn poll_models(&mut self) {
         for task in [Task::Mask, Task::Face] {

crates/engine/src/analysis/roto.rs

@@ -112,6 +112,10 @@ pub struct RotoProgress {
     pub error: Option<String>,
     /// The job works on an Object Matte (else a Roto Brush & Refine Edge).
     pub object_matte: bool,
+    /// What the model's tracker found, when one followed the object (SAM 2).
+    pub tracked: Option<fx::TrackSummary>,
+    /// Why the tracker was not used, when a tracking model was chosen but the classic engine ran.
+    pub tracker_note: Option<String>,
 }
 
 impl RotoProgress {
@@ -436,6 +440,31 @@ async fn run_work(w: Work, shared: &RotoShared) {
     let model = fx::installed_model();
     let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) };
     let mut frozen = FrozenCache { key, ..Default::default() };
+    // A tracking model (SAM 2) follows the object through the whole span in one pass; the loop
+    // below then finds every frame cached. Without one (or if it fails) nothing changes: the
+    // classic engine runs frame by frame.
+    if w.task == RotoTask::Propagate && !w.object_matte && chain.has_tracker() {
+        let mut get = |g: i64| src.get(g);
+        let mut report = |n: usize| {
+            let mut s = lock(&shared.state);
+            s.done = n as u64;
+            s.elapsed = t0.elapsed().as_secs_f64();
+            s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
+            crate::offload::report(s.done, s.total);
+        };
+        let result = chain.track(&mut get, &cancel, &mut report);
+        if cancel() {
+            let mut s = lock(&shared.state);
+            s.cancelled = true;
+            s.finished = true;
+            return;
+        }
+        let mut s = lock(&shared.state);
+        match result {
+            Ok(summary) => s.tracked = summary,
+            Err(e) => s.tracker_note = Some(format!("tracker unavailable, classic engine used: {e}")),
+        }
+    }
     for (k, f) in w.frames.iter().enumerate() {
         if cancel() {
             let mut s = lock(&shared.state);

crates/engine/src/cmd/roto_cmds.rs

@@ -463,6 +463,19 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
             }
         }
     }
+    // Where each frame's matte came from, and how far to trust it: frames a tracker followed
+    // report its confidence and whether the object was absent; the rest came from the classic engine.
+    {
+        let size = rt::layer_size(&s.project, &comp, &layer);
+        let chain = fx::Chain::new(&params_static(&g), size, comp.frame_rate.as_f64(), 1.0);
+        let tracked: serde_json::Map<String, Value> = chain
+            .keys
+            .iter()
+            .filter_map(|(f, k)| fx::trust(*k).map(|t| (f.to_string(), json!({"confidence": t.confidence, "occluded": t.occluded}))))
+            .collect();
+        out["tracker"] = json!({"model": s.models_roto_active(), "frames": tracked});
+        out["source"] = json!(if tracked.contains_key(&frame.to_string()) { "tracker" } else { "classical" });
+    }
     let computed = s.roto_computed(cid, lid, uid);
     out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
     Ok(out)

crates/engine/src/tests/roto.rs

@@ -668,3 +668,41 @@ fn object_matte_mattes_persist_in_the_disk_cache() {
     assert!(om::ensure(&moved, size, 25.0, 0, None, &mut |_| None, &|| false).is_none());
     let _ = std::fs::remove_dir_all(&dir);
 }
+
+/// SAM 2 tracks the clip end to end (needs the real weights: `EFFECTCRAFT_SAM2=path/to/model.safetensors`).
+/// Every frame comes from the tracker with a confidence, the mattes are right, and nothing changes
+/// once the model is deselected.
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn sam2_tracks_the_clip_through_roto_propagate() {
+    let Ok(weights) = std::env::var("EFFECTCRAFT_SAM2") else { return };
+    let dir = std::env::temp_dir().join(format!("ec-roto-sam2-{}", std::process::id()));
+    let (mut s, clip, _) = painted(center, "disk-sam2.mov");
+    s.models_dir = Some(dir.join("models"));
+    std::fs::create_dir_all(dir.join("models")).unwrap();
+    s.execute_checked("roto.model.install", json!({"path": weights})).unwrap();
+    s.execute_checked("roto.model.select", json!({"id": "sam2.1-hiera-tiny", "wait": true})).unwrap();
+    assert_eq!(s.models.status(effectcraft_segment::Task::Mask).active.as_deref(), Some("sam2.1-hiera-tiny"));
+    let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap();
+    assert_eq!(r["frames"], 21, "{r}");
+    let mut worst: f64 = 1.0;
+    for f in 0..=20 {
+        let j = iou_at(&mut s, clip, f, center(f));
+        let st = s.execute("roto.status", json!({"layer": clip.0, "frame": f})).unwrap();
+        eprintln!("frame {f}: IoU {j:.3}, {:?}", st["tracker"]["frames"][f.to_string()]);
+        worst = worst.min(j);
+    }
+    assert!(worst > 0.9, "worst IoU over the span with SAM 2: {worst}");
+    let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+    assert_eq!(st["source"], "tracker", "{st}");
+    let frames = st["tracker"]["frames"].as_object().unwrap();
+    assert_eq!(frames.len(), 21);
+    assert!(frames.values().all(|t| t["occluded"] == false && t["confidence"].as_f64().unwrap() > 0.3), "{frames:?}");
+    assert_eq!(st["tracker"]["model"], "sam2.1-hiera-tiny");
+    // Back to the classic engine: same strokes, its own segmentation, no tracker data.
+    s.execute_checked("roto.model.select", json!({"id": "classical", "wait": true})).unwrap();
+    let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+    assert_eq!(st["source"], "classical", "{st}");
+    assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0);
+    let _ = std::fs::remove_dir_all(dir);
+}