Roto Brush: keep saved tracking when nothing is retracked, read saved object mattes under their own keys, let Freeze use complete saved tracking without its model

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RDvxCS8xcR28VArjzn1x7U

crates/effects/src/roto.rs

@@ -428,7 +428,13 @@ impl Chain {
     /// has several objects and the tracker followed them there; computes nothing.
     pub fn object_seg(&self, f: i64, o: u32) -> Option<Arc<FrameSeg>> {
         let sc = self.opts.scale;
-        let k = *self.keys.get(&f)?;
+        // A frame the saved tracking has is read from it, under the keys it was made with: the model chosen now
+        // may not be the one that tracked it, and its own cache must not hold another model's matte.
+        let saved = self.saved.as_ref().filter(|t| t.cache.frames.contains_key(&f)).and_then(|t| t.keys.get(&f).map(|k| (t, *k)));
+        let k = match saved {
+            Some((_, saved_key)) => saved_key,
+            None => *self.keys.get(&f)?,
+        };
         let ok = object_key(k, o);
         if let Some(s) = cached(ok, sc) {
             return Some(s);
@@ -436,7 +442,7 @@ impl Chain {
         let full = match cached(ok, 1.0) {
             Some(s) => s,
             None => {
-                let t = self.saved.as_ref()?;
+                let (t, _) = saved?;
                 let s = Arc::new(decode_object(&t.cache, f, o)?);
                 store(ok, 1.0, s.clone());
                 s
@@ -596,6 +602,11 @@ impl Chain {
             || self.saved.as_ref().is_some_and(|t| t.cache.frames.contains_key(&f) && t.keys.contains_key(&f))
     }
 
+    /// Whether the tracking saved with the project has frame `f` (and applies to this chain).
+    pub fn saved_has(&self, f: i64) -> bool {
+        self.saved.as_ref().is_some_and(|t| t.cache.frames.contains_key(&f) && t.keys.contains_key(&f))
+    }
+
     /// Whether the chosen model can follow an object through a clip (SAM 2).
     pub fn has_tracker(&self) -> bool {
         self.model.as_ref().is_some_and(|m| m.tracker().is_some())
@@ -1032,15 +1043,21 @@ fn decode_object(cache: &TrackCache, f: i64, o: u32) -> Option<FrameSeg> {
 /// tracker followed, at full resolution, with its trust. `None` when it followed none.
 pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
     let model = ch.model.as_ref()?.info().id;
-    let mut size = None;
+    // The saved tracking applies to this chain's frames when the same model made it: what it has and this session did
+    // not track again stays (a project reopened and propagated with nothing new to track must keep its tracking).
+    let saved = ch.saved.as_ref().filter(|t| t.cache.model == model);
+    let mut size = saved.map(|t| t.cache.size);
     let mut frames = BTreeMap::new();
     for (f, k) in &ch.keys {
-        let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) else { continue };
-        size.get_or_insert([seg.w, seg.h]);
-        frames.insert(
-            *f,
-            TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded },
-        );
+        if let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) {
+            size.get_or_insert([seg.w, seg.h]);
+            frames.insert(
+                *f,
+                TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded },
+            );
+        } else if let Some(kept) = saved.filter(|t| t.keys.get(f) == Some(k)).and_then(|t| t.cache.frames.get(f)) {
+            frames.insert(*f, kept.clone());
+        }
     }
     let mut objects: BTreeMap<u32, BTreeMap<i64, ObjectFrame>> = BTreeMap::new();
     let ids = ch.data.objects();
@@ -1052,6 +1069,8 @@ pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
             for o in &ids {
                 if let Some(seg) = cached(object_key(*k, *o), 1.0) {
                     objects.entry(*o).or_default().insert(*f, ObjectFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine) });
+                } else if let Some(kept) = saved.filter(|t| t.keys.get(f) == Some(k)).and_then(|t| t.cache.objects.get(o)).and_then(|m| m.get(f)) {
+                    objects.entry(*o).or_default().insert(*f, kept.clone());
                 }
             }
         }
@@ -1060,6 +1079,12 @@ pub fn tracks_of(ch: &Chain) -> Option<TrackCache> {
     (!cache.frames.is_empty()).then_some(cache)
 }
 
+/// Whether the tracking saved with `params` covers every one of `frames`: they need no model to be read.
+pub fn saved_covers(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64]) -> bool {
+    let chain = Chain::new(params, layer_size, fps, 1.0);
+    !frames.is_empty() && frames.iter().all(|f| chain.saved_has(*f))
+}
+
 /// Encode one frame for [`FrozenCache`].
 pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame {
     let a: Vec<u8> = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect();
@@ -1402,6 +1427,42 @@ mod tests {
         assert_eq!(completed, 0, "partial objects are not complete tracked frames");
     }
 
+    /// A cold chain with saved tracking and the same model keeps the saved frames when asked what to save (nothing
+    /// was tracked this session), and Show Object reads a frame the saved tracking has under its own keys.
+    #[test]
+    fn saved_tracking_is_kept_and_its_objects_stay_out_of_another_models_keys() {
+        let mut d = RotoData::default();
+        d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0]], object: 0 }, 2, [0, 24]);
+        let mut params = params_with(&d);
+        params.values.insert(INPUT_KEY.into(), Value::Str("keep-saved-and-objects".into()));
+        let size = [100.0, 80.0];
+        let model = effectcraft_segment::SAM2_TINY.id;
+        let keys = d.chain_keys(seed_with(&params, size, 25.0, model));
+        let (matte, empty) = (vec![1u8; 100 * 80], vec![0u8; 100 * 80]);
+        let frame = TrackFrame { m: rle::encode(&matte), r: rle::encode(&empty), c: 0.9, ..Default::default() };
+        let object = ObjectFrame { m: rle::encode(&matte), r: rle::encode(&empty) };
+        let saved = TrackCache {
+            key: tracks_key(&keys),
+            model: model.into(),
+            size: [100, 80],
+            frames: keys.keys().map(|f| (*f, frame.clone())).collect(),
+            objects: [(1, keys.keys().map(|f| (*f, object.clone())).collect())].into_iter().collect(),
+        };
+        params.values.insert(TRACKED.into(), Value::Str(saved.to_json()));
+        let mut chain = Chain::new(&params, size, 25.0, 1.0);
+        chain.model = Some(Arc::new(TwoObjects(Default::default())));
+        chain.keys = keys.clone();
+        let kept = tracks_of(&chain).expect("the saved tracking is kept");
+        assert_eq!(kept.frames.keys().collect::<Vec<_>>(), saved.frames.keys().collect::<Vec<_>>());
+        assert_eq!(kept.frames.get(&1), saved.frames.get(&1));
+        // another model is chosen now: its keys differ, and the saved object matte is not stored under them
+        let other = d.chain_keys(seed_with(&params, size, 25.0, effectcraft_segment::SAM2_SMALL.id));
+        chain.keys = other.clone();
+        let seg = chain.object_seg(1, 1).expect("the saved object matte");
+        assert_eq!(seg.matte.len(), 100 * 80);
+        assert!(cached(object_key(other[&1], 1), 1.0).is_none(), "the other model's cache holds none of it");
+    }
+
     #[test]
     fn a_one_direction_pass_tracks_and_reports_only_its_frames() {
         let mut d = RotoData::default();

crates/engine/src/analysis/roto.rs

@@ -642,6 +642,21 @@ impl Session {
         }
     }
 
+    /// Whether the tracking saved with Roto Brush `effect` covers every frame a Freeze would write.
+    fn freeze_is_saved(&self, comp: ItemId, layer: LayerId, effect: Uid) -> bool {
+        let Some(c) = self.project.comp(comp) else { return false };
+        let Some(l) = c.layer(layer) else { return false };
+        let Some(g) = l.effects().and_then(|f| f.groups().find(|g| g.uid == effect)).filter(|g| is_roto(g)) else { return false };
+        let pstat = params_static(g);
+        let d = fx::data(&pstat);
+        if d.is_empty() {
+            return false;
+        }
+        let t = comp_time_of(c, l, d.base.unwrap_or(0));
+        let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
+        fx::saved_covers(&params, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &job_frames(&d, Direction::Both))
+    }
+
     /// Whether `effect` on `layer` segments with the model Settings chose: Object Matte always,
     /// Roto Brush unless it is version 1.0 (the classic engine).
     pub(crate) fn uses_roto_model(&self, comp: ItemId, layer: LayerId, effect: Uid) -> bool {
@@ -654,7 +669,8 @@ impl Session {
     pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result<usize, String> {
         // The model must be in use before the job starts: otherwise it would run on the classic
         // engine without saying so.
-        if self.uses_roto_model(comp, layer, effect) {
+        // A Freeze whose frames all have saved tracking reads them without the model.
+        if self.uses_roto_model(comp, layer, effect) && !(task == RotoTask::Freeze && self.freeze_is_saved(comp, layer, effect)) {
             self.ensure_roto_model(wait)?;
         }
         if self.is_roto_running() {

crates/engine/src/tests/roto.rs

@@ -840,6 +840,45 @@ fn saved_tracking_is_kept_in_the_project_and_applies_when_it_reopens() {
     let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());
 }
 
+/// A reopened project propagated again with the same model and nothing new to track keeps the tracking it has: the
+/// saved frames are not replaced by an empty one, so a later reopen still finds them.
+#[test]
+fn propagating_with_nothing_new_keeps_the_saved_tracking() {
+    let (s, clip, uid, path) = saved_project("keep-saved");
+    forget(&s, clip, uid);
+    let mut s2 = Session::default();
+    s2.execute_checked("file.open", json!({"path": path})).unwrap();
+    s2.footage = s.footage.clone();
+    effectcraft_effects::roto::set_model(Some(std::sync::Arc::new(Exact)));
+    let result = s2.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true}));
+    effectcraft_effects::roto::set_model(None);
+    result.unwrap();
+    let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+    assert_eq!(st["tracker"]["saved"], "sam2.1-hiera-tiny", "{st}");
+    assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}");
+    // and it is still in the file after saving and reopening
+    s2.execute_checked("file.save", json!({})).unwrap();
+    let mut s3 = Session::default();
+    s3.execute_checked("file.open", json!({"path": path})).unwrap();
+    let st = s3.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap();
+    assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}");
+    let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());
+}
+
+/// Freeze reads saved tracking without its model: choosing a model that is not installed does not stop it when
+/// the saved frames cover every frame it freezes.
+#[test]
+fn freeze_uses_complete_saved_tracking_without_the_model() {
+    let (s, clip, uid, path) = saved_project("freeze-saved");
+    forget(&s, clip, uid);
+    let mut s2 = Session::default();
+    s2.execute_checked("file.open", json!({"path": path})).unwrap();
+    s2.footage = s.footage.clone();
+    s2.execute_checked("roto.model.select", json!({"id": "sam2.1-hiera-tiny"})).unwrap();
+    s2.execute_checked("roto.freeze", json!({"layer": clip.0, "wait": true})).unwrap();
+    let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap());
+}
+
 /// Propagate replaces a fully saved span when a different tracking model is active.
 #[test]
 fn propagating_saved_tracking_uses_the_new_model() {