From 36ed77820d11d4390b1a1bcb0a348bdd095fe8e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 16:09:57 +0000 Subject: [PATCH] Roto Brush: keep saved tracking when nothing is retracked, read saved object mattes under their own keys, let Freeze use complete saved tracking without its model Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01RDvxCS8xcR28VArjzn1x7U --- crates/effects/src/roto.rs | 79 ++++++++++++++++++++++++++---- crates/engine/src/analysis/roto.rs | 18 ++++++- crates/engine/src/tests/roto.rs | 39 +++++++++++++++ 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index 235c26afa..615ea8114 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -428,7 +428,13 @@ impl Chain { /// has several objects and the tracker followed them there; computes nothing. pub fn object_seg(&self, f: i64, o: u32) -> Option> { let sc = self.opts.scale; - let k = *self.keys.get(&f)?; + // A frame the saved tracking has is read from it, under the keys it was made with: the model chosen now + // may not be the one that tracked it, and its own cache must not hold another model's matte. + let saved = self.saved.as_ref().filter(|t| t.cache.frames.contains_key(&f)).and_then(|t| t.keys.get(&f).map(|k| (t, *k))); + let k = match saved { + Some((_, saved_key)) => saved_key, + None => *self.keys.get(&f)?, + }; let ok = object_key(k, o); if let Some(s) = cached(ok, sc) { return Some(s); @@ -436,7 +442,7 @@ impl Chain { let full = match cached(ok, 1.0) { Some(s) => s, None => { - let t = self.saved.as_ref()?; + let (t, _) = saved?; let s = Arc::new(decode_object(&t.cache, f, o)?); store(ok, 1.0, s.clone()); s @@ -596,6 +602,11 @@ impl Chain { || self.saved.as_ref().is_some_and(|t| t.cache.frames.contains_key(&f) && t.keys.contains_key(&f)) } + /// Whether the tracking saved with the project has frame `f` (and applies to this chain). + pub fn saved_has(&self, f: i64) -> bool { + self.saved.as_ref().is_some_and(|t| t.cache.frames.contains_key(&f) && t.keys.contains_key(&f)) + } + /// Whether the chosen model can follow an object through a clip (SAM 2). pub fn has_tracker(&self) -> bool { self.model.as_ref().is_some_and(|m| m.tracker().is_some()) @@ -1032,15 +1043,21 @@ fn decode_object(cache: &TrackCache, f: i64, o: u32) -> Option { /// tracker followed, at full resolution, with its trust. `None` when it followed none. pub fn tracks_of(ch: &Chain) -> Option { let model = ch.model.as_ref()?.info().id; - let mut size = None; + // The saved tracking applies to this chain's frames when the same model made it: what it has and this session did + // not track again stays (a project reopened and propagated with nothing new to track must keep its tracking). + let saved = ch.saved.as_ref().filter(|t| t.cache.model == model); + let mut size = saved.map(|t| t.cache.size); let mut frames = BTreeMap::new(); for (f, k) in &ch.keys { - let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) else { continue }; - size.get_or_insert([seg.w, seg.h]); - frames.insert( - *f, - TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded }, - ); + if let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) { + size.get_or_insert([seg.w, seg.h]); + frames.insert( + *f, + TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded }, + ); + } else if let Some(kept) = saved.filter(|t| t.keys.get(f) == Some(k)).and_then(|t| t.cache.frames.get(f)) { + frames.insert(*f, kept.clone()); + } } let mut objects: BTreeMap> = BTreeMap::new(); let ids = ch.data.objects(); @@ -1052,6 +1069,8 @@ pub fn tracks_of(ch: &Chain) -> Option { for o in &ids { if let Some(seg) = cached(object_key(*k, *o), 1.0) { objects.entry(*o).or_default().insert(*f, ObjectFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine) }); + } else if let Some(kept) = saved.filter(|t| t.keys.get(f) == Some(k)).and_then(|t| t.cache.objects.get(o)).and_then(|m| m.get(f)) { + objects.entry(*o).or_default().insert(*f, kept.clone()); } } } @@ -1060,6 +1079,12 @@ pub fn tracks_of(ch: &Chain) -> Option { (!cache.frames.is_empty()).then_some(cache) } +/// Whether the tracking saved with `params` covers every one of `frames`: they need no model to be read. +pub fn saved_covers(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64]) -> bool { + let chain = Chain::new(params, layer_size, fps, 1.0); + !frames.is_empty() && frames.iter().all(|f| chain.saved_has(*f)) +} + /// Encode one frame for [`FrozenCache`]. pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame { let a: Vec = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect(); @@ -1402,6 +1427,42 @@ mod tests { assert_eq!(completed, 0, "partial objects are not complete tracked frames"); } + /// A cold chain with saved tracking and the same model keeps the saved frames when asked what to save (nothing + /// was tracked this session), and Show Object reads a frame the saved tracking has under its own keys. + #[test] + fn saved_tracking_is_kept_and_its_objects_stay_out_of_another_models_keys() { + let mut d = RotoData::default(); + d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0]], object: 0 }, 2, [0, 24]); + let mut params = params_with(&d); + params.values.insert(INPUT_KEY.into(), Value::Str("keep-saved-and-objects".into())); + let size = [100.0, 80.0]; + let model = effectcraft_segment::SAM2_TINY.id; + let keys = d.chain_keys(seed_with(¶ms, size, 25.0, model)); + let (matte, empty) = (vec![1u8; 100 * 80], vec![0u8; 100 * 80]); + let frame = TrackFrame { m: rle::encode(&matte), r: rle::encode(&empty), c: 0.9, ..Default::default() }; + let object = ObjectFrame { m: rle::encode(&matte), r: rle::encode(&empty) }; + let saved = TrackCache { + key: tracks_key(&keys), + model: model.into(), + size: [100, 80], + frames: keys.keys().map(|f| (*f, frame.clone())).collect(), + objects: [(1, keys.keys().map(|f| (*f, object.clone())).collect())].into_iter().collect(), + }; + params.values.insert(TRACKED.into(), Value::Str(saved.to_json())); + let mut chain = Chain::new(¶ms, size, 25.0, 1.0); + chain.model = Some(Arc::new(TwoObjects(Default::default()))); + chain.keys = keys.clone(); + let kept = tracks_of(&chain).expect("the saved tracking is kept"); + assert_eq!(kept.frames.keys().collect::>(), saved.frames.keys().collect::>()); + assert_eq!(kept.frames.get(&1), saved.frames.get(&1)); + // another model is chosen now: its keys differ, and the saved object matte is not stored under them + let other = d.chain_keys(seed_with(¶ms, size, 25.0, effectcraft_segment::SAM2_SMALL.id)); + chain.keys = other.clone(); + let seg = chain.object_seg(1, 1).expect("the saved object matte"); + assert_eq!(seg.matte.len(), 100 * 80); + assert!(cached(object_key(other[&1], 1), 1.0).is_none(), "the other model's cache holds none of it"); + } + #[test] fn a_one_direction_pass_tracks_and_reports_only_its_frames() { let mut d = RotoData::default(); diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rs index a07b0010e..03b287d6e 100644 --- a/crates/engine/src/analysis/roto.rs +++ b/crates/engine/src/analysis/roto.rs @@ -642,6 +642,21 @@ impl Session { } } + /// Whether the tracking saved with Roto Brush `effect` covers every frame a Freeze would write. + fn freeze_is_saved(&self, comp: ItemId, layer: LayerId, effect: Uid) -> bool { + let Some(c) = self.project.comp(comp) else { return false }; + let Some(l) = c.layer(layer) else { return false }; + let Some(g) = l.effects().and_then(|f| f.groups().find(|g| g.uid == effect)).filter(|g| is_roto(g)) else { return false }; + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return false; + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + fx::saved_covers(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &job_frames(&d, Direction::Both)) + } + /// Whether `effect` on `layer` segments with the model Settings chose: Object Matte always, /// Roto Brush unless it is version 1.0 (the classic engine). pub(crate) fn uses_roto_model(&self, comp: ItemId, layer: LayerId, effect: Uid) -> bool { @@ -654,7 +669,8 @@ impl Session { pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result { // The model must be in use before the job starts: otherwise it would run on the classic // engine without saying so. - if self.uses_roto_model(comp, layer, effect) { + // A Freeze whose frames all have saved tracking reads them without the model. + if self.uses_roto_model(comp, layer, effect) && !(task == RotoTask::Freeze && self.freeze_is_saved(comp, layer, effect)) { self.ensure_roto_model(wait)?; } if self.is_roto_running() { diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rs index 6c57d5c29..afc3adad9 100644 --- a/crates/engine/src/tests/roto.rs +++ b/crates/engine/src/tests/roto.rs @@ -840,6 +840,45 @@ fn saved_tracking_is_kept_in_the_project_and_applies_when_it_reopens() { let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); } +/// A reopened project propagated again with the same model and nothing new to track keeps the tracking it has: the +/// saved frames are not replaced by an empty one, so a later reopen still finds them. +#[test] +fn propagating_with_nothing_new_keeps_the_saved_tracking() { + let (s, clip, uid, path) = saved_project("keep-saved"); + forget(&s, clip, uid); + let mut s2 = Session::default(); + s2.execute_checked("file.open", json!({"path": path})).unwrap(); + s2.footage = s.footage.clone(); + effectcraft_effects::roto::set_model(Some(std::sync::Arc::new(Exact))); + let result = s2.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})); + effectcraft_effects::roto::set_model(None); + result.unwrap(); + let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["tracker"]["saved"], "sam2.1-hiera-tiny", "{st}"); + assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}"); + // and it is still in the file after saving and reopening + s2.execute_checked("file.save", json!({})).unwrap(); + let mut s3 = Session::default(); + s3.execute_checked("file.open", json!({"path": path})).unwrap(); + let st = s3.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}"); + let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); +} + +/// Freeze reads saved tracking without its model: choosing a model that is not installed does not stop it when +/// the saved frames cover every frame it freezes. +#[test] +fn freeze_uses_complete_saved_tracking_without_the_model() { + let (s, clip, uid, path) = saved_project("freeze-saved"); + forget(&s, clip, uid); + let mut s2 = Session::default(); + s2.execute_checked("file.open", json!({"path": path})).unwrap(); + s2.footage = s.footage.clone(); + s2.execute_checked("roto.model.select", json!({"id": "sam2.1-hiera-tiny"})).unwrap(); + s2.execute_checked("roto.freeze", json!({"layer": clip.0, "wait": true})).unwrap(); + let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); +} + /// Propagate replaces a fully saved span when a different tracking model is active. #[test] fn propagating_saved_tracking_uses_the_new_model() {