From 1eee687ebc7062a60bc0c269338e86d57c9e5267 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 13:38:06 +0000 Subject: [PATCH] Roto Brush: save the tracked frames in the project The tracked frames lived only in memory, so a saved project lost them: a render in another process (effectcraft-cli render-frame) segmented the frames again with the classic engine. A propagate with a tracker now saves the frames it followed with the instance, in a hidden parameter beside Freeze's (TrackCache: each frame's matte, Refine Edge band, colour models and confidence, keyed by the inputs it was tracked from). - They apply while the strokes and span they came from are unchanged, whichever model Settings choose, so a project renders the same when it reopens, without the model loaded or chosen. - A stroke or span change makes them stale until the next Propagate. A Propagate replaces them; with the classic engine it leaves none. - roto.status reports tracker.saved (the model they came from) and a notice when Settings choose another model. - The save at job completion is not an undo step (the frames are a cache of the strokes) and is skipped if the strokes changed while tracking. Decoding checks the saved size, so a corrupt project is ignored rather than overflowing. Object Matte saves nothing. Projects saved before this change open unchanged (the parameter is filled in on open). Tests: a saved project reopens with the model unloaded and renders the saved frames through Chain::ensure; choosing the classic engine keeps them until a classic Propagate replaces them; a stroke change makes them stale. The SAM 2 tests and the failing-tracker test take the Roto test lock. Claude-Session: https://claude.ai/code/session_0136kZtXrYo5VUWEFxWduwUD --- crates/effects/src/object_matte.rs | 2 +- crates/effects/src/roto.rs | 187 ++++++++++++++++++++++++++- crates/engine/src/analysis/models.rs | 12 ++ crates/engine/src/analysis/roto.rs | 28 +++- crates/engine/src/cmd/roto_cmds.rs | 11 +- crates/engine/src/tests/roto.rs | 117 ++++++++++++++++- docs/agents.md | 3 + docs/roto-brush-sam2.md | 9 +- 8 files changed, 353 insertions(+), 16 deletions(-) diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs index 6804ea2c7..4b5e1fd42 100644 --- a/crates/effects/src/object_matte.rs +++ b/crates/effects/src/object_matte.rs @@ -227,7 +227,7 @@ pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec { pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option>) -> Option { let data = roto_data(params, layer_size, f, model.is_some())?; let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); - Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }) + Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model, saved: None }) } /// Frame `f`'s segmentation at full resolution, computing what is missing (the background diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index ffc2be376..9596b6d50 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -31,7 +31,7 @@ use effectcraft_project::ParamUi; use effectcraft_raster::Image; use effectcraft_segment::{ClipFrame, MaskModel, TrackRequest}; use effectcraft_track::roto::refine::{self, MatteParams}; -use effectcraft_track::roto::{self as rb, RotoData, SegOpts, rle}; +use effectcraft_track::roto::{self as rb, Gmm, RotoData, SegOpts, rle}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; @@ -46,6 +46,8 @@ pub const ID: &str = "ec.matte.rotobrush"; pub const STROKES: &str = "strokes"; pub const INPUT_KEY: &str = "inputKey"; pub const FROZEN: &str = "frozen"; +/// The saved tracking (see [`TrackCache`]). +pub const TRACKED: &str = "tracked"; /// Default segmentation span each side of the base frame (frames). pub const DEFAULT_SPAN: i64 = 20; @@ -95,6 +97,7 @@ pub fn specs() -> Vec { p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden), p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKED, "Tracked", Value::Str(String::new()), ParamUi::Hidden), ], render, gpu: false, @@ -214,6 +217,11 @@ pub fn installed_model() -> Option> { /// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer /// size, the frame rate, the segmentation settings and the model. pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 { + seed_with(params, layer_size, fps, model_for(params).map_or(effectcraft_segment::CLASSICAL, |m| m.info().id)) +} + +/// [`seed`] with the segmenting model named by its id (saved tracking names the one that made it). +fn seed_with(params: &Params, layer_size: [f64; 2], fps: f64, model: &str) -> u64 { let mut h = 0xcbf2_9ce4_8422_2325; fnv(&mut h, s(params, INPUT_KEY).as_bytes()); for v in [ @@ -227,7 +235,7 @@ pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 { fnv(&mut h, &v.to_bits().to_le_bytes()); } fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]); - fnv(&mut h, model_for(params).map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes()); + fnv(&mut h, model.as_bytes()); h } @@ -388,13 +396,57 @@ pub struct Chain { pub refine_radius: f64, /// The trained model (Version 2.0 / 3.0 with one installed). pub model: Option>, + /// The saved tracking, while it applies (see [`TrackCache`]). + pub(crate) saved: Option, } impl Chain { pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain { let data = data(params); let keys = data.chain_keys(seed(params, layer_size, fps)); - Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params) } + let saved = saved_for(params, &data, layer_size, fps); + Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale), model: model_for(params), saved } + } + + /// Frame `f`'s segmentation at `scale` from the saved tracking or the caches; computes nothing. + pub fn cached_seg(&self, f: i64, scale: f64) -> Option> { + self.saved_seg(f, scale).or_else(|| self.keys.get(&f).and_then(|k| cached(*k, scale))) + } + + /// The tracker's trust in frame `f`: from the saved tracking, else from this session's tracking. + pub fn trust_at(&self, f: i64) -> Option { + if let Some(fr) = self.saved.as_ref().and_then(|t| t.cache.frames.get(&f)) { + return Some(FrameTrust { confidence: fr.c, occluded: fr.o }); + } + trust(*self.keys.get(&f)?) + } + + /// The model the saved tracking was made with, while it applies. + pub fn saved_model(&self) -> Option<&str> { + self.saved.as_ref().map(|t| t.cache.model.as_str()) + } + + /// Frame `f` of the saved tracking at `scale` (decoded once, then cached). + fn saved_seg(&self, f: i64, scale: f64) -> Option> { + let t = self.saved.as_ref()?; + let k = tracked_key(*t.keys.get(&f)?); + if let Some(s) = cached(k, scale) { + return Some(s); + } + let full = match cached(k, 1.0) { + Some(s) => s, + None => { + let s = Arc::new(decode_tracked(&t.cache, f)?); + store(k, 1.0, s.clone()); + s + } + }; + if scale == 1.0 { + return Some(full); + } + let s = Arc::new(downscale(&full, scale)); + store(k, scale, s.clone()); + Some(s) } /// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest @@ -403,9 +455,12 @@ impl Chain { pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool) -> Option> { let sc = self.opts.scale; let lookup = |g: i64| -> Option> { + // A frame the tracker followed wins over the classic engine's: the saved tracking, else + // this session's. + if let Some(s) = self.saved_seg(g, sc) { + return Some(s); + } let k = *self.keys.get(&g)?; - // A frame the tracker followed wins over the classic engine's (memory only: its - // confidence is not persisted). if let Some(s) = cached(tracked_key(k), sc) { return Some(s); } @@ -654,6 +709,128 @@ fn frozen(params: &Params) -> Option> { Some(d) } +// ------------------------------------------------------------------ saved tracking + +/// The tracker's frames of an instance, saved with the project (the hidden [`TRACKED`] parameter, +/// as [`FrozenCache`] is). They apply while `key` matches the instance's inputs ([`tracks_key`]), +/// whichever model Settings choose: a stroke or span change makes them stale, and the next Propagate +/// replaces them (with the classic engine, that leaves none). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TrackCache { + /// What the frames were tracked from ([`tracks_key`]). + pub key: String, + /// The tracker's model id. + pub model: String, + /// The mattes' width and height (the layer's size). + pub size: [usize; 2], + pub frames: BTreeMap, +} + +/// One tracked frame: its matte and Refine Edge band (RLE, base64), its colour models and the +/// tracker's trust in it. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct TrackFrame { + /// Matte (RLE, base64). + pub m: String, + /// Refine Edge band (RLE, base64). + pub r: String, + pub fg: Gmm, + pub bg: Gmm, + /// 0–1: the tracker's confidence. + pub c: f32, + /// The object is not in the frame. + pub o: bool, +} + +impl TrackCache { + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } +} + +/// What saved tracking depends on: the chain keys of its frames (the inputs and the model). +pub fn tracks_key(keys: &BTreeMap) -> String { + let mut h = 0xcbf2_9ce4_8422_2325; + for (f, k) in keys { + fnv(&mut h, &f.to_le_bytes()); + fnv(&mut h, &k.to_le_bytes()); + } + format!("{h:016x}") +} + +/// The saved tracking of `params`, parsed once per content. +fn saved_tracks(params: &Params) -> Option> { + static C: OnceLock>>> = OnceLock::new(); + let j = s(params, TRACKED); + if j.is_empty() { + return None; + } + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(TrackCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + +/// A saved tracking that applies to a chain: its frames, keyed under the model that made them. +pub(crate) struct SavedTracking { + cache: Arc, + keys: BTreeMap, +} + +/// The saved tracking of `params` if it applies: the inputs are the instance's now. It does not depend +/// on which model Settings choose or has loaded, so a project renders the same anywhere. +fn saved_for(params: &Params, data: &RotoData, layer_size: [f64; 2], fps: f64) -> Option { + let cache = saved_tracks(params)?; + let keys = data.chain_keys(seed_with(params, layer_size, fps, &cache.model)); + (tracks_key(&keys) == cache.key).then_some(SavedTracking { cache, keys }) +} + +/// Decode a tracked frame of saved tracking (full resolution). +fn decode_tracked(cache: &TrackCache, f: i64) -> Option { + let fr = cache.frames.get(&f)?; + // The size comes from the project file: checked, so a corrupt one is ignored rather than overflowing. + let [w, h] = cache.size; + let n = w.checked_mul(h)?; + let matte = rle::decode(&fr.m)?; + let refine = rle::decode(&fr.r)?; + (matte.len() == n && refine.len() == n).then(|| FrameSeg { w, h, matte, refine, fg: fr.fg.clone(), bg: fr.bg.clone() }) +} + +/// The tracking a chain has just made, to save with its instance: every frame of its span that the +/// tracker followed, at full resolution, with its trust. `None` when it followed none. +pub fn tracks_of(ch: &Chain) -> Option { + let model = ch.model.as_ref()?.info().id; + let mut size = None; + let mut frames = BTreeMap::new(); + for (f, k) in &ch.keys { + let (Some(seg), Some(t)) = (cached(tracked_key(*k), 1.0), trust(*k)) else { continue }; + size.get_or_insert([seg.w, seg.h]); + frames.insert( + *f, + TrackFrame { m: rle::encode(&seg.matte), r: rle::encode(&seg.refine), fg: seg.fg.clone(), bg: seg.bg.clone(), c: t.confidence, o: t.occluded }, + ); + } + let cache = TrackCache { key: tracks_key(&ch.keys), model: model.to_string(), size: size?, frames }; + (!cache.frames.is_empty()).then_some(cache) +} + /// Encode one frame for [`FrozenCache`]. pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame { let a: Vec = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect(); diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs index 11f856d5f..bad9fd5ff 100644 --- a/crates/engine/src/analysis/models.rs +++ b/crates/engine/src/analysis/models.rs @@ -240,6 +240,18 @@ impl Session { } } + /// What `roto.status` says when the saved tracking applies but Settings choose another model (`None`: + /// they agree, or nothing is saved). + pub fn saved_tracking_notice(&self, saved: Option<&str>) -> Option { + let saved = saved?; + let chosen = self.chosen_model(Task::Mask); + if saved == chosen { + return None; + } + let name = |id: &str| seg::info(id).map_or(ROTO.classic_name, |m| m.name); + Some(format!("The saved tracking was made with {}; {} is chosen: Propagate again to replace it", name(saved), name(chosen))) + } + /// Make sure the Roto Brush model Settings chose is in use before a job that segments with it /// starts, so the job never runs on the classic engine without saying so. With `wait`, a load /// under way is waited for and one not started is loaded here; without it, a model that is not diff --git a/crates/engine/src/analysis/roto.rs b/crates/engine/src/analysis/roto.rs index 357e77b13..286b95095 100644 --- a/crates/engine/src/analysis/roto.rs +++ b/crates/engine/src/analysis/roto.rs @@ -22,7 +22,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use effectcraft_effects::object_matte as om; -use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; +use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY, TRACKED}; use effectcraft_effects::{Params, flatten_params}; use effectcraft_keyframe::Value; use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; @@ -135,6 +135,8 @@ pub struct RotoShared { pub cancel: AtomicBool, pub state: Mutex, frozen: Mutex>, + /// The tracker's frames, as the saved tracking (JSON) to store with the instance. + saved: Mutex>, } pub struct RotoJob { @@ -253,6 +255,7 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI set_str(g, INPUT_KEY, key); if had { set_str(g, FROZEN, String::new()); + set_str(g, TRACKED, String::new()); } } } @@ -459,11 +462,19 @@ async fn run_work(w: Work, shared: &RotoShared) { s.finished = true; return; } + // The frames it followed replace the saved tracking (see `poll_roto_job`): none if it failed. + *lock(&shared.saved) = Some(match &result { + Ok(_) => fx::tracks_of(&chain).map(|t| t.to_json()).unwrap_or_default(), + Err(_) => String::new(), + }); let mut s = lock(&shared.state); match result { Ok(summary) => s.tracked = summary, Err(e) => s.tracker_note = Some(format!("tracker unavailable, classic engine used: {e}")), } + } else if w.task == RotoTask::Propagate && !w.object_matte { + // With the classic engine, Propagate leaves no saved tracking behind. + *lock(&shared.saved) = Some(String::new()); } for (k, f) in w.frames.iter().enumerate() { if cancel() { @@ -738,6 +749,17 @@ impl Session { Ok(n) } + /// Save the tracker's frames in the instance, unless its strokes changed while it tracked. No undo + /// step: the frames are a cache of the strokes and the span (a stale copy is ignored). + fn save_tracks(&mut self, cid: ItemId, lid: LayerId, uid: Uid, strokes: &str, json: String) { + let p = Arc::make_mut(&mut self.project); + if let Some(g) = p.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) + && guard_of(g) == strokes + { + set_str(g, TRACKED, json); + } + } + /// Finish a done job: Freeze writes the frozen mattes (one undo step). fn poll_roto_job(&mut self) -> bool { let Some(job) = &self.roto_job else { return false }; @@ -764,6 +786,10 @@ impl Session { RotoTask::Propagate => { let verb = if job.object_matte { "tracked" } else { "propagated" }; self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + // The frames the tracker followed are saved with the instance. + if let Some(json) = lock(&job.shared.saved).take() { + self.save_tracks(job.comp, job.layer, job.effect, &job.strokes, json); + } // Rendered frames depend on the cache: redraw. self.bump(); } diff --git a/crates/engine/src/cmd/roto_cmds.rs b/crates/engine/src/cmd/roto_cmds.rs index c7f190fde..0931236a2 100644 --- a/crates/engine/src/cmd/roto_cmds.rs +++ b/crates/engine/src/cmd/roto_cmds.rs @@ -433,7 +433,6 @@ fn status(s: &mut Session, p: &Value) -> Result { out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); // The model Settings chose and whether this layer segments with it (propagate refuses or waits otherwise). out["model"] = s.roto_model(); - out["notice"] = json!(s.roto_model_notice(s.uses_roto_model(cid, lid, uid))); out["limits"] = json!(rt::frame_limits(&comp, &layer)); let frame = frame_p(s, p, cid, &layer); out["frame"] = json!(frame); @@ -444,7 +443,7 @@ fn status(s: &mut Session, p: &Value) -> Result { let fps = comp.frame_rate.as_f64(); let ps = params_static(&g); let chain = fx::Chain::new(&ps, size, fps, 1.0); - let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) { + let seg = match chain.cached_seg(frame, 1.0) { Some(sg) => Some(sg), None if compute => { let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0); @@ -474,10 +473,12 @@ fn status(s: &mut Session, p: &Value) -> Result { let chain = fx::Chain::new(¶ms_static(&g), size, comp.frame_rate.as_f64(), 1.0); let tracked: serde_json::Map = chain .keys - .iter() - .filter_map(|(f, k)| fx::trust(*k).map(|t| (f.to_string(), json!({"confidence": t.confidence, "occluded": t.occluded})))) + .keys() + .filter_map(|f| chain.trust_at(*f).map(|t| (f.to_string(), json!({"confidence": t.confidence, "occluded": t.occluded})))) .collect(); - out["tracker"] = json!({"model": s.models_roto_active(), "frames": tracked}); + out["tracker"] = json!({"model": s.models_roto_active(), "saved": chain.saved_model(), "frames": tracked}); + let notice = s.roto_model_notice(s.uses_roto_model(cid, lid, uid)).or_else(|| s.saved_tracking_notice(chain.saved_model())); + out["notice"] = json!(notice); out["source"] = json!(if tracked.contains_key(&frame.to_string()) { "tracker" } else if chain.model.is_some() { diff --git a/crates/engine/src/tests/roto.rs b/crates/engine/src/tests/roto.rs index 9ef6a848f..b18cd386d 100644 --- a/crates/engine/src/tests/roto.rs +++ b/crates/engine/src/tests/roto.rs @@ -700,9 +700,14 @@ fn sam2_tracks_the_clip_through_roto_propagate() { assert_eq!(frames.len(), 21); assert!(frames.values().all(|t| t["occluded"] == false && t["confidence"].as_f64().unwrap() > 0.3), "{frames:?}"); assert_eq!(st["tracker"]["model"], "sam2.1-hiera-tiny"); - // Back to the classic engine: same strokes, its own segmentation, no tracker data. + // Settings choose the classic engine: the saved tracking stays until a Propagate replaces it. s.execute_checked("roto.model.select", json!({"id": "classical", "wait": true})).unwrap(); let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["source"], "tracker", "{st}"); + assert!(st["notice"].as_str().unwrap().contains("Propagate again"), "{st}"); + // A classic Propagate: its own segmentation, no tracker data. + s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap(); + let st = s.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); assert_eq!(st["source"], "classical", "{st}"); assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0); let _ = std::fs::remove_dir_all(dir); @@ -751,6 +756,116 @@ fn sam2_selected_without_wait_is_loaded_before_propagate() { let _ = std::fs::remove_dir_all(dir); } +/// A tracker that follows the disc exactly: its true mask on every frame of the request. A stand-in +/// for SAM 2, so the saved tracking can be tested without the weights. +struct Exact; + +impl effectcraft_segment::MaskModel for Exact { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::SAM2_TINY + } + fn segment(&self, _: &[[f32; 3]], _: usize, _: usize, _: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + Err("no weights".into()) + } + fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> { + Some(self) + } +} + +impl effectcraft_segment::ObjectTracker for Exact { + fn track( + &self, + req: &effectcraft_segment::TrackRequest, + frame: &mut dyn FnMut(i64) -> Option>, + _: &dyn Fn() -> bool, + out: &mut dyn FnMut(i64, effectcraft_segment::TrackedFrame), + ) -> effectcraft_segment::Result<()> { + for g in req.first..req.first + req.count as i64 { + let Some(img) = frame(g) else { return Err(format!("no frame {g}")) }; + let c = center(g as u32); + let prob = (0..img.w * img.h) + .map(|i| ((((i % img.w) as f64 + 0.5 - c[0]).hypot((i / img.w) as f64 + 0.5 - c[1])) <= R) as u8 as f32) + .collect::>(); + out(g, effectcraft_segment::TrackedFrame { prob, w: img.w, h: img.h, confidence: 0.99, occluded: false }); + } + Ok(()) + } +} + +/// Saved tracking is part of the project: written with the instance, it applies when the project +/// opens, with or without the model loaded, and Settings choosing the classic engine leaves it in place +/// until a Propagate replaces it. +#[test] +fn saved_tracking_is_kept_in_the_project_and_applies_when_it_reopens() { + let sam2 = "sam2.1-hiera-tiny"; + let (s, clip, uid, path) = saved_project("saved"); + // Reopened with SAM 2 chosen but not loaded (none is installed here): the saved frames apply. + forget(&s, clip, uid); + let mut s2 = Session::default(); + s2.execute_checked("file.open", json!({"path": path})).unwrap(); + s2.execute_checked("roto.model.select", json!({"id": sam2})).unwrap(); + let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["source"], "tracker", "{st}"); + assert_eq!(st["tracker"]["saved"], sam2, "{st}"); + assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 21, "{st}"); + let j = iou_at(&mut s2, clip, 12, center(12)); + assert!(j > 0.9, "IoU of a saved frame: {j}"); + // The render path (Chain::ensure) reads the same frame. + let comp = s2.active_comp().unwrap(); + let l = comp.layer(clip).unwrap(); + let g = l.props.find_group(uid).unwrap(); + let size = crate::roto::layer_size(&s2.project, comp, l); + let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0); + let seg = ch.ensure(12, &mut |_| None, &|| false).expect("the render path reads the saved frame"); + let truth = rle::decode(>(center(12))).unwrap(); + let inter = seg.matte.iter().zip(&truth).filter(|(a, b)| **a != 0 && **b != 0).count(); + let uni = seg.matte.iter().zip(&truth).filter(|(a, b)| **a != 0 || **b != 0).count(); + assert!(inter as f64 / uni as f64 > 0.9, "the render path's frame: IoU {}", inter as f64 / uni as f64); + // Settings choose the classic engine: the saved frames still apply, and the notice says so. + s2.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap(); + let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["source"], "tracker", "{st}"); + assert!(st["notice"].as_str().unwrap().contains("Propagate again"), "{st}"); + // A classic Propagate replaces the saved tracking with nothing. + s2.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap(); + let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["source"], "classical", "{st}"); + assert!(st["tracker"]["saved"].is_null() && st["notice"].is_null(), "{st}"); + let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); +} + +/// A tracked clip saved to a project file: the session that tracked it, its layer and instance, and the +/// file. +fn saved_project(name: &str) -> (Session, LayerId, u64, String) { + let (mut s, clip, uid) = painted(center, &format!("disk-{name}.mov")); + effectcraft_effects::roto::set_model(Some(std::sync::Arc::new(Exact))); + let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})); + effectcraft_effects::roto::set_model(None); + assert_eq!(r.unwrap()["frames"], 21); + let dir = std::env::temp_dir().join(format!("ec-roto-{name}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("tracked.ecproj").to_string_lossy().to_string(); + s.execute_checked("file.saveAs", json!({"path": path})).unwrap(); + let saved = std::fs::read_to_string(&path).unwrap(); + assert!(saved.contains("sam2.1-hiera-tiny"), "the project file carries the tracking and the model that made it"); + (s, clip, uid, path) +} + +/// A stroke change makes the saved tracking stale: the instance is classical until the next Propagate. +#[test] +fn a_stroke_change_makes_the_saved_tracking_stale() { + let (_s, clip, _uid, path) = saved_project("stale"); + let mut s2 = Session::default(); + s2.execute_checked("file.open", json!({"path": path})).unwrap(); + s2.execute_checked("roto.model.select", json!({"id": "sam2.1-hiera-tiny"})).unwrap(); + assert_eq!(s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap()["source"], "tracker"); + s2.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line([10.0, 10.0], [60.0, 12.0]), "radius": 4})).unwrap(); + let st = s2.execute("roto.status", json!({"layer": clip.0, "frame": 12})).unwrap(); + assert_eq!(st["source"], "classical", "{st}"); + assert_eq!(st["tracker"]["frames"].as_object().unwrap().len(), 0, "{st}"); + let _ = std::fs::remove_dir_all(std::path::Path::new(&path).parent().unwrap()); +} + /// A model whose tracker always fails: Roto Brush must carry on with the classic engine and say so. #[test] fn a_failing_tracker_falls_back_to_the_classic_engine() { diff --git a/docs/agents.md b/docs/agents.md index 54ba21afa..ddd730557 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -511,6 +511,9 @@ Trained model (Roto Brush 2.0 / 3.0): - `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed to load, is an error, and without `"wait": true` so is a model still loading. `roto.status` reports `model` (`selected`, `installed`, `active`, `loading`, `error`) and a `notice` while the chosen model is not the one in use. + Tracked frames are saved in the project with the instance (`tracker.saved` names their model): they apply while the + strokes and span are unchanged, whichever model Settings choose (`notice` says when that differs), and a new + Propagate replaces them. - `roto.model.remove {id}` deletes the installed weights. - The `version` property (`effects/#1/version`: 0 = 1.0, 1 = 2.0, 2 = 3.0) picks between the classic engine (1.0) and the chosen model (2.0, 3.0). diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index f5946a98e..d5216e6d4 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -16,15 +16,18 @@ Apache-2.0). It is optional: nothing changes until you install the model and cho `classical`) and, for tracked frames, `tracker.frames[f]`: `confidence` (0–1, the model's object score times its predicted mask quality) and `occluded` (the object is not in the frame: an empty matte). `model` says what Settings chose and whether it is installed, loaded and loading; `notice` says when it is not the one in use for the layer. + `tracker.saved` names the model the saved frames came from. Propagate never falls back to the classic engine without saying so. With SAM 2 chosen but not installed, or failing to load, it refuses with the reason. While it is still loading, `wait: true` (agents) loads it first; without `wait` it refuses, and the load goes on in the background. Deselecting SAM 2 (choosing Classic) runs Propagate exactly as before, and Roto Brush 1.0 never uses a model. -Tracked frames live in the session that ran Propagate and are not saved in the project file. A separate render process -(`effectcraft-cli render-frame`) does not have them, so render in the same session: in the app, or in one `run` that -includes `roto.propagate` and then `comp.saveFrameAs`. +Tracked frames are saved in the project with the instance, as Freeze's are, so a saved project renders the same when it +reopens, in the app or with `effectcraft-cli`, without the model being loaded or chosen there. They apply while the +strokes and span they came from are unchanged. Changing a stroke or the span makes them stale until the next Propagate. +Choosing another model, or the classic engine, leaves them in place and `roto.status` says so; a classic Propagate +replaces them with classic mattes. Object Matte saves nothing. ## How it works `crates/sam2` is a Rust port (on [candle](https://github.com/huggingface/candle), native only) of the Hugging Face