aleleba/effectcraft
1
0
Fork 0
mirror of https://github.com/storytold/effectcraft.git synced 2026-11-01 05:58:19 -06:00
Code Issues Packages Projects Releases Wiki Activity

#1539: preserve stroke corrections and current main APIs

Browse Source
This commit is contained in:
Kristian Myram
2026-10-30 08:49:47 +01:00
parent fa46bbcb88
commit baa1ca31b3
14 changed files with 94 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/effects/src/roto.rs
+1 -1
View File
@@ -696,7 +696,7 @@ impl Chain {
let strokes = self.data.strokes_of(f, obj);
let p = match frame(f) {
Some(img) => rb::tracker_prompt_boxed(&strokes, self.opts.scale, &rb::segment(&img, &strokes, None, &self.opts, self.refine_radius)),
None => rb::tracker_prompt(&strokes, self.opts.scale),
None => return Err(format!("cannot read Roto Brush prompt frame {f}")),
};
if p.points.iter().any(|(_, fg)| *fg) {
prompts.push((f, p));
crates/engine/src/tests/models.rs
+37 -32
View File
@@ -101,37 +101,8 @@ fn models_are_listed_chosen_and_verified() {
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
}
// The SAM 2.1 sizes, each from its own file (EFFECTCRAFT_SAM2, _SAM2_SMALL, _SAM2_BASE_PLUS, _SAM2_LARGE =
// path/to/model.safetensors of facebook/sam2.1-hiera-*): installed with its notice, loaded (verified, native
// only) and used by Roto Brush 3.0, then dropped and removed.
#[cfg(not(target_arch = "wasm32"))]
for (var, id) in [
("EFFECTCRAFT_SAM2", "sam2.1-hiera-tiny"),
("EFFECTCRAFT_SAM2_SMALL", "sam2.1-hiera-small"),
("EFFECTCRAFT_SAM2_BASE_PLUS", "sam2.1-hiera-base-plus"),
("EFFECTCRAFT_SAM2_LARGE", "sam2.1-hiera-large"),
] {
let Ok(path) = std::env::var(var) else { continue };
let r = s.execute_checked("roto.model.install", json!({"path": path})).unwrap();
assert_eq!(r["id"], id, "{var}");
let dir = s.models_dir().unwrap();
let notice = std::fs::read_to_string(dir.join(format!("{id}.safetensors.NOTICE.txt"))).unwrap();
assert!(notice.contains("Meta AI") && notice.contains("Apache-2.0"), "{notice}");
s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
wait_loaded(&mut s, Task::Mask);
assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id), "{var}");
let params = crate::effects::Params { values: [("version".to_string(), effectcraft_keyframe::Value::Enum(2))].into_iter().collect() };
assert_eq!(crate::effects::roto::model_for(&params).map(|m| m.info().id), Some(id));
// the list says it is installed, and exactly one size is recommended
let list = s.execute_checked("roto.models", json!({})).unwrap();
let entry = list["models"].as_array().unwrap().iter().find(|m| m["id"] == id).unwrap();
assert_eq!(entry["installed"], true);
assert_eq!(list["models"].as_array().unwrap().iter().filter(|m| m["recommended"] == true).count(), 1);
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
assert_eq!(s.execute_checked("roto.model.remove", json!({"id": id})).unwrap()["removed"], true);
assert!(!dir.join(format!("{id}.safetensors")).exists(), "{id} removed");
}
verify_installed_sam2_sizes(&mut s);
// EFFECTCRAFT_FACE_LANDMARKER=path/to/face_landmarker.task: installed with its notice,
// loaded for face tracking, dropped when the classic tracker is chosen, removed.
if let Ok(path) = std::env::var("EFFECTCRAFT_FACE_LANDMARKER") {
@@ -150,6 +121,40 @@ fn models_are_listed_chosen_and_verified() {
let _ = std::fs::remove_dir_all(d);
}
#[cfg(not(target_arch = "wasm32"))]
fn verify_installed_sam2_sizes(s: &mut Session) {
// The SAM 2.1 sizes, each from its own file (EFFECTCRAFT_SAM2, _SAM2_SMALL, _SAM2_BASE_PLUS, _SAM2_LARGE =
// path/to/model.safetensors of facebook/sam2.1-hiera-*): installed with its notice, loaded (verified, native
// only) and used by Roto Brush 3.0, then dropped and removed.
for (var, id) in [
("EFFECTCRAFT_SAM2", "sam2.1-hiera-tiny"),
("EFFECTCRAFT_SAM2_SMALL", "sam2.1-hiera-small"),
("EFFECTCRAFT_SAM2_BASE_PLUS", "sam2.1-hiera-base-plus"),
("EFFECTCRAFT_SAM2_LARGE", "sam2.1-hiera-large"),
] {
let Ok(path) = std::env::var(var) else { continue };
let r = s.execute_checked("roto.model.install", json!({"path": path})).unwrap();
assert_eq!(r["id"], id, "{var}");
let dir = s.models_dir().unwrap();
let notice = std::fs::read_to_string(dir.join(format!("{id}.safetensors.NOTICE.txt"))).unwrap();
assert!(notice.contains("Meta AI") && notice.contains("Apache-2.0"), "{notice}");
s.execute_checked("roto.model.select", json!({"id": id})).unwrap();
wait_loaded(s, Task::Mask);
assert_eq!(s.models.status(Task::Mask).active.as_deref(), Some(id), "{var}");
let params = crate::effects::Params { values: [("version".to_string(), effectcraft_keyframe::Value::Enum(2))].into_iter().collect() };
assert_eq!(crate::effects::roto::model_for(&params).map(|m| m.info().id), Some(id));
// the list says it is installed, and exactly one size is recommended
let list = s.execute_checked("roto.models", json!({})).unwrap();
let entry = list["models"].as_array().unwrap().iter().find(|m| m["id"] == id).unwrap();
assert_eq!(entry["installed"], true);
assert_eq!(list["models"].as_array().unwrap().iter().filter(|m| m["recommended"] == true).count(), 1);
s.execute_checked("roto.model.select", json!({"id": "classical"})).unwrap();
assert!(crate::effects::roto::model_for(&params).is_none());
assert_eq!(s.execute_checked("roto.model.remove", json!({"id": id})).unwrap()["removed"], true);
assert!(!dir.join(format!("{id}.safetensors")).exists(), "{id} removed");
}
}
/// A Roto Brush model that has been idle for ten minutes is unloaded (the choice stays), one in use is not,
/// and a model that does not track use is never unloaded.
#[test]
@@ -167,7 +172,7 @@ fn an_idle_roto_model_is_unloaded() {
self.0
}
}
crate::tests_roto::hold_roto_cache_test_lock();
crate::tests::roto::hold_roto_cache_test_lock();
let (mut s, d) = session("idle");
for (idle, unloaded) in [(Some(5.0), false), (None, false), (Some(601.0), true)] {
let m: Arc<dyn MaskModel> = Arc::new(Fake(idle));
@@ -256,7 +261,7 @@ fn reloading_for_a_stroke_does_not_block_or_retry_a_failed_load() {
#[test]
fn downloading_a_size_installs_and_loads_it() {
let Ok(id) = std::env::var("EFFECTCRAFT_DOWNLOAD_SIZE") else { return };
crate::tests_roto::hold_roto_cache_test_lock();
crate::tests::roto::hold_roto_cache_test_lock();
let (mut s, d) = session("download");
let info = effectcraft_segment::info(&id).unwrap();
if info.url.is_empty() {
crates/sam2/src/gpu/encoder.rs
+1 -1
View File
@@ -237,7 +237,7 @@ impl GpuEncoder {
let px = g.upload(&pixels.contiguous()?.flatten_all()?.to_vec1::<f32>()?);
let (mut h, mut w) = (ph / 4, pw / 4);
let tokens = g.zeros(h * w * hidden)?;
g.patch_embed(&px, &self.patch_w, &self.patch_b, &tokens, ph, pw, hidden)?;
g.patch_embed(&px, &self.patch_w, &self.patch_b, &tokens, (ph, pw, hidden))?;
let mut x = g.zeros(h * w * hidden)?;
g.add(&tokens, &self.pos_buffer(h, w)?, &x, h * w * hidden)?;
let mut stages: Vec<(Buf, usize, usize, usize)> = vec![];
crates/sam2/src/gpu/ops.rs
+1 -1
View File
@@ -169,7 +169,7 @@ impl Gpu {
}
/// The patch embedding: `[3, h, w]` pixels, weight `[co, 3, 7, 7]`, bias `[co]`; output `[h/4, w/4, co]`.
pub fn patch_embed(&self, pixels: &Buf, weight: &Buf, bias: &Buf, out: &Buf, h: usize, w: usize, co: usize) -> Result<()> {
pub fn patch_embed(&self, pixels: &Buf, weight: &Buf, bias: &Buf, out: &Buf, (h, w, co): (usize, usize, usize)) -> Result<()> {
let (ho, wo) = ((h + 6 - 7) / 4 + 1, (w + 6 - 7) / 4 + 1);
need(pixels, 3 * h * w, "patch_embed pixels")?;
need(weight, co * 3 * 49, "patch_embed weight")?;
crates/sam2/src/gpu/tests.rs
+1 -1
View File
@@ -340,7 +340,7 @@ fn patch_embed_rope_and_upsample_match_the_cpu_definitions() {
let (gp, gw, gb) = (g.upload(&px), g.upload(&wt), g.upload(&bias));
let (ho, wo) = (h / 4, w / 4);
let out = g.zeros(ho * wo * co).unwrap();
g.patch_embed(&gp, &gw, &gb, &out, h, w, co).unwrap();
g.patch_embed(&gp, &gw, &gb, &out, (h, w, co)).unwrap();
let t = Tensor::from_vec(px, (1, 3, h, w), &Device::Cpu).unwrap();
let k = Tensor::from_vec(wt, (co, 3, 7, 7), &Device::Cpu).unwrap();
let want =
crates/track/src/roto.rs
+8 -8
View File
@@ -950,18 +950,18 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> {
/// SAM 2 was trained on a handful of clicks: with sixteen or more points it doubts its own mask
/// (measured: confidence 0.25 with 16 points, 0.00 with 32, 0.99 with 1 to 7), so this picks at
/// most [`TRACKER_FG_POINTS`] foreground and [`TRACKER_BG_POINTS`] background points, spread
/// along the strokes.
pub fn tracker_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
/// along the strokes. Like [`stroke_prompt`], later corrections win over earlier paint.
pub fn tracker_prompt(strokes: &[&Stroke], scale: f64, w: usize, h: usize) -> Prompt {
let hard = resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, scale, w, h);
let prompt = stroke_prompt(strokes, &hard, scale, w, h);
let mut points = vec![];
for (kind, fg, max) in [(StrokeKind::Fg, true, TRACKER_FG_POINTS), (StrokeKind::Bg, false, TRACKER_BG_POINTS)] {
let all: Vec<[f32; 2]> = strokes.iter().filter(|s| s.kind == kind).flat_map(|s| points_along(s, scale, 3)).collect();
for (fg, max) in [(true, TRACKER_FG_POINTS), (false, TRACKER_BG_POINTS)] {
let all: Vec<[f32; 2]> = prompt.points.iter().filter_map(|(p, label)| (*label == fg).then_some(*p)).collect();
let n = all.len().min(max);
// evenly spaced picks, keeping the first and the last
points.extend((0..n).filter_map(|i| all.get(if n > 1 { i * (all.len() - 1) / (n - 1) } else { all.len() / 2 }).map(|p| (*p, fg))));
}
// a box the user drew on the frame prompts the tracker too
let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale));
Prompt { points, bbox, ..Default::default() }
Prompt { points, ..prompt }
}
/// [`tracker_prompt`] plus a box around the classic engine's matte of the same strokes. Clicks alone
@@ -969,7 +969,7 @@ pub fn tracker_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
/// multi-part figure, against 1.00 with the box); the box says how far the object reaches. A box
/// stroke the user drew wins over the matte's.
pub fn tracker_prompt_boxed(strokes: &[&Stroke], scale: f64, classic: &FrameSeg) -> Prompt {
let mut p = tracker_prompt(strokes, scale);
let mut p = tracker_prompt(strokes, scale, classic.w, classic.h);
if p.bbox.is_none() {
let (w, h) = (classic.w, classic.h);
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
crates/track/src/roto/tests.rs
+36 -3
View File
@@ -591,13 +591,46 @@ fn a_correction_keeps_the_model_prompted_with_what_is_left() {
fn tracker_prompts_carry_a_handful_of_points() {
let st = base_strokes(0);
let refs: Vec<&Stroke> = st.iter().collect();
let p = tracker_prompt(&refs, 1.0);
let p = tracker_prompt(&refs, 1.0, W as usize, H as usize);
let (fg, bg) = (p.points.iter().filter(|(_, f)| *f).count(), p.points.iter().filter(|(_, f)| !*f).count());
assert!((1..=TRACKER_FG_POINTS).contains(&fg) && (1..=TRACKER_BG_POINTS).contains(&bg), "{fg} fg, {bg} bg");
// a long, finely sampled stroke still gives at most the limit, and an empty one gives none
let long = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: (0..400).map(|i| [i as f64 * 0.4, 50.0]).collect(), object: 0 };
assert!(tracker_prompt(&[&long], 1.0).points.len() <= TRACKER_FG_POINTS);
assert!(tracker_prompt(&[], 1.0).points.is_empty());
assert!(tracker_prompt(&[&long], 1.0, W as usize, H as usize).points.len() <= TRACKER_FG_POINTS);
assert!(tracker_prompt(&[], 1.0, W as usize, H as usize).points.is_empty());
}
/// Tracker clicks follow the same correction order as the classic matte and model prompts.
#[test]
fn tracker_prompts_follow_latest_corrections_and_keep_remaining_foreground() {
let (w, h) = (100, 80);
let green = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 15.0, points: vec![[50.0, 40.0]], object: 0 };
let red = Stroke { kind: StrokeKind::Bg, radius: 16.0, ..green.clone() };
for strokes in [&[&green, &red][..], &[&red, &green][..]] {
let classic = FrameSeg::empty(w, h);
for prompt in [tracker_prompt(strokes, 1.0, w, h), tracker_prompt_boxed(strokes, 1.0, &classic)] {
let expected = std::ptr::eq(strokes[1], &green);
assert_eq!(prompt.points.iter().any(|(_, fg)| *fg), expected, "{:?}", prompt.points);
let hard = resolve(strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(!prompt.points.is_empty());
}
}
// A small correction removes the sampled centre, while most of the foreground dot remains.
let tap = Stroke { radius: 2.0, ..red };
let strokes = [&green, &tap];
let hard = resolve(&strokes, StrokeKind::Fg, StrokeKind::Bg, 1.0, w, h);
for prompt in [tracker_prompt(&strokes, 1.0, w, h), tracker_prompt_boxed(&strokes, 1.0, &FrameSeg::empty(w, h))] {
assert!(prompt.points.iter().any(|(_, fg)| *fg));
assert!(prompt.points.iter().any(|(_, fg)| !*fg));
for (point, fg) in &prompt.points {
assert_eq!(owner(&hard, w, h, *point), Some(if *fg { HARD_FG } else { HARD_BG }));
}
assert!(prompt.points.iter().filter(|(_, fg)| *fg).count() <= TRACKER_FG_POINTS);
assert!(prompt.points.iter().filter(|(_, fg)| !*fg).count() <= TRACKER_BG_POINTS);
}
}
#[test]
crates/ui-egui/src/panels/roto_tool.rs
+3 -3
View File
@@ -452,7 +452,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(104.0, 20.0));
let strokes = fx::data(&params).strokes.iter().filter(|st| st.object == current && st.kind.is_segmentation()).count();
let label = if strokes == 0 { tr_args("Object {}", &[&(current + 1)]) } else { tr_args("Object {} ({})", &[&(current + 1), &strokes]) };
if widgets::text_button(ui, r, &label, false, &t, egui::Id::new(("roto-object", uid))).clicked() {
if widgets::text_button(ui, r, &label, false, &t, egui::Id::unique(("roto-object", uid))).clicked() {
// cycle through the objects that have strokes (and the empty one being drawn)
let mut all = objects.clone();
if !known {
@@ -466,7 +466,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
app.auto.add("layerPanel.roto.object", r, &label);
x += 108.0;
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
if widgets::text_button(ui, r, "+", false, &t, egui::Id::new(("roto-object-new", uid))).clicked() {
if widgets::text_button(ui, r, "+", false, &t, egui::Id::unique(("roto-object-new", uid))).clicked() {
let next = objects.iter().copied().chain([current]).max().map_or(0, |m| m + 1);
let _ = app.session.execute("roto.options", json!({"object": next}));
}
@@ -475,7 +475,7 @@ pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painte
// remove the current object's strokes (the object stays selected; with none left it is simply empty)
if strokes > 0 {
let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(26.0, 20.0));
let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::new(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
let c = widgets::text_button(ui, r, "×", false, &t, egui::Id::unique(("roto-object-clear", uid))).on_hover_text(tr("Remove this object's strokes"));
if c.clicked()
&& let Err(e) = app.session.execute("roto.clearStrokes", json!({"layer": layer.id.0, "effect": uid, "object": current}))
{
crates/ui-egui/tests/it/main.rs
+2
View File
@@ -72,6 +72,8 @@ mod ui_quick_offset;
mod ui_ram_preview;
mod ui_render_queue;
mod ui_render_queue_keyboard;
mod ui_roto_objects;
mod ui_roto_span_confidence;
mod ui_script_console;
mod ui_scriptui;
mod ui_scrolling;
crates/ui-egui/tests/ui_roto_objects.rs → crates/ui-egui/tests/it/ui_roto_objects.rs
+1
View File
@@ -59,6 +59,7 @@ fn count(image: &image::RgbaImage, want: [u8; 3]) -> usize {
#[test]
fn the_boundary_view_outlines_each_object_and_show_object_picks_one() {
let _gpu = crate::gpu_test::lock();
let mut s = Session::default();
s.execute("comp.new", json!({"name": "View", "width": 640, "height": 360, "duration": 4})).unwrap();
s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();
crates/ui-egui/tests/ui_roto_span_confidence.rs → crates/ui-egui/tests/it/ui_roto_span_confidence.rs
+1
View File
@@ -64,6 +64,7 @@ fn close(got: [u8; 4], want: [u8; 3]) -> bool {
#[test]
fn the_span_bar_shows_the_trackers_confidence() {
let _gpu = crate::gpu_test::lock();
let mut s = Session::default();
s.execute("comp.new", json!({"name": "View", "width": 640, "height": 360, "duration": 4})).unwrap();
s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();