aleleba/effectcraft
1
0
Fork 0
mirror of https://github.com/storytold/effectcraft.git synced 2026-10-31 21:48:20 -06:00
Code Issues Packages Projects Releases Wiki Activity

#783: Object Matte effect from a point / box prompt, tracked

Browse Source 
Problem: After Effects 26.2's Object Matte (pick an object with a click,
optionally a box, get a matte that follows it) had no counterpart; only
stroke-driven Roto Brush existed.

Fix: an Object Matte effect (Effect > Matte, ec.matte.objectmatte) turns
the Object Point (and optional box) on a Reference Frame into Roto Brush
strokes and segments them with Roto Brush's engine, content-keyed cache
and the model chosen in Settings > Roto Brush (MobileSAM, classical graph
cut without it). Track Object carries the matte frame by frame up to 600
frames each way; otherwise every frame is segmented on its own. Feather,
Contrast, Shift Edge, Invert and a Matte view. The engine keeps its Input
Key current so upstream edits recompute it. objectMatte.apply creates one
from agents / the CLI. Unusable prompts pass the layer through.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
vangel76
2026-10-11 13:32:10 +02:00
co-authored by Claude Opus 5.5
parent 9a7d3e7699
commit 953fbbc730
9 changed files with 674 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/engine/src/commands/roto_cmds.rs
+90
View File
@@ -1,6 +1,9 @@
//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer
//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status.
//!
//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on
//! a frame; it segments with the same engine and model as Roto Brush.
//!
//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the
//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base
//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze
@@ -51,6 +54,15 @@ pub fn specs() -> Vec<CommandSpec> {
always,
options
),
cmd!(
"objectMatte.apply",
"Object Matte",
[],
None,
"{layer?, point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}",
has_comp,
object_matte_apply
),
query!(
"roto.status",
"Roto Brush Status",
@@ -422,3 +434,81 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
Ok(out)
}
/// Two finite numbers.
fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result<Option<Vec<f64>>> {
let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) };
let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?;
let out: Vec<f64> = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect();
if out.len() != n {
return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")));
}
Ok(Some(out))
}
fn object_matte_apply(s: &mut Session, p: &Value) -> Result<Value> {
use effectcraft_effects::object_matte as om;
let cmd = "objectMatte.apply";
let (cid, lid) = layer_p(s, p, cmd)?;
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() {
return Err(bad(cmd, "Object Matte works on footage, solid and composition layers"));
}
let size = rt::layer_size(&s.project, comp, layer);
let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?;
let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y);
if !inside(point[0], point[1]) {
return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1])));
}
let bx = xy_p(p, "box", 4, cmd)?;
if let Some(b) = &bx {
let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3]));
if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) {
return Err(bad(cmd, "`point` must be inside `box`"));
}
}
let frame = frame_p(s, p, cid, layer);
let limits = rt::frame_limits(comp, layer);
if frame < limits[0] || frame > limits[1] {
return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1])));
}
let track = b_p(p, "track").unwrap_or(true);
let reach = |k: &str| f_p(p, k).filter(|v| v.is_finite()).unwrap_or(om::DEFAULT_REACH).round().clamp(0.0, om::MAX_REACH);
let (before, after) = (reach("before"), reach("after"));
let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?;
let uid = s.edit("Object Matte", None, |proj, _| {
let mut next = proj.next_id;
let l = layer_mut(proj, cid, lid)?;
let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?;
let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count();
let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) };
let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size);
let mut set = |path: &str, v: KV| {
let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b));
let target = match grp {
Some(sub) => g.sub_mut(sub),
None => Some(&mut g),
};
if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) {
pr.value = v;
}
};
set("objectPoint", KV::Vec2([point[0], point[1]]));
if let Some(b) = &bx {
set("useBox", KV::Bool(true));
set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])]));
set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])]));
}
set("objectTracking/trackObject", KV::Bool(track));
set("objectTracking/referenceFrame", KV::Scalar(frame as f64));
set("objectTracking/framesBefore", KV::Scalar(before));
set("objectTracking/framesAfter", KV::Scalar(after));
let u = g.uid;
fxg.children.push(g.into());
proj.next_id = proj.next_id.max(next);
Ok(u)
})?;
s.state.selected_layers = vec![lid];
Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "span": [frame - before as i64, frame + after as i64]}))
}
crates/engine/src/roto.rs
+15 -5
View File
@@ -147,6 +147,12 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool {
matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID)
}
/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on
/// demand while rendering).
pub(crate) fn is_object_matte(g: &PropGroup) -> bool {
matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID)
}
fn str_of(g: &PropGroup, m: &str) -> String {
match g.get(m).map(|p| &p.value) {
Some(Value::Str(s)) => s.clone(),
@@ -188,8 +194,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
format!("{:016x}", fnv(&s))
}
/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped.
/// Returns the instances whose segmentation changed (to propagate again).
/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from
/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to
/// propagate again).
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
let mut todo = vec![];
let mut set = vec![];
@@ -197,13 +204,16 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
let ItemKind::Comp(c) = &it.kind else { continue };
for l in &c.layers {
let Some(fxg) = l.effects() else { continue };
for g in fxg.groups().filter(|g| is_roto(g)) {
for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) {
let key = input_key(after, *cid, l, g.uid);
let roto = is_roto(g);
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
if str_of(g, INPUT_KEY) != key {
set.push((*cid, l.id, g.uid, key));
todo.push((*cid, l.id, g.uid));
} else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
if roto {
todo.push((*cid, l.id, g.uid));
}
} else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
todo.push((*cid, l.id, g.uid));
}
}
crates/engine/src/tests_roto.rs
+75
View File
@@ -351,3 +351,78 @@ fn offloaded_propagation_streams_segmentations_back() {
s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap();
assert!(s.offloaded.is_empty());
}
/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is
/// followed to; the Input Key follows the layer's frames; bad prompts are refused.
#[test]
fn object_matte_from_a_click() {
hold_roto_cache_test_lock();
let (mut s, clip, solid) = setup(|f| draw(center(f)));
let cid = s.active_comp_id().unwrap();
s.edit("remove solid", None, |p, _| {
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
for it in p.items.values_mut() {
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
f.path = "object-matte.mov".into();
}
}
Ok(())
})
.unwrap();
s.execute("time.set", json!({"time": 0})).unwrap();
let c = center(0);
let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap();
assert_eq!(r["frame"], 0);
assert_eq!(r["span"], json!([-30, 6]));
let uid = r["effect"].as_u64().unwrap();
let key = |s: &Session| {
let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned()
};
let k0 = key(&s);
assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}");
let wrong = |s: &Session, f: u32| {
let c = center(f);
alpha_at(s, f)
.iter()
.enumerate()
.filter(|(i, v)| {
let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R;
(d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5)
})
.count()
};
let w0 = wrong(&s, 0);
assert!(w0 < 40, "{w0} wrong pixels on the reference frame");
let w4 = wrong(&s, 4);
assert!(w4 < 60, "{w4} wrong pixels on frame 4");
// Past Frames After: transparent.
assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0));
// Other frames under the effect: a new Input Key (the mattes are recomputed).
s.edit("swap clip", None, |p, _| {
for it in p.items.values_mut() {
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
f.path = "object-matte-2.mov".into();
}
}
Ok(())
})
.unwrap();
assert_ne!(key(&s), k0);
// Bad prompts are refused.
for bad in [
json!({"layer": clip.0}),
json!({"layer": clip.0, "point": [-10, 5]}),
json!({"layer": clip.0, "point": [10, "x"]}),
json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}),
json!({"layer": clip.0, "point": c, "frame": 999}),
] {
assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}");
}
// A box and a static prompt are accepted; the second instance is numbered.
let r = s
.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false}))
.unwrap();
let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone();
assert_eq!(g.name, "Object Matte 2");
}