mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-26 11:07:54 -06:00
Merge branch 'worktree-agent-a128c7366e475da55'
This commit is contained in:
Brandon Thomas
@@ -24,6 +24,7 @@ mod styles;
|
||||
mod text_anim;
|
||||
mod three_d;
|
||||
mod time;
|
||||
mod track;
|
||||
mod view;
|
||||
|
||||
use std::sync::OnceLock;
|
||||
@@ -102,6 +103,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
|
||||
v.extend(file_more::specs());
|
||||
v.extend(comp_more::specs());
|
||||
v.extend(frontend::specs());
|
||||
v.extend(track::specs());
|
||||
v.extend(stubs::specs());
|
||||
v
|
||||
})
|
||||
|
||||
@@ -83,9 +83,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
||||
stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"),
|
||||
stub!("track.camera", "Track Camera", ["Animation"], "{}"),
|
||||
stub!("track.warpStabilizer", "Warp Stabilizer VFX", ["Animation"], "{}"),
|
||||
stub!("track.motion", "Track Motion", ["Animation"], "{}"),
|
||||
stub!("track.mask", "Track Mask", ["Animation"], "{}"),
|
||||
stub!("track.property", "Track this Property", ["Animation"], "{}"),
|
||||
// Workspaces and panels that don't exist yet.
|
||||
stub!("window.saveWorkspace", "Save Changes to this Workspace", ["Window", "Workspace"], "{}"),
|
||||
stub!("window.saveWorkspaceAs", "Save as New Workspace...", ["Window", "Workspace"], "{name}"),
|
||||
|
||||
@@ -0,0 +1,584 @@
|
||||
//! Motion tracking commands (Animation ▸ Track Motion / Stabilize Motion / Track this Property and
|
||||
//! the Tracker panel): create trackers, set type / target / options, move track points, analyze
|
||||
//! (background job or blocking), apply, reset.
|
||||
//!
|
||||
//! A tracker is addressed by `layer` (the tracked layer, a.k.a. Motion Source) and `tracker`
|
||||
//! (group uid, name or 1-based index); both default to the Tracker panel's Current Track.
|
||||
|
||||
use effectcraft_project::build::Ids;
|
||||
use effectcraft_project::tracking::{self, LowConfidence, TrackChannel, TrackKind, TrackerSettings};
|
||||
use effectcraft_project::{ItemId, Layer, LayerId, Node, PropGroup, Uid, Value};
|
||||
use effectcraft_time::Tick;
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CommandSpec, b_p, bad, comp_id, f_p, frontend, has_comp, layer_mut, layer_p, resolve_layer, str_p};
|
||||
use crate::tracking::{Dims, Direction, Work, point_values};
|
||||
use crate::{EngineError, Result, Session, cmd, query};
|
||||
|
||||
type V = serde_json::Value;
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![
|
||||
cmd!("track.motion", "Track Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Transform, "track.motion")),
|
||||
cmd!("track.stabilize", "Stabilize Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Stabilize, "track.stabilize")),
|
||||
cmd!(
|
||||
"track.new",
|
||||
"New Tracker",
|
||||
[],
|
||||
None,
|
||||
"{layer?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?, target?: layer}",
|
||||
can_track,
|
||||
new_tracker
|
||||
),
|
||||
cmd!("track.property", "Track this Property", ["Animation"], None, "{layer?, source?: layer}", has_prop_selected, track_property),
|
||||
cmd!("track.select", "Current Track", [], None, "{layer?, tracker?: uid|name|#n}", has_comp, select),
|
||||
cmd!(
|
||||
"track.setType",
|
||||
"Track Type",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?}",
|
||||
has_track,
|
||||
set_type
|
||||
),
|
||||
cmd!("track.setTarget", "Edit Target", [], None, "{layer?, tracker?, target: layer|null}", has_track, set_target),
|
||||
cmd!(
|
||||
"track.options",
|
||||
"Motion Tracker Options",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, name?, channel?: rgb|luminance|saturation, blur? (px, 0 = off), enhance?, subpixel?, adaptEveryFrame?, threshold? (%), action?: continue|stop|extrapolate|adapt, trackShape?}",
|
||||
has_track,
|
||||
options
|
||||
),
|
||||
cmd!(
|
||||
"track.setPoint",
|
||||
"Move Track Point",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, point: n (1-based), center?: [x,y], featureSize?: [w,h], searchOffset?: [x,y], searchSize?: [w,h], attachOffset?: [x,y], move?: [dx,dy], time? (s)}",
|
||||
has_track,
|
||||
set_point
|
||||
),
|
||||
cmd!(
|
||||
"track.analyze",
|
||||
"Analyze",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
|
||||
has_track,
|
||||
analyze
|
||||
),
|
||||
cmd!("track.stop", "Stop Analysis", [], None, "{}", is_tracking, |s, _| Ok(json!({"stopped": s.stop_track()}))),
|
||||
cmd!("track.apply", "Apply", [], None, "{layer?, tracker?, dimensions?: xy|x|y}", has_track, apply),
|
||||
cmd!("track.reset", "Reset", [], None, "{layer?, tracker?}", has_track, reset),
|
||||
cmd!("track.delete", "Delete Tracker", [], None, "{layer?, tracker?}", has_track, delete),
|
||||
cmd!("track.editTargetDialog", "Edit Target...", [], None, "{}", has_track, |s, p| frontend(s, "track.editTargetDialog", p)),
|
||||
cmd!("track.optionsDialog", "Options...", [], None, "{}", has_track, |s, p| frontend(s, "track.optionsDialog", p)),
|
||||
query!("track.status", "Tracker Status", "{layer?, tracker?}", status),
|
||||
]
|
||||
}
|
||||
|
||||
// ---------- enablement ----------
|
||||
|
||||
fn trackable(l: &Layer) -> bool {
|
||||
l.has_video() && !matches!(l.source, effectcraft_project::LayerSource::Text | effectcraft_project::LayerSource::Shape)
|
||||
}
|
||||
|
||||
fn can_track(s: &Session) -> std::result::Result<(), String> {
|
||||
has_comp(s)?;
|
||||
let c = s.active_comp().ok_or("no composition is open")?;
|
||||
match s.state.selected_layers.first().and_then(|l| c.layer(*l)) {
|
||||
Some(l) if trackable(l) => Ok(()),
|
||||
Some(_) => Err("select a footage, composition or solid layer to track".into()),
|
||||
None => Err("select a layer to track".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn has_prop_selected(s: &Session) -> std::result::Result<(), String> {
|
||||
has_comp(s)?;
|
||||
if s.state.selected_props.is_empty() { Err("select a property to track".into()) } else { Ok(()) }
|
||||
}
|
||||
|
||||
fn has_track(s: &Session) -> std::result::Result<(), String> {
|
||||
current(s, &json!({}), "track").map(|_| ()).map_err(|e| e.to_string().replace("invalid parameters for `track`: ", ""))
|
||||
}
|
||||
|
||||
fn is_tracking(s: &Session) -> std::result::Result<(), String> {
|
||||
if s.is_tracking() { Ok(()) } else { Err("no track analysis is running".into()) }
|
||||
}
|
||||
|
||||
// ---------- addressing ----------
|
||||
|
||||
/// (comp, tracked layer, tracker uid) from `layer` / `tracker` or the Current Track.
|
||||
pub(crate) fn current(s: &Session, p: &V, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
|
||||
let cid = comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let lid = match p.get("layer") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?,
|
||||
None => match s.state.current_track {
|
||||
Some((l, _)) if comp.layer(l).is_some() => l,
|
||||
_ => s
|
||||
.state
|
||||
.selected_layers
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|l| comp.layer(*l).is_some_and(|l| l.trackers().next().is_some()))
|
||||
.ok_or_else(|| bad(cmd, "no current track: use Track Motion first"))?,
|
||||
},
|
||||
};
|
||||
let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
|
||||
let uid = match p.get("tracker") {
|
||||
Some(V::Number(n)) => {
|
||||
let n = n.as_u64().unwrap_or(0);
|
||||
layer.tracker(n).map(|(g, _)| g.uid).or_else(|| layer.trackers().nth((n as usize).saturating_sub(1)).map(|(g, _)| g.uid))
|
||||
}
|
||||
Some(V::String(name)) => match name.strip_prefix('#').and_then(|i| i.parse::<usize>().ok()) {
|
||||
Some(i) => layer.trackers().nth(i.saturating_sub(1)).map(|(g, _)| g.uid),
|
||||
None => layer.trackers().find(|(g, _)| &g.name == name).map(|(g, _)| g.uid),
|
||||
},
|
||||
_ => match s.state.current_track {
|
||||
Some((l, t)) if l == lid && layer.tracker(t).is_some() => Some(t),
|
||||
_ => layer.trackers().next().map(|(g, _)| g.uid),
|
||||
},
|
||||
}
|
||||
.ok_or_else(|| bad(cmd, format!("layer `{}` has no such tracker", layer.name)))?;
|
||||
Ok((cid, lid, uid))
|
||||
}
|
||||
|
||||
fn tracker_mut(proj: &mut effectcraft_project::Project, cid: ItemId, lid: LayerId, uid: Uid) -> Result<&mut PropGroup> {
|
||||
layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("tracker gone".into()))
|
||||
}
|
||||
|
||||
fn layer_size(s: &Session, cid: ItemId, l: &Layer) -> [f64; 2] {
|
||||
let (w, h) = effectcraft_render::source_size(&s.project, l);
|
||||
if w == 0 {
|
||||
let c = s.project.comp(cid);
|
||||
c.map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0])
|
||||
} else {
|
||||
[w as f64, h as f64]
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer directly above `lid` (the default Motion Target).
|
||||
fn layer_above(comp: &effectcraft_project::Comp, lid: LayerId) -> Option<LayerId> {
|
||||
let i = comp.layers.iter().position(|l| l.id == lid)?;
|
||||
i.checked_sub(1).map(|i| comp.layers[i].id)
|
||||
}
|
||||
|
||||
fn kind_p(p: &V, cmd: &str) -> Result<Option<TrackKind>> {
|
||||
match str_p(p, "kind") {
|
||||
Some(k) => TrackKind::from_name(k).map(Some).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (transform|stabilize|affine|perspective|raw)"))),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- create ----------
|
||||
|
||||
fn new_with(s: &mut Session, p: &V, kind: TrackKind, cmd: &str) -> Result<V> {
|
||||
let mut q = p.clone();
|
||||
q["kind"] = json!(kind.id());
|
||||
create(s, &q, cmd)
|
||||
}
|
||||
|
||||
fn new_tracker(s: &mut Session, p: &V) -> Result<V> {
|
||||
create(s, p, "track.new")
|
||||
}
|
||||
|
||||
fn create(s: &mut Session, p: &V, cmd: &str) -> Result<V> {
|
||||
let (cid, lid) = layer_p(s, p, cmd)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
if !trackable(layer) {
|
||||
return Err(bad(cmd, format!("layer `{}` has no pixels to track", layer.name)));
|
||||
}
|
||||
let kind = kind_p(p, cmd)?.unwrap_or_default();
|
||||
let mut st = TrackerSettings::new(kind);
|
||||
st.position = b_p(p, "position").unwrap_or(true);
|
||||
st.rotation = b_p(p, "rotation").unwrap_or(false);
|
||||
st.scale = b_p(p, "scale").unwrap_or(false);
|
||||
// Motion Target: the layer above the tracked layer (Stabilize applies to the layer itself).
|
||||
st.target = match p.get("target") {
|
||||
Some(V::Null) => None,
|
||||
Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no target layer {v}")))?),
|
||||
None if kind == TrackKind::Stabilize => Some(lid),
|
||||
None => layer_above(comp, lid),
|
||||
};
|
||||
let size = layer_size(s, cid, layer);
|
||||
let n_existing = layer.trackers().count();
|
||||
let name = format!("Tracker {}", n_existing + 1);
|
||||
let pts = tracking::default_points(kind, st.point_count(), size);
|
||||
let uid = s.edit(&format!("New {}", name), None, |proj, state| {
|
||||
let mut next = proj.next_id;
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
let mut ids = Ids(&mut next);
|
||||
let g = tracking::tracker(&mut ids, &name, st, &pts);
|
||||
let uid = g.uid;
|
||||
l.motion_trackers_mut(&mut ids).children.push(g.into());
|
||||
proj.next_id = next;
|
||||
state.current_track = Some((lid, uid));
|
||||
state.selected_layers = vec![lid];
|
||||
Ok(uid)
|
||||
})?;
|
||||
let _ = frontend(s, "window.panel", &json!({"panel": "tracker"}));
|
||||
Ok(json!({"tracker": uid, "name": name, "layer": lid.0, "points": pts.len()}))
|
||||
}
|
||||
|
||||
/// Track this Property: track another layer and drive the selected property's layer.
|
||||
fn track_property(s: &mut Session, p: &V) -> Result<V> {
|
||||
let cid = comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let target = match p.get("layer") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?,
|
||||
None => s.state.selected_props.first().map(|(l, _)| *l).ok_or_else(|| bad("track.property", "select a property first"))?,
|
||||
};
|
||||
let source = match p.get("source") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?,
|
||||
None => {
|
||||
let ti = comp.layers.iter().position(|l| l.id == target).unwrap_or(0);
|
||||
comp.layers
|
||||
.iter()
|
||||
.skip(ti + 1)
|
||||
.chain(comp.layers.iter().take(ti))
|
||||
.find(|l| l.id != target && trackable(l))
|
||||
.map(|l| l.id)
|
||||
.ok_or_else(|| bad("track.property", "no layer to track"))?
|
||||
}
|
||||
};
|
||||
create(s, &json!({"layer": source.0, "kind": "transform", "target": target.0, "comp": cid.0}), "track.property")
|
||||
}
|
||||
|
||||
// ---------- settings ----------
|
||||
|
||||
fn select(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (_, lid, uid) = current(s, p, "track.select")?;
|
||||
s.state.current_track = Some((lid, uid));
|
||||
s.bump();
|
||||
Ok(json!({"layer": lid.0, "tracker": uid}))
|
||||
}
|
||||
|
||||
fn set_type(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setType")?;
|
||||
let kind = kind_p(p, "track.setType")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let size = layer_size(s, cid, layer);
|
||||
let above = layer_above(comp, lid);
|
||||
let n = s.edit("Track Type", None, |proj, _| {
|
||||
let mut next = proj.next_id;
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let st = g.tracker_settings_mut().ok_or_else(|| bad("track.setType", "not a tracker"))?;
|
||||
if let Some(k) = kind {
|
||||
if k == TrackKind::Stabilize {
|
||||
st.target = Some(lid);
|
||||
} else if st.kind == TrackKind::Stabilize && st.target == Some(lid) {
|
||||
st.target = above;
|
||||
}
|
||||
st.kind = k;
|
||||
}
|
||||
for (k, f) in [("position", &mut st.position), ("rotation", &mut st.rotation), ("scale", &mut st.scale)] {
|
||||
if let Some(v) = b_p(p, k) {
|
||||
*f = v;
|
||||
}
|
||||
}
|
||||
let (kind, n) = (st.kind, st.point_count());
|
||||
// Add or remove Track Points to match the type.
|
||||
let have = g.track_points().count();
|
||||
if have > n {
|
||||
let mut seen = 0;
|
||||
g.children.retain(|c| match c {
|
||||
Node::Group(tp) if tp.match_id == tracking::TRACK_POINT => {
|
||||
seen += 1;
|
||||
seen <= n
|
||||
}
|
||||
_ => true,
|
||||
});
|
||||
} else if have < n {
|
||||
let defaults = tracking::default_points(kind, n, size);
|
||||
let mut ids = Ids(&mut next);
|
||||
for (i, d) in defaults.iter().enumerate().skip(have) {
|
||||
g.children.push(tracking::track_point(&mut ids, &format!("Track Point {}", i + 1), d).into());
|
||||
}
|
||||
}
|
||||
proj.next_id = next;
|
||||
Ok(n)
|
||||
})?;
|
||||
Ok(json!({"points": n}))
|
||||
}
|
||||
|
||||
fn set_target(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setTarget")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let target = match p.get("target") {
|
||||
None | Some(V::Null) => None,
|
||||
Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad("track.setTarget", format!("no layer {v}")))?),
|
||||
};
|
||||
s.edit("Edit Target", None, |proj, _| {
|
||||
let st = tracker_mut(proj, cid, lid, uid)?.tracker_settings_mut().ok_or_else(|| bad("track.setTarget", "not a tracker"))?;
|
||||
if st.kind == TrackKind::Stabilize && target.is_some_and(|t| t != lid) {
|
||||
return Err(bad("track.setTarget", "Stabilize applies to the tracked layer"));
|
||||
}
|
||||
st.target = target;
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(json!({"target": target.map(|t| t.0)}))
|
||||
}
|
||||
|
||||
fn options(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.options")?;
|
||||
let channel = match str_p(p, "channel").map(|c| c.to_ascii_lowercase()) {
|
||||
None => None,
|
||||
Some(c) => Some(match c.as_str() {
|
||||
"rgb" => TrackChannel::Rgb,
|
||||
"luminance" | "luma" => TrackChannel::Luminance,
|
||||
"saturation" => TrackChannel::Saturation,
|
||||
_ => return Err(bad("track.options", "channel: rgb|luminance|saturation")),
|
||||
}),
|
||||
};
|
||||
let action = match str_p(p, "action") {
|
||||
None => None,
|
||||
Some(a) => Some(match effectcraft_track::ConfidenceAction::from_name(a) {
|
||||
Some(effectcraft_track::ConfidenceAction::Continue) => LowConfidence::Continue,
|
||||
Some(effectcraft_track::ConfidenceAction::Stop) => LowConfidence::Stop,
|
||||
Some(effectcraft_track::ConfidenceAction::Extrapolate) => LowConfidence::Extrapolate,
|
||||
Some(effectcraft_track::ConfidenceAction::Adapt) => LowConfidence::Adapt,
|
||||
None => return Err(bad("track.options", "action: continue|stop|extrapolate|adapt")),
|
||||
}),
|
||||
};
|
||||
let r = s.edit("Motion Tracker Options", None, |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
if let Some(n) = str_p(p, "name") {
|
||||
g.name = n.to_string();
|
||||
}
|
||||
let o = &mut g.tracker_settings_mut().ok_or_else(|| bad("track.options", "not a tracker"))?.options;
|
||||
if let Some(c) = channel {
|
||||
o.channel = c;
|
||||
}
|
||||
if let Some(b) = f_p(p, "blur") {
|
||||
o.blur = b.clamp(0.0, 100.0);
|
||||
}
|
||||
if let Some(v) = b_p(p, "enhance") {
|
||||
o.enhance = v;
|
||||
}
|
||||
if let Some(v) = b_p(p, "subpixel") {
|
||||
o.subpixel = v;
|
||||
}
|
||||
if let Some(v) = b_p(p, "adaptEveryFrame") {
|
||||
o.adapt_every_frame = v;
|
||||
}
|
||||
if let Some(v) = f_p(p, "threshold") {
|
||||
o.threshold = v.clamp(0.0, 100.0);
|
||||
}
|
||||
if let Some(a) = action {
|
||||
o.action = a;
|
||||
}
|
||||
if let Some(v) = b_p(p, "trackShape") {
|
||||
o.track_shape = v;
|
||||
}
|
||||
Ok(serde_json::to_value(&*o).unwrap_or(V::Null))
|
||||
})?;
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
fn vec2_p(p: &V, k: &str) -> Option<[f64; 2]> {
|
||||
let a = p.get(k)?.as_array()?;
|
||||
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?])
|
||||
}
|
||||
|
||||
fn set_point(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setPoint")?;
|
||||
let n = p.get("point").and_then(V::as_u64).unwrap_or(1).max(1) as usize;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let t = f_p(p, "time").map(Tick::from_seconds_f64).map(|t| comp.frame_rate.snap_nearest(t)).unwrap_or_else(|| s.time());
|
||||
let lt = layer.layer_time(t);
|
||||
let merge = super::merge_p(p).map(str::to_string);
|
||||
s.edit("Move Track Point", merge.as_deref(), |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let tp_uid = g.track_points().nth(n - 1).map(|g| g.uid).ok_or_else(|| bad("track.setPoint", format!("no track point {n}")))?;
|
||||
let tp = g.find_group_mut(tp_uid).ok_or_else(|| bad("track.setPoint", "no track point"))?;
|
||||
let cur = point_values(tp, lt);
|
||||
let mut center = vec2_p(p, "center").unwrap_or(cur.spec.center);
|
||||
if let Some(d) = vec2_p(p, "move") {
|
||||
center = [center[0] + d[0], center[1] + d[1]];
|
||||
}
|
||||
let set = |tp: &mut PropGroup, m: &str, v: [f64; 2]| {
|
||||
if let Some(pr) = tp.get_mut(m) {
|
||||
pr.set_value_at(lt, Value::Vec2(v));
|
||||
}
|
||||
};
|
||||
set(tp, "featureCenter", center);
|
||||
for (k, m) in [("featureSize", "featureSize"), ("searchOffset", "searchOffset"), ("searchSize", "searchSize"), ("attachOffset", "attachPointOffset")] {
|
||||
if let Some(v) = vec2_p(p, k) {
|
||||
let v = if k.ends_with("Size") { [v[0].max(4.0), v[1].max(4.0)] } else { v };
|
||||
set(tp, m, v);
|
||||
}
|
||||
}
|
||||
let off = tp.get("attachPointOffset").map(|pr| pr.value_at(lt).as_vec2()).unwrap_or([0.0; 2]);
|
||||
set(tp, "attachPoint", [center[0] + off[0], center[1] + off[1]]);
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
// ---------- analyze / apply / reset ----------
|
||||
|
||||
fn analyze(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.analyze")?;
|
||||
let dir = match str_p(p, "direction") {
|
||||
Some(d) => Direction::from_name(d).ok_or_else(|| bad("track.analyze", "direction: forward|backward|frameForward|frameBackward"))?,
|
||||
None => Direction::Forward,
|
||||
};
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let (tg, settings) = layer.tracker(uid).ok_or_else(|| bad("track.analyze", "no tracker"))?;
|
||||
let fd = comp.frame_duration();
|
||||
let now = comp.frame_rate.snap_nearest(if s.state.active_comp == Some(cid) { s.time() } else { Tick::ZERO });
|
||||
let start = f_p(p, "start").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v)));
|
||||
let end = f_p(p, "end").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v)));
|
||||
let lo = layer.in_point.max(Tick::ZERO);
|
||||
let hi = layer.out_point.min(comp.duration) - fd;
|
||||
let mut times = vec![];
|
||||
match dir {
|
||||
Direction::Forward | Direction::FrameForward => {
|
||||
let mut t = start.unwrap_or(now).clamp(lo, hi);
|
||||
let stop = if dir == Direction::FrameForward { t + fd } else { end.unwrap_or(hi).min(hi) };
|
||||
while t <= stop && t <= hi {
|
||||
times.push(t);
|
||||
t += fd;
|
||||
}
|
||||
}
|
||||
Direction::Backward | Direction::FrameBackward => {
|
||||
let mut t = end.unwrap_or(now).clamp(lo, hi);
|
||||
let stop = if dir == Direction::FrameBackward { t - fd } else { start.unwrap_or(lo).max(lo) };
|
||||
while t >= stop && t >= lo {
|
||||
times.push(t);
|
||||
t -= fd;
|
||||
}
|
||||
}
|
||||
}
|
||||
if times.len() < 2 {
|
||||
return Err(bad("track.analyze", "nothing to analyze in that direction (at the layer's end)"));
|
||||
}
|
||||
let lt0 = layer.layer_time(times[0]);
|
||||
let points: Vec<_> = tg.track_points().map(|g| point_values(g, lt0)).collect();
|
||||
if points.is_empty() {
|
||||
return Err(bad("track.analyze", "the tracker has no track points"));
|
||||
}
|
||||
let work = Work {
|
||||
project: s.project.clone(),
|
||||
footage: s.footage.clone(),
|
||||
expr: s.expr.clone(),
|
||||
cache: s.layer_cache.clone(),
|
||||
comp: cid,
|
||||
layer: lid,
|
||||
settings: settings.clone(),
|
||||
points,
|
||||
times: times.clone(),
|
||||
};
|
||||
s.state.current_track = Some((lid, uid));
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
s.start_track(work, uid, dir, wait).map_err(EngineError::Other)?;
|
||||
let prog = s.track_progress();
|
||||
Ok(json!({
|
||||
"frames": times.len() - 1,
|
||||
"start": times[0].seconds(),
|
||||
"end": times.last().map(|t| t.seconds()),
|
||||
"running": s.is_tracking(),
|
||||
"progress": prog,
|
||||
}))
|
||||
}
|
||||
|
||||
fn apply(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.apply")?;
|
||||
let dims = match str_p(p, "dimensions") {
|
||||
Some(d) => Dims::from_name(d).ok_or_else(|| bad("track.apply", "dimensions: xy|x|y"))?,
|
||||
None => Dims::XY,
|
||||
};
|
||||
if s.is_tracking() {
|
||||
return Err(bad("track.apply", "wait for the analysis to finish"));
|
||||
}
|
||||
let n = s.edit("Apply Track", None, |proj, _| crate::tracking::apply(proj, cid, lid, uid, dims).map_err(|e| bad("track.apply", e)))?;
|
||||
Ok(json!({"frames": n}))
|
||||
}
|
||||
|
||||
fn reset(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.reset")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let size = layer_size(s, cid, layer);
|
||||
s.edit("Reset Track", None, |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let st = g.tracker_settings().cloned().unwrap_or_default();
|
||||
let n = g.track_points().count();
|
||||
let defaults = tracking::default_points(st.kind, n, size);
|
||||
let mut k = 0;
|
||||
for c in &mut g.children {
|
||||
let Node::Group(tp) = c else { continue };
|
||||
if tp.match_id != tracking::TRACK_POINT {
|
||||
continue;
|
||||
}
|
||||
let d = defaults.get(k).copied().unwrap_or(defaults[0]);
|
||||
k += 1;
|
||||
for (m, v) in [
|
||||
("featureCenter", Value::Vec2(d.center)),
|
||||
("featureSize", Value::Vec2(d.feature_size)),
|
||||
("searchOffset", Value::Vec2(d.search_offset)),
|
||||
("searchSize", Value::Vec2(d.search_size)),
|
||||
("confidence", Value::Scalar(0.0)),
|
||||
("attachPoint", Value::Vec2(d.center)),
|
||||
("attachPointOffset", Value::Vec2(d.attach_offset)),
|
||||
] {
|
||||
if let Some(pr) = tp.get_mut(m) {
|
||||
pr.keys.clear();
|
||||
pr.value = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
fn delete(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.delete")?;
|
||||
s.edit("Delete Tracker", None, |proj, st| {
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
if let Some(mt) = l.props.sub_mut(tracking::MOTION_TRACKERS) {
|
||||
mt.children.retain(|c| c.uid() != uid);
|
||||
if mt.children.is_empty() {
|
||||
l.props.children.retain(|c| c.match_id() != tracking::MOTION_TRACKERS);
|
||||
}
|
||||
}
|
||||
if st.current_track == Some((lid, uid)) {
|
||||
st.current_track = None;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
fn status(s: &mut Session, p: &V) -> Result<V> {
|
||||
s.poll_track();
|
||||
let progress = s.track_progress();
|
||||
let cur = current(s, p, "track.status").ok();
|
||||
let mut out = json!({"running": s.is_tracking(), "progress": progress});
|
||||
if let Some((cid, lid, uid)) = cur
|
||||
&& let Some(layer) = s.project.comp(cid).and_then(|c| c.layer(lid))
|
||||
&& let Some((g, st)) = layer.tracker(uid)
|
||||
{
|
||||
let points: Vec<V> = g
|
||||
.track_points()
|
||||
.map(|tp| {
|
||||
let keys = tp.get("featureCenter").map(|p| p.keys.len()).unwrap_or(0);
|
||||
let v = point_values(tp, layer.layer_time(s.time()));
|
||||
json!({"name": tp.name, "uid": tp.uid, "center": v.spec.center, "featureSize": v.spec.feature_size, "searchOffset": v.spec.search_offset, "searchSize": v.spec.search_size, "attachOffset": v.attach_offset, "keys": keys})
|
||||
})
|
||||
.collect();
|
||||
let trackers: Vec<V> = layer.trackers().map(|(g, _)| json!({"uid": g.uid, "name": g.name})).collect();
|
||||
out["layer"] = json!(lid.0);
|
||||
out["tracker"] = json!({"uid": uid, "name": g.name, "settings": st, "points": points});
|
||||
out["trackers"] = json!(trackers);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
@@ -13,6 +13,7 @@ pub mod demo;
|
||||
pub mod links;
|
||||
pub mod menus;
|
||||
pub mod render_queue;
|
||||
pub mod tracking;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -157,6 +158,9 @@ pub struct EditorState {
|
||||
/// Viewer 3D view per comp (Active Camera / Front / … / Custom View 3 and edited view cameras).
|
||||
#[serde(default)]
|
||||
pub views3d: std::collections::BTreeMap<ItemId, effectcraft_render::three_d::Views3D>,
|
||||
/// Tracker panel ▸ Current Track: (tracked layer, tracker group uid) in the active comp.
|
||||
#[serde(default)]
|
||||
pub current_track: Option<(LayerId, Uid)>,
|
||||
}
|
||||
|
||||
/// What Copy with Property Links / Copy Expression Only put on the clipboard.
|
||||
@@ -207,6 +211,8 @@ pub struct Session {
|
||||
pub exporter: Option<Arc<dyn Exporter>>,
|
||||
/// The running (or finished, not yet polled) render.
|
||||
pub render_job: Option<render_queue::RenderJob>,
|
||||
/// The running (or finished, not yet polled) track analysis.
|
||||
pub track_job: Option<tracking::TrackJob>,
|
||||
pub events: Vec<Event>,
|
||||
/// Commands executed: (id, params).
|
||||
pub journal: Vec<(String, Value)>,
|
||||
@@ -230,6 +236,7 @@ impl Default for Session {
|
||||
importer: None,
|
||||
exporter: None,
|
||||
render_job: None,
|
||||
track_job: None,
|
||||
events: vec![],
|
||||
journal: vec![],
|
||||
layer_cache: Arc::new(LayerCache::default()),
|
||||
@@ -336,6 +343,11 @@ impl Session {
|
||||
self.state.selected_keys.retain(|k| ids.contains(&k.layer));
|
||||
self.state.selected_vertices.retain(|v| ids.contains(&v.layer));
|
||||
}
|
||||
if let Some((l, t)) = self.state.current_track
|
||||
&& self.active_comp().and_then(|c| c.layer(l)).and_then(|l| l.tracker(t)).is_none()
|
||||
{
|
||||
self.state.current_track = None;
|
||||
}
|
||||
self.state.project_selection.retain(|i| p.item(*i).is_some());
|
||||
}
|
||||
|
||||
@@ -380,6 +392,7 @@ impl Session {
|
||||
self.state.selected_props.clear();
|
||||
self.state.selected_keys.clear();
|
||||
self.state.selected_vertices.clear();
|
||||
self.state.current_track = None;
|
||||
}
|
||||
self.events.push(Event::OpenComp(id));
|
||||
}
|
||||
@@ -437,6 +450,8 @@ impl Session {
|
||||
job.wait();
|
||||
}
|
||||
self.render_job = None;
|
||||
self.stop_track();
|
||||
self.track_job = None;
|
||||
p.fix_next_id();
|
||||
self.project = Arc::new(p);
|
||||
self.history = History::default();
|
||||
@@ -467,6 +482,8 @@ mod tests_styles;
|
||||
mod tests_text;
|
||||
#[cfg(test)]
|
||||
mod tests_timeline;
|
||||
#[cfg(test)]
|
||||
mod tests_track;
|
||||
|
||||
/// Font families available to text layers (bundled + scanned system fonts).
|
||||
/// Text animation presets: (id, name), for menus.
|
||||
|
||||
@@ -788,6 +788,7 @@ Animation
|
||||
Track Camera | track.camera
|
||||
Warp Stabilizer VFX | track.warpStabilizer
|
||||
Track Motion | track.motion
|
||||
Stabilize Motion | track.stabilize
|
||||
Track Mask | track.mask
|
||||
Track this Property | track.property
|
||||
---
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
//! Motion tracking through the commands: synthetic footage generated in-test (a textured patch on
|
||||
//! a known path, or a whole frame under a known homography), analysis (blocking and background),
|
||||
//! apply (Transform, Stabilize, Corner Pin), undo/redo and serde.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_color::Label;
|
||||
use effectcraft_project::build;
|
||||
use effectcraft_project::tracking::TrackKind;
|
||||
use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, FootageSource};
|
||||
use effectcraft_time::{FrameRate, Tick};
|
||||
use effectcraft_track::Homography;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::Session;
|
||||
|
||||
const W: u32 = 320;
|
||||
const H: u32 = 240;
|
||||
const FPS: u32 = 25;
|
||||
|
||||
fn texture(x: f64, y: f64, seed: u32) -> f32 {
|
||||
let mut v = 0.0f64;
|
||||
let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345);
|
||||
let mut rnd = || {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 17;
|
||||
s ^= s << 5;
|
||||
(s % 10_000) as f64 / 10_000.0
|
||||
};
|
||||
for _ in 0..6 {
|
||||
let ang = rnd() * std::f64::consts::TAU;
|
||||
let period = 9.0 + rnd() * 25.0;
|
||||
let phase = rnd() * std::f64::consts::TAU;
|
||||
let k = std::f64::consts::TAU / period;
|
||||
v += (k * (x * ang.cos() + y * ang.sin()) + phase).sin() * (0.4 + rnd()) * 0.1;
|
||||
}
|
||||
(0.5 + v).clamp(0.0, 1.0) as f32
|
||||
}
|
||||
|
||||
/// Footage computed per frame (cached).
|
||||
struct Synth {
|
||||
make: Box<dyn Fn(u32) -> Image + Send + Sync>,
|
||||
cache: Mutex<HashMap<u32, Arc<Image>>>,
|
||||
}
|
||||
|
||||
impl FootageSource for Synth {
|
||||
fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
let f = (t.seconds() * FPS as f64).round().max(0.0) as u32;
|
||||
let mut c = self.cache.lock().unwrap();
|
||||
Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Patch pose on frame `f`: centre, rotation (degrees), scale.
|
||||
type Pose = fn(u32) -> ([f64; 2], f64, f64);
|
||||
|
||||
fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync {
|
||||
move |f| {
|
||||
let (c, rot, sc) = pose(f);
|
||||
let (sn, cs) = rot.to_radians().sin_cos();
|
||||
let mut img = Image::new(W, H);
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let d = [(p[0] - c[0]) / sc, (p[1] - c[1]) / sc];
|
||||
let l = [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]];
|
||||
let v = if l[0].abs() <= 50.0 && l[1].abs() <= 50.0 { texture(l[0], l[1], 7) } else { 0.6 * texture(p[0] * 0.7, p[1] * 0.7, 3) };
|
||||
img.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
}
|
||||
|
||||
/// A comp with the synthetic clip (layer `clip`) and a 100 × 100 solid above it (layer `solid`).
|
||||
fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) {
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "Track", "width": W, "height": H, "frameRate": FPS, "duration": 1})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let clip = s
|
||||
.edit("clip", None, |p, _| {
|
||||
let fr = FrameRate::new(FPS as i64, 1);
|
||||
let footage = Footage {
|
||||
path: "synthetic.mov".into(),
|
||||
kind: FootageKind::Video,
|
||||
width: W,
|
||||
height: H,
|
||||
pixel_aspect: 1.0,
|
||||
frame_rate: fr,
|
||||
native_rate: None,
|
||||
duration: Tick::from_seconds_f64(1.0),
|
||||
has_video: true,
|
||||
has_audio: false,
|
||||
alpha: Default::default(),
|
||||
premul_color: [0.0; 3],
|
||||
loop_count: 1,
|
||||
codec: String::new(),
|
||||
missing: false,
|
||||
sequence: vec![],
|
||||
};
|
||||
let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage));
|
||||
let comp = p.comp(cid).unwrap().clone();
|
||||
let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (W, H), None);
|
||||
let id = l.id;
|
||||
p.comp_mut(cid).unwrap().layers.insert(0, l);
|
||||
Ok(id)
|
||||
})
|
||||
.unwrap();
|
||||
s.footage = Arc::new(Synth { make: Box::new(make), cache: Mutex::default() });
|
||||
let solid = s.execute("layer.newSolid", json!({"width": 100, "height": 100, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap();
|
||||
s.execute("layer.select", json!({"layers": [clip.0]})).unwrap();
|
||||
(s, clip, LayerId(solid))
|
||||
}
|
||||
|
||||
fn tracker_points(s: &Session, layer: LayerId) -> Vec<effectcraft_project::PropGroup> {
|
||||
let l = s.active_comp().unwrap().layer(layer).unwrap();
|
||||
l.trackers().next().unwrap().0.track_points().cloned().collect()
|
||||
}
|
||||
|
||||
fn frame_time(f: u32) -> Tick {
|
||||
FrameRate::new(FPS as i64, 1).tick_of(f as i64)
|
||||
}
|
||||
|
||||
fn dist(a: [f64; 2], b: [f64; 2]) -> f64 {
|
||||
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt()
|
||||
}
|
||||
|
||||
fn translate_pose(f: u32) -> ([f64; 2], f64, f64) {
|
||||
let t = f as f64;
|
||||
([150.0 + 2.37 * t + 4.0 * (t * 0.4).sin(), 110.0 + 1.13 * t], 0.0, 1.0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_motion_analyze_apply_transform_undo_redo() {
|
||||
let (mut s, clip, solid) = setup(patch_frames(translate_pose));
|
||||
// The menu entry and the Tracker panel's Track Motion button.
|
||||
assert!(s.is_enabled("track.motion"));
|
||||
let r = s.execute("track.motion", json!({})).unwrap();
|
||||
assert_eq!(r["points"], 1);
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["tracker"]["settings"]["target"], json!(solid.0), "{st}");
|
||||
// Put the track point on the patch and analyze to the end (blocking).
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
let r = s.execute("track.analyze", json!({"direction": "forward", "wait": true})).unwrap();
|
||||
assert_eq!(r["frames"], 24);
|
||||
s.poll_track();
|
||||
assert!(s.track_job.is_none());
|
||||
let tp = &tracker_points(&s, clip)[0];
|
||||
let fc = tp.get("featureCenter").unwrap();
|
||||
assert_eq!(fc.keys.len(), 25);
|
||||
let conf = tp.get("confidence").unwrap();
|
||||
let mut worst: f64 = 0.0;
|
||||
for f in 0..25 {
|
||||
let t = frame_time(f);
|
||||
worst = worst.max(dist(fc.value_at(t).as_vec2(), translate_pose(f).0));
|
||||
assert!(conf.value_at(t).as_f64() > 90.0);
|
||||
}
|
||||
assert!(worst < 0.25, "worst {worst}");
|
||||
// CTI moved to the last analysed frame.
|
||||
assert_eq!(s.time(), frame_time(24));
|
||||
// Apply: the solid's position follows the attach point.
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
|
||||
assert_eq!(pos.keys.len(), 25);
|
||||
for f in [0, 7, 24] {
|
||||
let v = pos.value_at(frame_time(f)).as_vec2();
|
||||
assert!(dist(v, translate_pose(f).0) < 0.25, "frame {f}: {v:?}");
|
||||
}
|
||||
// Undo the apply, then the analysis; redo both.
|
||||
assert!(s.undo());
|
||||
assert!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.is_empty());
|
||||
assert!(s.undo());
|
||||
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
|
||||
assert!(s.redo());
|
||||
assert_eq!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.len(), 25);
|
||||
assert!(s.redo());
|
||||
assert_eq!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.len(), 25);
|
||||
// X only keeps the solid's Y.
|
||||
s.undo();
|
||||
s.execute("track.apply", json!({"dimensions": "x"})).unwrap();
|
||||
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
|
||||
let v = pos.value_at(frame_time(10)).as_vec2();
|
||||
assert!((v[0] - translate_pose(10).0[0]).abs() < 0.25 && (v[1] - 120.0).abs() < 1e-9, "{v:?}");
|
||||
// Serde round trip keeps the tracker.
|
||||
let json = s.project.to_json();
|
||||
let back = effectcraft_project::Project::from_json(&json).unwrap();
|
||||
let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap();
|
||||
let (g, st) = l.trackers().next().unwrap();
|
||||
assert_eq!(st.kind, TrackKind::Transform);
|
||||
assert_eq!(g.track_points().next().unwrap().get("featureCenter").unwrap().keys.len(), 25);
|
||||
assert_eq!(l.props.children[0].match_id(), "motionTrackers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stabilize_keeps_the_feature_fixed() {
|
||||
let (mut s, clip, _) = setup(patch_frames(translate_pose));
|
||||
s.execute("track.stabilize", json!({})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
s.execute("track.analyze", json!({"wait": true})).unwrap();
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let comp = s.project.comp(cid).unwrap().clone();
|
||||
let l = comp.layer(clip).unwrap();
|
||||
let at = |f: u32| {
|
||||
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(f));
|
||||
let (m, _) = ctx.layer_to_comp(l);
|
||||
let c = translate_pose(f).0;
|
||||
let q = m.apply(effectcraft_geom::vec2(c[0], c[1]));
|
||||
[q.x, q.y]
|
||||
};
|
||||
let p0 = at(0);
|
||||
// The layer itself didn't move on the first frame.
|
||||
assert!(dist(p0, translate_pose(0).0) < 1e-6, "{p0:?}");
|
||||
let worst = (1..25).map(|f| dist(at(f), p0)).fold(0.0, f64::max);
|
||||
assert!(worst < 0.25, "feature drifts by {worst} px");
|
||||
}
|
||||
|
||||
fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) {
|
||||
let t = f as f64;
|
||||
([160.0 + 1.2 * t, 120.0 - 0.7 * t], 1.5 * t, 1.0 + 0.008 * t)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transform_rotation_scale_and_background_job() {
|
||||
let (mut s, clip, solid) = setup(patch_frames(rotating_pose));
|
||||
s.execute("track.new", json!({"kind": "transform", "rotation": true, "scale": true})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 2);
|
||||
let (c, _, _) = rotating_pose(0);
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": [c[0] - 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 2, "center": [c[0] + 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
|
||||
// Background analysis with progress, polled like the UI does.
|
||||
let r = s.execute("track.analyze", json!({"direction": "forward", "end": 0.6})).unwrap();
|
||||
assert_eq!(r["frames"], 15);
|
||||
let t0 = std::time::Instant::now();
|
||||
while s.is_tracking() {
|
||||
assert!(t0.elapsed().as_secs() < 120);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
s.poll_track();
|
||||
}
|
||||
s.poll_track();
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["running"], false);
|
||||
assert_eq!(st["tracker"]["points"][0]["keys"], 16);
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let tr = s.active_comp().unwrap().layer(solid).unwrap().props.sub("transform").unwrap().clone();
|
||||
let rot = tr.get("rotation").unwrap();
|
||||
let sc = tr.get("scale").unwrap();
|
||||
for f in [5, 10, 15] {
|
||||
let (_, r, k) = rotating_pose(f);
|
||||
let got = rot.value_at(frame_time(f)).as_f64();
|
||||
assert!((got - r).abs() < 0.3, "frame {f}: rotation {got} vs {r}");
|
||||
let gs = sc.value_at(frame_time(f)).as_vec3()[0];
|
||||
assert!((gs - 100.0 * k).abs() < 0.5, "frame {f}: scale {gs} vs {}", 100.0 * k);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perspective_corner_pin_follows_a_known_homography() {
|
||||
// Frame f shows the texture under H_f (identity → `end` over 12 frames).
|
||||
let end = Homography([[1.04, 0.05, 6.0], [-0.03, 0.97, 4.0], [0.00012, -0.00008, 1.0]]);
|
||||
let hf = move |f: u32| {
|
||||
let k = f as f64 / 12.0;
|
||||
let mut m = [[0.0; 3]; 3];
|
||||
for i in 0..3 {
|
||||
for j in 0..3 {
|
||||
let id = if i == j { 1.0 } else { 0.0 };
|
||||
m[i][j] = id + (end.0[i][j] - id) * k;
|
||||
}
|
||||
}
|
||||
Homography(m)
|
||||
};
|
||||
let inv = move |f: u32, p: [f64; 2]| {
|
||||
let h = hf(f);
|
||||
// Solve H x = p by mapping four points back (exact for a homography).
|
||||
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
|
||||
let dst = src.map(|q| h.apply(q));
|
||||
Homography::from_points(dst, src).unwrap().apply(p)
|
||||
};
|
||||
let make = move |f: u32| {
|
||||
let h = hf(f);
|
||||
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
|
||||
let back = Homography::from_points(src.map(|q| h.apply(q)), src).unwrap();
|
||||
let _ = inv;
|
||||
let mut img = Image::new(W, H);
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let q = back.apply([x as f64 + 0.5, y as f64 + 0.5]);
|
||||
let v = texture(q[0], q[1], 11);
|
||||
img.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
};
|
||||
let (mut s, clip, solid) = setup(make);
|
||||
s.execute("track.new", json!({"kind": "perspective"})).unwrap();
|
||||
let corners = [[90.0, 70.0], [230.0, 70.0], [90.0, 170.0], [230.0, 170.0]];
|
||||
for (i, c) in corners.iter().enumerate() {
|
||||
s.execute("track.setPoint", json!({"point": i + 1, "center": c, "featureSize": [30, 30], "searchSize": [60, 60]})).unwrap();
|
||||
}
|
||||
s.execute("track.analyze", json!({"wait": true, "end": 12.0 / FPS as f64})).unwrap();
|
||||
let pts = tracker_points(&s, clip);
|
||||
for (i, tp) in pts.iter().enumerate() {
|
||||
let got = tp.get("featureCenter").unwrap().value_at(frame_time(12)).as_vec2();
|
||||
assert!(dist(got, end.apply(corners[i])) < 0.3, "point {i}: {got:?} vs {:?}", end.apply(corners[i]));
|
||||
}
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let l = s.active_comp().unwrap().layer(solid).unwrap().clone();
|
||||
let fx = l.effects().unwrap().groups().next().unwrap().clone();
|
||||
assert_eq!(fx.match_id, "ec.distort.cornerpin");
|
||||
// The solid is 100 × 100, centred: comp = layer + (110, 70).
|
||||
for (m, i) in [("ul", 0), ("ur", 1), ("ll", 2), ("lr", 3)] {
|
||||
let p = fx.get(m).unwrap();
|
||||
assert_eq!(p.keys.len(), 13);
|
||||
for f in [6, 12] {
|
||||
let want = hf(f).apply(corners[i]);
|
||||
let got = p.value_at(frame_time(f)).as_vec2();
|
||||
assert!(dist([got[0] + 110.0, got[1] + 70.0], want) < 0.35, "{m} frame {f}: {got:?} vs {want:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_frame_steps_options_and_reset() {
|
||||
let (mut s, clip, _) = setup(patch_frames(translate_pose));
|
||||
s.execute("track.motion", json!({})).unwrap();
|
||||
s.execute("time.set", json!({"time": 10.0 / FPS as f64})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(10).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
s.execute("track.options", json!({"channel": "rgb", "blur": 2, "threshold": 70, "action": "extrapolate", "name": "Patch"})).unwrap();
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["tracker"]["name"], "Patch");
|
||||
assert_eq!(st["tracker"]["settings"]["options"]["channel"], "rgb");
|
||||
s.execute("track.analyze", json!({"direction": "frameBackward", "wait": true})).unwrap();
|
||||
s.execute("track.analyze", json!({"direction": "backward", "wait": true})).unwrap();
|
||||
assert_eq!(s.time(), Tick::ZERO);
|
||||
s.execute("track.analyze", json!({"direction": "frameForward", "wait": true, "start": 10.0 / FPS as f64})).unwrap();
|
||||
let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone();
|
||||
assert_eq!(fc.keys.len(), 12);
|
||||
for f in 0..12 {
|
||||
assert!(dist(fc.value_at(frame_time(f)).as_vec2(), translate_pose(f).0) < 0.25, "frame {f}");
|
||||
}
|
||||
// Unknown params are rejected for agents.
|
||||
assert!(s.execute_checked("track.analyze", json!({"dir": "forward"})).is_err());
|
||||
s.execute("track.setType", json!({"kind": "perspective"})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 4);
|
||||
s.execute("track.setType", json!({"kind": "transform", "rotation": false})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 1);
|
||||
s.execute("track.reset", json!({})).unwrap();
|
||||
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
|
||||
s.execute("track.delete", json!({})).unwrap();
|
||||
assert!(s.active_comp().unwrap().layer(clip).unwrap().motion_trackers().is_none());
|
||||
assert!(s.execute("track.analyze", json!({})).is_err());
|
||||
}
|
||||
|
||||
/// End-to-end analysis speed at 1080p for one track point: footage frames through the renderer's
|
||||
/// layer source and the tracker (frames are prepared before timing).
|
||||
#[test]
|
||||
fn analyze_1080p_single_point_speed() {
|
||||
let (w, h) = (1920u32, 1080u32);
|
||||
let mut base = Image::new(w + 64, h + 64);
|
||||
for y in 0..base.height {
|
||||
for x in 0..base.width {
|
||||
let v = 0.5 + 0.25 * ((x as f32 * 0.21).sin() * (y as f32 * 0.13).cos()) + 0.15 * ((x + 2 * y) as f32 * 0.07).sin();
|
||||
base.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
let frames: Vec<Arc<Image>> = (0..31u32)
|
||||
.map(|f| {
|
||||
let (dx, dy) = (f as i64, (f / 2) as i64);
|
||||
let mut img = Image::new(w, h);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
img.set(x, y, base.get(x as i64 + 32 - dx, y as i64 + 32 - dy));
|
||||
}
|
||||
}
|
||||
Arc::new(img)
|
||||
})
|
||||
.collect();
|
||||
struct Pre(Vec<Arc<Image>>);
|
||||
impl FootageSource for Pre {
|
||||
fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
let f = (t.seconds() * FPS as f64).round().max(0.0) as usize;
|
||||
self.0.get(f.min(self.0.len() - 1)).cloned()
|
||||
}
|
||||
}
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "HD", "width": w, "height": h, "frameRate": FPS, "duration": 2})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let clip = s
|
||||
.edit("clip", None, |p, _| {
|
||||
let footage = Footage {
|
||||
path: "hd.mov".into(),
|
||||
kind: FootageKind::Video,
|
||||
width: w,
|
||||
height: h,
|
||||
pixel_aspect: 1.0,
|
||||
frame_rate: FrameRate::new(FPS as i64, 1),
|
||||
native_rate: None,
|
||||
duration: Tick::from_seconds_f64(2.0),
|
||||
has_video: true,
|
||||
has_audio: false,
|
||||
alpha: Default::default(),
|
||||
premul_color: [0.0; 3],
|
||||
loop_count: 1,
|
||||
codec: String::new(),
|
||||
missing: false,
|
||||
sequence: vec![],
|
||||
};
|
||||
let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage));
|
||||
let comp = p.comp(cid).unwrap().clone();
|
||||
let l = build::layer(p, &comp, "HD", LayerSource::Footage { item }, (w, h), None);
|
||||
let id = l.id;
|
||||
p.comp_mut(cid).unwrap().layers.insert(0, l);
|
||||
Ok(id)
|
||||
})
|
||||
.unwrap();
|
||||
s.footage = Arc::new(Pre(frames));
|
||||
s.execute("track.motion", json!({"layer": clip.0})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": [960, 540], "featureSize": [48, 48], "searchSize": [128, 128]})).unwrap();
|
||||
let t0 = std::time::Instant::now();
|
||||
s.execute("track.analyze", json!({"wait": true, "end": 30.0 / FPS as f64})).unwrap();
|
||||
let secs = t0.elapsed().as_secs_f64();
|
||||
let fps = 30.0 / secs;
|
||||
eprintln!("1080p single-point analysis: {fps:.0} frames/s end to end ({:.1} ms/frame)", secs * 1e3 / 30.0);
|
||||
let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone();
|
||||
assert!(dist(fc.value_at(frame_time(30)).as_vec2(), [990.0, 555.0]) < 0.25);
|
||||
assert!(fps > 30.0, "{fps} frames/s");
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
//! Motion tracking runtime: the analysis job (background thread with progress and cancel, or
|
||||
//! blocking for the CLI / agents / wasm), writing track results into the tracker's Track Point
|
||||
//! properties, and applying tracks (Transform, Stabilize, Corner Pin) to layers.
|
||||
//!
|
||||
//! The algorithms are in `effectcraft-track`; the property layout in
|
||||
//! `effectcraft_project::tracking`.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_geom::{Mat3, vec2};
|
||||
use effectcraft_project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions, TrackerSettings};
|
||||
use effectcraft_project::{Comp, ItemId, Keyframe, Layer, LayerId, Project, PropGroup, Uid, Value};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track as trk;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Tracker panel ▸ Analyze buttons.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
#[default]
|
||||
Forward,
|
||||
Backward,
|
||||
FrameForward,
|
||||
FrameBackward,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn from_name(s: &str) -> Option<Direction> {
|
||||
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
|
||||
Some(match n.as_str() {
|
||||
"forward" | "fwd" => Direction::Forward,
|
||||
"backward" | "back" | "bwd" => Direction::Backward,
|
||||
"frameforward" | "oneframeforward" | "1frameforward" => Direction::FrameForward,
|
||||
"framebackward" | "oneframebackward" | "1framebackward" => Direction::FrameBackward,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
pub fn forward(self) -> bool {
|
||||
matches!(self, Direction::Forward | Direction::FrameForward)
|
||||
}
|
||||
}
|
||||
|
||||
/// Track options for the algorithms.
|
||||
pub fn track_options(o: &TrackerOptions) -> trk::TrackOptions {
|
||||
trk::TrackOptions {
|
||||
channel: match o.channel {
|
||||
TrackChannel::Rgb => trk::Channel::Rgb,
|
||||
TrackChannel::Luminance => trk::Channel::Luminance,
|
||||
TrackChannel::Saturation => trk::Channel::Saturation,
|
||||
},
|
||||
blur: o.blur,
|
||||
enhance: o.enhance,
|
||||
subpixel: o.subpixel,
|
||||
adapt_every_frame: o.adapt_every_frame,
|
||||
threshold: o.threshold,
|
||||
action: match o.action {
|
||||
LowConfidence::Continue => trk::ConfidenceAction::Continue,
|
||||
LowConfidence::Stop => trk::ConfidenceAction::Stop,
|
||||
LowConfidence::Extrapolate => trk::ConfidenceAction::Extrapolate,
|
||||
LowConfidence::Adapt => trk::ConfidenceAction::Adapt,
|
||||
},
|
||||
track_shape: o.track_shape,
|
||||
}
|
||||
}
|
||||
|
||||
/// A track point's regions and attach offset read from its properties at a layer time.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct PointValues {
|
||||
pub uid: Uid,
|
||||
pub spec: trk::PointSpec,
|
||||
pub attach_offset: [f64; 2],
|
||||
}
|
||||
|
||||
pub fn point_values(tp: &PropGroup, lt: Tick) -> PointValues {
|
||||
let v = |m: &str, d: [f64; 2]| tp.get(m).map(|p| p.value_at(lt).as_vec2()).unwrap_or(d);
|
||||
PointValues {
|
||||
uid: tp.uid,
|
||||
spec: trk::PointSpec {
|
||||
center: v("featureCenter", [0.0; 2]),
|
||||
feature_size: v("featureSize", [32.0; 2]),
|
||||
search_offset: v("searchOffset", [0.0; 2]),
|
||||
search_size: v("searchSize", [64.0; 2]),
|
||||
},
|
||||
attach_offset: v("attachPointOffset", [0.0; 2]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach points of a frame from the feature centres, given the start frame's centres and attach
|
||||
/// points: two-point transforms rotate and scale the offsets with the track, Parallel Corner Pin
|
||||
/// maps them through the 3-point affine, Perspective Corner Pin through the 4-point homography.
|
||||
pub fn attach_points(kind: TrackKind, rot_scale: bool, ref_c: &[[f64; 2]], ref_a: &[[f64; 2]], cur: &[[f64; 2]]) -> Vec<[f64; 2]> {
|
||||
let translate = || cur.iter().zip(ref_c.iter().zip(ref_a)).map(|(c, (r, a))| [c[0] + a[0] - r[0], c[1] + a[1] - r[1]]).collect::<Vec<_>>();
|
||||
let through = |h: Option<trk::Homography>| match h {
|
||||
Some(h) => ref_a.iter().map(|a| h.apply(*a)).collect(),
|
||||
None => translate(),
|
||||
};
|
||||
match kind {
|
||||
TrackKind::Perspective if cur.len() >= 4 => {
|
||||
let h = trk::Homography::from_points([ref_c[0], ref_c[1], ref_c[2], ref_c[3]], [cur[0], cur[1], cur[2], cur[3]]);
|
||||
through(h)
|
||||
}
|
||||
TrackKind::Affine if cur.len() >= 3 => through(trk::Homography::affine_from_points([ref_c[0], ref_c[1], ref_c[2]], [cur[0], cur[1], cur[2]])),
|
||||
_ if rot_scale && cur.len() >= 2 => {
|
||||
let Some(s) = trk::Homography::similarity_from_points([ref_c[0], ref_c[1]], [cur[0], cur[1]]) else { return translate() };
|
||||
let m = s.0;
|
||||
cur.iter()
|
||||
.zip(ref_c.iter().zip(ref_a))
|
||||
.map(|(c, (r, a))| {
|
||||
let d = [a[0] - r[0], a[1] - r[1]];
|
||||
[c[0] + m[0][0] * d[0] + m[0][1] * d[1], c[1] + m[1][0] * d[0] + m[1][1] * d[1]]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => translate(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Live analysis progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrackProgress {
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
/// Analysed frames per second.
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
/// Stopped by "If Confidence is Below … ▸ Stop Tracking".
|
||||
pub low_confidence_stop: bool,
|
||||
pub error: Option<String>,
|
||||
/// Comp time (seconds) of the last analysed frame.
|
||||
pub time: f64,
|
||||
}
|
||||
|
||||
/// One analysed frame.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FrameResult {
|
||||
pub comp_time: Tick,
|
||||
pub layer_time: Tick,
|
||||
/// Per point: (point group uid, result, attach point).
|
||||
pub points: Vec<(Uid, trk::PointResult, [f64; 2])>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TrackShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<TrackProgress>,
|
||||
pub frames: Mutex<Vec<FrameResult>>,
|
||||
}
|
||||
|
||||
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
m.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) analysis.
|
||||
pub struct TrackJob {
|
||||
pub shared: Arc<TrackShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub tracker: Uid,
|
||||
pub direction: Direction,
|
||||
}
|
||||
|
||||
impl TrackJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> TrackProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
pub fn cancel(&self) {
|
||||
self.shared.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
pub(crate) struct Work {
|
||||
pub project: Arc<Project>,
|
||||
pub footage: Arc<dyn FootageSource>,
|
||||
pub expr: Option<Arc<dyn ExprHost>>,
|
||||
pub cache: Arc<LayerCache>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub settings: TrackerSettings,
|
||||
pub points: Vec<PointValues>,
|
||||
/// Comp times to analyse; the first is the start frame.
|
||||
pub times: Vec<Tick>,
|
||||
}
|
||||
|
||||
/// The layer's source frame at comp time `t` at 100 % (layer pixels + offset = image pixels).
|
||||
/// Footage at its native size is used straight from the footage source (no copy).
|
||||
fn source_frame(
|
||||
r: &Renderer,
|
||||
project: &Project,
|
||||
cid: ItemId,
|
||||
comp: &Comp,
|
||||
layer: &Layer,
|
||||
t: Tick,
|
||||
expr: Option<&dyn ExprHost>,
|
||||
) -> Option<(Arc<Image>, [f64; 2])> {
|
||||
let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr };
|
||||
if let effectcraft_project::LayerSource::Footage { item } = &layer.source
|
||||
&& let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind)
|
||||
&& f.has_video
|
||||
&& let Some(img) = r.footage.frame(*item, f, ctx.source_time(layer))
|
||||
&& img.width == f.width
|
||||
&& img.height == f.height
|
||||
{
|
||||
return Some((img, [0.0; 2]));
|
||||
}
|
||||
let buf = r.layer_source(&ctx, layer)?;
|
||||
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
// Footage decoded at another size: bring it to layer pixels.
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
|
||||
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
|
||||
}
|
||||
Some((Arc::new(buf.img), buf.offset))
|
||||
}
|
||||
|
||||
pub(crate) fn run_work(w: Work, shared: &TrackShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() });
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref());
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.total = w.times.len().saturating_sub(1) as u64;
|
||||
}
|
||||
let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") };
|
||||
let specs: Vec<trk::PointSpec> = w.points.iter().map(|p| p.spec).collect();
|
||||
let mut tracker = trk::Tracker::new(track_options(&w.settings.options), &specs, &trk::Frame { img: &first.0, offset: first.1 });
|
||||
let ref_c: Vec<[f64; 2]> = specs.iter().map(|s| s.center).collect();
|
||||
let ref_a: Vec<[f64; 2]> = w.points.iter().map(|p| [p.spec.center[0] + p.attach_offset[0], p.spec.center[1] + p.attach_offset[1]]).collect();
|
||||
let rot_scale = w.settings.rotation || w.settings.scale;
|
||||
// The start frame keeps its regions and gets keys too.
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[0],
|
||||
layer_time: layer.layer_time(w.times[0]),
|
||||
points: w
|
||||
.points
|
||||
.iter()
|
||||
.zip(&ref_a)
|
||||
.map(|(p, a)| (p.uid, trk::PointResult { center: p.spec.center, confidence: 100.0, rotation: 0.0, scale: 1.0, status: trk::Status::Tracked }, *a))
|
||||
.collect(),
|
||||
});
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
// Track this frame while the next one renders.
|
||||
let (res, nf) =
|
||||
rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None });
|
||||
next = nf;
|
||||
if res.iter().any(|p| p.status == trk::Status::Stopped) {
|
||||
lock(&shared.state).low_confidence_stop = true;
|
||||
break;
|
||||
}
|
||||
let centers: Vec<[f64; 2]> = res.iter().map(|p| p.center).collect();
|
||||
let attach = attach_points(w.settings.kind, rot_scale, &ref_c, &ref_a, ¢ers);
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[k],
|
||||
layer_time: layer.layer_time(w.times[k]),
|
||||
points: w.points.iter().zip(res).zip(attach).map(|((p, r), a)| (p.uid, r, a)).collect(),
|
||||
});
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Set (or add) a key at layer time `t`.
|
||||
pub(crate) fn key(g: &mut PropGroup, m: &str, t: Tick, v: Value) {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
effectcraft_keyframe::set_key(&mut p.keys, Keyframe::new(t, v));
|
||||
}
|
||||
}
|
||||
|
||||
/// Write analysed frames into the tracker's points.
|
||||
fn write_frames(p: &mut Project, job: &TrackJob, frames: &[FrameResult]) {
|
||||
let Some(comp) = p.comp_mut(job.comp) else { return };
|
||||
let Some(layer) = comp.layer_mut(job.layer) else { return };
|
||||
for f in frames {
|
||||
for (uid, r, a) in &f.points {
|
||||
let Some(tp) = layer.props.find_group_mut(*uid) else { continue };
|
||||
key(tp, "featureCenter", f.layer_time, Value::Vec2(r.center));
|
||||
key(tp, "confidence", f.layer_time, Value::Scalar(r.confidence));
|
||||
key(tp, "attachPoint", f.layer_time, Value::Vec2(*a));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Whether a track analysis is running.
|
||||
pub fn is_tracking(&self) -> bool {
|
||||
self.track_job.as_ref().is_some_and(|j| !j.is_finished())
|
||||
}
|
||||
|
||||
/// Live progress of the running (or just finished) analysis.
|
||||
pub fn track_progress(&self) -> Option<TrackProgress> {
|
||||
self.track_job.as_ref().map(|j| j.progress())
|
||||
}
|
||||
|
||||
/// Cancel the running analysis (results so far are kept).
|
||||
pub fn stop_track(&mut self) -> bool {
|
||||
match &self.track_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.cancel();
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start analysing. `wait`: block until done (wasm32 always does). One undo step covers the
|
||||
/// whole analysis.
|
||||
pub(crate) fn start_track(&mut self, work: Work, tracker: Uid, direction: Direction, wait: bool) -> Result<(), String> {
|
||||
if self.is_tracking() {
|
||||
return Err("a track analysis is already running".into());
|
||||
}
|
||||
self.poll_track();
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
self.history.undo.push(("Analyze Track".into(), self.project.clone()));
|
||||
self.history.redo.clear();
|
||||
self.history.merge_key = None;
|
||||
let shared = Arc::new(TrackShared::default());
|
||||
let (comp, layer) = (work.comp, work.layer);
|
||||
let thread = if wait {
|
||||
run_work(work, &shared);
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(std::thread::Builder::new().name("track-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?)
|
||||
};
|
||||
self.track_job = Some(TrackJob { shared, thread, comp, layer, tracker, direction });
|
||||
self.poll_track();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write new results into the project and move the CTI to the last analysed frame; drop the
|
||||
/// job once it finished. Frontends call this every frame while tracking.
|
||||
pub fn poll_track(&mut self) -> bool {
|
||||
let Some(job) = &self.track_job else { return false };
|
||||
let frames = std::mem::take(&mut *lock(&job.shared.frames));
|
||||
let finished = job.is_finished();
|
||||
let changed = !frames.is_empty();
|
||||
if changed {
|
||||
write_frames(Arc::make_mut(&mut self.project), job, &frames);
|
||||
if self.state.active_comp == Some(job.comp)
|
||||
&& let Some(last) = frames.last()
|
||||
{
|
||||
self.state.times.insert(job.comp, last.comp_time);
|
||||
}
|
||||
self.bump();
|
||||
}
|
||||
if finished {
|
||||
let mut job = self.track_job.take().expect("job");
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let msg = match (&st.error, st.cancelled, st.low_confidence_stop) {
|
||||
(Some(e), _, _) => format!("Tracking stopped: {e}"),
|
||||
(None, true, _) => format!("Tracking stopped after {} frame(s)", st.done),
|
||||
(None, false, true) => format!("Tracking stopped: confidence below threshold after {} frame(s)", st.done),
|
||||
_ => format!("Tracked {} frame(s) in {:.1} s", st.done, st.elapsed),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
}
|
||||
changed || finished
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- apply
|
||||
|
||||
/// Apply Dimensions.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Dims {
|
||||
#[default]
|
||||
XY,
|
||||
X,
|
||||
Y,
|
||||
}
|
||||
|
||||
impl Dims {
|
||||
pub fn from_name(s: &str) -> Option<Dims> {
|
||||
match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() {
|
||||
"xy" | "xandy" | "both" => Some(Dims::XY),
|
||||
"x" | "xonly" => Some(Dims::X),
|
||||
"y" | "yonly" => Some(Dims::Y),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn mix(self, old: [f64; 3], new: [f64; 2]) -> [f64; 3] {
|
||||
match self {
|
||||
Dims::XY => [new[0], new[1], old[2]],
|
||||
Dims::X => [new[0], old[1], old[2]],
|
||||
Dims::Y => [old[0], new[1], old[2]],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_m(m: &Mat3, p: [f64; 2]) -> [f64; 2] {
|
||||
let q = m.apply(vec2(p[0], p[1]));
|
||||
[q.x, q.y]
|
||||
}
|
||||
|
||||
/// Parent space of a layer (comp space when unparented), as a 2D matrix from comp space.
|
||||
fn comp_to_parent(ctx: &EvalCtx, layer: &Layer) -> Mat3 {
|
||||
match layer.parent.and_then(|p| ctx.comp.layer(p)) {
|
||||
Some(p) => ctx.layer_to_comp(p).0.inverse().unwrap_or(Mat3::IDENTITY),
|
||||
None => Mat3::IDENTITY,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per analysed time: (source layer time, attach points in source layer space).
|
||||
pub fn attach_track(tracker: &PropGroup) -> Vec<(Tick, Vec<[f64; 2]>)> {
|
||||
let pts: Vec<&PropGroup> = tracker.track_points().collect();
|
||||
let Some(first) = pts.first().and_then(|g| g.get("attachPoint")) else { return vec![] };
|
||||
first.keys.iter().map(|k| (k.time, pts.iter().map(|g| g.get("attachPoint").map(|p| p.value_at(k.time).as_vec2()).unwrap_or([0.0; 2])).collect())).collect()
|
||||
}
|
||||
|
||||
/// Apply a tracker (Tracker panel ▸ Apply). Returns the number of keyframed frames.
|
||||
pub fn apply(p: &mut Project, cid: ItemId, src: LayerId, tracker: Uid, dims: Dims) -> Result<usize, String> {
|
||||
let comp = p.comp(cid).ok_or("no composition")?.clone();
|
||||
let source = comp.layer(src).ok_or("no source layer")?.clone();
|
||||
let (tg, settings) = source.tracker(tracker).ok_or("no such tracker")?;
|
||||
let settings = settings.clone();
|
||||
let track = attach_track(tg);
|
||||
if track.is_empty() {
|
||||
return Err("analyze the track first".into());
|
||||
}
|
||||
let target_id = match settings.kind {
|
||||
TrackKind::Stabilize => src,
|
||||
TrackKind::Raw => return Err("a Raw track has nothing to apply".into()),
|
||||
_ => settings.target.ok_or("set a Motion Target first (Edit Target…)")?,
|
||||
};
|
||||
let target = comp.layer(target_id).ok_or("the motion target layer is gone")?.clone();
|
||||
let ctx0 = EvalCtx::new(p, cid, &comp, Tick::ZERO);
|
||||
let t_first = source.comp_time(track[0].0);
|
||||
let rot_scale = settings.point_count() >= 2 && track[0].1.len() >= 2;
|
||||
let (ang0, len0) = if rot_scale { effectcraft_track::angle_and_length(track[0].1[0], track[0].1[1]) } else { (0.0, 1.0) };
|
||||
// (target layer time, values) to write.
|
||||
let mut pos_keys = vec![];
|
||||
let mut anchor_keys = vec![];
|
||||
let mut rot_keys = vec![];
|
||||
let mut scale_keys = vec![];
|
||||
let mut corner_keys = vec![];
|
||||
let ttr = target.transform().ok_or("the target has no transform")?;
|
||||
let c0 = ctx0.at(t_first);
|
||||
let rot0 = c0.f(&target, ttr, "rotation", 0.0);
|
||||
let scale0 = c0.v3(&target, ttr, "scale", [100.0; 3]);
|
||||
let mut prev_ang = 0.0;
|
||||
// Stabilize: the feature stays where it was on the first tracked frame.
|
||||
let stab_pos = {
|
||||
let (m, _) = c0.layer_to_comp(&source);
|
||||
apply_m(&comp_to_parent(&c0, &source), apply_m(&m, track[0].1[0]))
|
||||
};
|
||||
for (lt, pts) in &track {
|
||||
let ct = source.comp_time(*lt);
|
||||
let ctx = ctx0.at(ct);
|
||||
let tt = target.layer_time(ct);
|
||||
let (ang, len) = if rot_scale { effectcraft_track::angle_and_length(pts[0], pts[1]) } else { (ang0, len0) };
|
||||
let dang = effectcraft_track::unwrap_degrees(prev_ang, ang - ang0);
|
||||
prev_ang = dang;
|
||||
let ratio = if len0 > 1e-9 { len / len0 } else { 1.0 };
|
||||
match settings.kind {
|
||||
TrackKind::Stabilize => {
|
||||
if settings.position {
|
||||
let old = ctx.v3(&target, ttr, "anchor", [0.0; 3]);
|
||||
anchor_keys.push((tt, dims.mix(old, pts[0])));
|
||||
let oldp = ctx.position(&target, ttr);
|
||||
pos_keys.push((tt, dims.mix(oldp, stab_pos)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 - dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] / ratio, scale0[1] / ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Transform => {
|
||||
let (m, _) = ctx.layer_to_comp(&source);
|
||||
if settings.position {
|
||||
let c = apply_m(&m, pts[0]);
|
||||
let pp = apply_m(&comp_to_parent(&ctx, &target), c);
|
||||
pos_keys.push((tt, dims.mix(ctx.position(&target, ttr), pp)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 + dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] * ratio, scale0[1] * ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Affine | TrackKind::Perspective => {
|
||||
let (ms, _) = ctx.layer_to_comp(&source);
|
||||
let (mt, _) = ctx.layer_to_comp(&target);
|
||||
let inv = mt.inverse().ok_or("the target layer's transform is not invertible")?;
|
||||
let n = pts.len();
|
||||
if n < 3 {
|
||||
return Err("corner pin tracks need 3 or 4 track points".into());
|
||||
}
|
||||
let lr =
|
||||
if settings.kind == TrackKind::Affine || n < 4 { [pts[1][0] + pts[2][0] - pts[0][0], pts[1][1] + pts[2][1] - pts[0][1]] } else { pts[3] };
|
||||
// Track Points 1–4: upper left, upper right, lower left, lower right.
|
||||
let map = |q: [f64; 2]| apply_m(&inv, apply_m(&ms, q));
|
||||
corner_keys.push((tt, [map(pts[0]), map(pts[1]), map(pts[2]), map(lr)]));
|
||||
}
|
||||
TrackKind::Raw => {}
|
||||
}
|
||||
}
|
||||
let n = track.len();
|
||||
let tsize = effectcraft_render::source_size(p, &target);
|
||||
let mut ids_next = p.next_id;
|
||||
let layer = p.comp_mut(cid).and_then(|c| c.layer_mut(target_id)).ok_or("no target layer")?;
|
||||
if !corner_keys.is_empty() {
|
||||
let spec = effectcraft_effects::find("ec.distort.cornerpin").ok_or("Corner Pin is not available")?;
|
||||
let size = tsize;
|
||||
let size = if size.0 == 0 { [comp.width as f64, comp.height as f64] } else { [size.0 as f64, size.1 as f64] };
|
||||
let fx = layer.props.sub_mut("effects").ok_or("the target layer can't have effects")?;
|
||||
let same = fx.groups().filter(|g| g.match_id == spec.id).count();
|
||||
let name = if same == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, same + 1) };
|
||||
let mut g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut ids_next), &name, size);
|
||||
for (t, c) in &corner_keys {
|
||||
for (m, v) in ["ul", "ur", "ll", "lr"].iter().zip(c) {
|
||||
key(&mut g, m, *t, Value::Vec2(*v));
|
||||
}
|
||||
}
|
||||
fx.children.push(g.into());
|
||||
}
|
||||
let tr = layer.transform_mut().ok_or("no transform")?;
|
||||
for (t, v) in anchor_keys {
|
||||
key(tr, "anchor", t, Value::Vec3(v));
|
||||
}
|
||||
let separated = tr.get("positionX").is_some();
|
||||
for (t, v) in pos_keys {
|
||||
if separated {
|
||||
key(tr, "positionX", t, Value::Scalar(v[0]));
|
||||
key(tr, "positionY", t, Value::Scalar(v[1]));
|
||||
} else {
|
||||
key(tr, "position", t, Value::Vec3(v));
|
||||
}
|
||||
}
|
||||
for (t, v) in rot_keys {
|
||||
key(tr, "rotation", t, Value::Scalar(v));
|
||||
}
|
||||
for (t, v) in scale_keys {
|
||||
key(tr, "scale", t, Value::Vec3(v));
|
||||
}
|
||||
p.next_id = ids_next;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attach_points_follow_the_track() {
|
||||
let rc = [[0.0, 0.0], [10.0, 0.0]];
|
||||
let ra = [[0.0, 5.0], [10.0, 5.0]];
|
||||
// Rotated 90° and doubled.
|
||||
let cur = [[100.0, 100.0], [100.0, 120.0]];
|
||||
let a = attach_points(TrackKind::Transform, true, &rc, &ra, &cur);
|
||||
assert!((a[0][0] - 90.0).abs() < 1e-9 && (a[0][1] - 100.0).abs() < 1e-9, "{a:?}");
|
||||
let a = attach_points(TrackKind::Transform, false, &rc, &ra, &cur);
|
||||
assert_eq!(a[0], [100.0, 105.0]);
|
||||
let h = trk::Homography([[1.1, 0.1, 5.0], [0.05, 0.9, -3.0], [0.001, 0.0005, 1.0]]);
|
||||
let rc4 = [[0.0, 0.0], [100.0, 0.0], [0.0, 100.0], [100.0, 100.0]];
|
||||
let ra4 = [[10.0, 10.0], [90.0, 10.0], [10.0, 90.0], [90.0, 90.0]];
|
||||
let cur4: Vec<[f64; 2]> = rc4.iter().map(|p| h.apply(*p)).collect();
|
||||
let a = attach_points(TrackKind::Perspective, false, &rc4, &ra4, &cur4);
|
||||
for (got, want) in a.iter().zip(ra4.iter().map(|p| h.apply(*p))) {
|
||||
assert!((got[0] - want[0]).abs() < 1e-6 && (got[1] - want[1]).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user