Merge branch 'worktree-agent-a5b25d366383f4675'

# Conflicts:
#	docs/parity.md
This commit is contained in:
Brandon Thomas
2026-10-03 05:09:01 -04:00
26 changed files with 2911 additions and 111 deletions
+16 -2
View File
@@ -23,7 +23,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Track Camera",
["Animation"],
None,
"{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}",
"{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}",
can_track,
track_camera
),
@@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
return Err(bad("track.camera", "aov: 1..170 degrees"));
}
let detailed = b_p(p, "detailed");
let lens = b_p(p, "lensDistortion");
let undistort = b_p(p, "undistort");
let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid));
let uid = match existing {
Some(u) => u,
@@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))?
}
};
if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() {
if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() {
s.edit("3D Camera Tracker Settings", None, |proj, _| {
let g = proj
.comp_mut(cid)
@@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
if let Some(d) = detailed {
set("advanced/detailedAnalysis", KV::Bool(d));
}
if let Some(d) = lens {
set("advanced/lensDistortion", KV::Bool(d));
}
if let Some(d) = undistort {
set("advanced/undistort", KV::Bool(d));
}
Ok(())
})?;
}
@@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
out["shotType"] = json!(settings.shot.label());
out["solveMethod"] = json!(settings.method.label());
out["deleted"] = json!(settings.deleted.len());
out["lensDistortionSolved"] = json!(settings.lens_distortion);
let solve = ct::solve(&params);
out["solved"] = json!(solve.is_some());
if let Some(sv) = &solve {
@@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
out["focalLength"] = json!(f);
out["horizontalAngleOfView"] = json!(sv.hfov(f));
out["groundPlane"] = json!(sv.ground.is_some());
out["lensDistortion"] = match &sv.distortion {
Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}),
None => Value::Null,
};
out["undistort"] = json!(ct::undistorts(&params, sv));
if let Some(pl) = placed(&s.project, cid, lid, uid) {
let k = pl.frame_at(s.time_of(cid));
let (pos, ori, zoom) = pl.camera(k);
+126 -7
View File
@@ -1,5 +1,6 @@
//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask
//! Interpolation (Window ▸ Mask Interpolation).
//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face
//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation
//! (Window ▸ Mask Interpolation).
use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV};
use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching};
@@ -20,7 +21,7 @@ pub fn specs() -> Vec<CommandSpec> {
"Track Mask",
["Animation"],
None,
"{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
"{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
has_mask,
track_mask
),
@@ -29,10 +30,19 @@ pub fn specs() -> Vec<CommandSpec> {
"Mask Tracking Method",
[],
None,
"{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}",
"{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}",
super::always,
mask_method
),
cmd!(
"track.extractFaceMeasurements",
"Extract & Copy Face Measurements",
[],
None,
"{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys",
has_face_points,
extract_face
),
cmd!(
"mask.interpolate",
"Apply Mask Interpolation",
@@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> {
fn method_p(p: &Value, cmd: &str) -> Result<Option<MaskMethod>> {
match str_p(p, "method") {
Some(m) => MaskMethod::from_name(m)
.map(Some)
.ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))),
Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| {
bad(
cmd,
format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"),
)
}),
None => Ok(None),
}
}
@@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result<Value> {
}))
}
// ---------------------------------------------------------------- face measurements
fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> {
use effectcraft_effects::face_track::POINTS_ID;
let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID));
if p.get("layer").is_some() {
return layer_p(s, p, "track.extractFaceMeasurements").ok();
}
let cid = s.active_comp_id()?;
let comp = s.project.comp(cid)?;
let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l));
sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id))
}
fn has_face_points(s: &Session) -> std::result::Result<(), String> {
has_comp(s)?;
face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into())
}
/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them
/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard.
fn extract_face(s: &mut Session, p: &Value) -> Result<Value> {
use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID};
use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure};
const CMD: &str = "track.extractFaceMeasurements";
let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?;
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?;
let mut times: Vec<Tick> = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect();
times.sort();
times.dedup();
if times.is_empty() {
times.push(layer.layer_time(s.time_of(cid)));
}
let rows: Vec<(Tick, [f64; 14])> = {
let pts_at = |t: Tick| {
let mut l = [[0.0; 2]; N];
for (i, (id, _, _)) in LANDMARKS.iter().enumerate() {
if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) {
l[i] = v;
}
}
l
};
let reference = face_frame(&pts_at(times[0])).1;
times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect()
};
let comp_times: Vec<Tick> = times.iter().map(|t| layer.comp_time(*t)).collect();
let fd = comp.frame_duration();
let uid = s.edit("Extract Face Measurements", None, |proj, _| {
let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?;
let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?;
for (k, (id, _)) in MEASUREMENTS.iter().enumerate() {
let Some(pr) = g.get_mut(id) else { continue };
pr.keys.clear();
for (t, m) in &rows {
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k])));
}
}
Ok(uid)
})?;
// Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes).
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?;
let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO);
let mut clip = vec![];
for (id, _) in MEASUREMENTS {
let Some(pr) = mg.get(id) else { continue };
let Some(path) = layer.props.match_path_of(pr.uid) else { continue };
let keys = pr
.keys
.iter()
.map(|k| {
let mut k = k.clone();
k.time = layer.comp_time(k.time) - earliest;
k
})
.collect();
clip.push(crate::KeyClip { layer: lid, path, keys });
}
s.state.key_clipboard = clip;
s.state.clip_is_keys = true;
// The same data as text (tab-separated, one row per frame) for other applications.
let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime");
for (_, name) in MEASUREMENTS {
text.push('\t');
text.push_str(name);
}
text.push('\n');
let mut table = vec![];
for ((_, m), ct) in rows.iter().zip(&comp_times) {
let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 };
text.push_str(&format!("{frame}\t{:.4}", ct.seconds()));
for v in m {
text.push_str(&format!("\t{v:.3}"));
}
text.push('\n');
let obj: serde_json::Map<String, Value> = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect();
table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj}));
}
s.bump();
Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text}))
}
// ---------------------------------------------------------------- mask interpolation
/// Mask Interpolation panel options (kept in the editor state, serde for agents).
+2
View File
@@ -774,6 +774,8 @@ mod tests_effects;
#[cfg(test)]
mod tests_essential;
#[cfg(test)]
mod tests_face;
#[cfg(test)]
mod tests_fidelity;
#[cfg(test)]
mod tests_lottie;
+148 -11
View File
@@ -6,6 +6,12 @@
//! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the
//! chosen motion model), and every frame's motion is applied to the mask's vertices and
//! tangents, so the shape's Bezier structure is kept.
//!
//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn
//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path
//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track
//! Points** effect on the layer (created on first use), from which Extract & Copy Face
//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
@@ -31,11 +37,22 @@ pub enum MaskMethod {
PositionScaleRotation,
PositionScaleRotationSkew,
Perspective,
/// Face Tracking (Outline Only).
FaceOutline,
/// Face Tracking (Detailed Features).
FaceDetailed,
}
impl MaskMethod {
pub const ALL: [MaskMethod; 5] =
[MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective];
pub const ALL: [MaskMethod; 7] = [
MaskMethod::Position,
MaskMethod::PositionScale,
MaskMethod::PositionScaleRotation,
MaskMethod::PositionScaleRotationSkew,
MaskMethod::Perspective,
MaskMethod::FaceOutline,
MaskMethod::FaceDetailed,
];
pub fn label(self) -> &'static str {
match self {
MaskMethod::Position => "Position",
@@ -43,8 +60,14 @@ impl MaskMethod {
MaskMethod::PositionScaleRotation => "Position, Scale & Rotation",
MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew",
MaskMethod::Perspective => "Perspective",
MaskMethod::FaceOutline => "Face Tracking (Outline Only)",
MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)",
}
}
/// One of the Face Tracking methods.
pub fn is_face(self) -> bool {
matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed)
}
pub fn id(self) -> &'static str {
match self {
MaskMethod::Position => "position",
@@ -52,6 +75,8 @@ impl MaskMethod {
MaskMethod::PositionScaleRotation => "positionScaleRotation",
MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew",
MaskMethod::Perspective => "perspective",
MaskMethod::FaceOutline => "faceOutline",
MaskMethod::FaceDetailed => "faceDetailed",
}
}
pub fn from_name(s: &str) -> Option<MaskMethod> {
@@ -67,6 +92,7 @@ impl MaskMethod {
MaskMethod::PositionScaleRotation => Model::Similarity,
MaskMethod::PositionScaleRotationSkew => Model::Affine,
MaskMethod::Perspective => Model::Homography,
MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity,
}
}
}
@@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath {
}
}
/// One tracked frame: (layer time, comp time, mask path).
type MaskFrame = (Tick, Tick, ShapePath);
/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom
/// tangents. `like` lends its feather points.
pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath {
let m = poly.len();
let n = n.clamp(3, m.max(3));
let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect();
let tan: Vec<[f64; 2]> = (0..n)
.map(|i| {
let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]);
[(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0]
})
.collect();
ShapePath {
vertices: v,
in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(),
out_tangents: tan,
closed: true,
feather: if like.len() == n { like.feather.clone() } else { vec![] },
}
}
/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features).
type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>);
#[derive(Default)]
pub struct MaskTrackShared {
@@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref());
lock(&shared.state).total = w.times.len().saturating_sub(1) as u64;
let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") };
if w.method.is_face() {
return run_face(&w, shared, first, &frame_at, layer, t0);
}
let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 });
let mut path = w.path.clone();
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone()));
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None));
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
for k in 1..w.times.len() {
if shared.cancel.load(Ordering::Relaxed) {
@@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
break;
};
path = transform_path(&path, &step.motion);
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone()));
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None));
let mut s = lock(&shared.state);
s.done = k as u64;
s.elapsed = t0.elapsed().as_secs_f64();
@@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
s.finished = true;
}
/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks.
fn run_face(
w: &MaskWork,
shared: &MaskTrackShared,
first: (Arc<effectcraft_raster::Image>, [f64; 2]),
frame_at: &(dyn Fn(Tick) -> Option<(Arc<effectcraft_raster::Image>, [f64; 2])> + Sync),
layer: &effectcraft_project::Layer,
t0: web_time::Instant,
) {
let detailed = w.method == MaskMethod::FaceDetailed;
let b = trk::mask::bounds(&flatten(&w.path, 8));
let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else {
let mut s = lock(&shared.state);
s.error = Some("no face found inside the mask (draw the mask around a face)".into());
s.finished = true;
return;
};
let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks));
let (path, pts) = key(&fit);
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts));
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
for k in 1..w.times.len() {
if shared.cancel.load(Ordering::Relaxed) {
lock(&shared.state).cancelled = true;
break;
}
let Some(cur) = next.take() else {
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
break;
};
let (res, nf) =
rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None });
next = nf;
let Some(fit) = res else {
lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds()));
break;
};
let (path, pts) = key(&fit);
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts));
let mut s = lock(&shared.state);
s.done = k as u64;
s.elapsed = t0.elapsed().as_secs_f64();
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
s.time = w.times[k].seconds();
crate::offload::report(s.done, s.total);
}
let mut s = lock(&shared.state);
s.elapsed = t0.elapsed().as_secs_f64();
s.finished = true;
}
/// The layer's Face Track Points effect (created when missing). Returns its uid.
pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option<Uid> {
face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID)
}
/// The layer's effect `id`, applied when missing (data effects of face tracking).
pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option<Uid> {
let spec = effectcraft_effects::find(id)?;
let l = p.comp(comp)?.layer(layer)?;
if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) {
return Some(g.uid);
}
let (w, h) = effectcraft_render::source_size(p, l);
let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] };
let mut next = p.next_id;
let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size);
let uid = g.uid;
let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?;
fx.children.push(g.into());
p.next_id = next;
Some(uid)
}
fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) {
let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None };
let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return };
let Some(g) = layer.props.find_group_mut(job.mask) else { return };
let Some(pr) = g.get_mut("path") else { return };
for (lt, _, path) in frames {
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone())));
if let Some(g) = layer.props.find_group_mut(job.mask)
&& let Some(pr) = g.get_mut("path")
{
for (lt, _, path, _) in frames {
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone())));
}
}
if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) {
for (lt, _, _, pts) in frames {
let Some(pts) = pts else { continue };
for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() {
if let Some(pr) = g.get_mut(id) {
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i])));
}
}
}
}
}
@@ -242,7 +379,7 @@ impl Session {
return Err("a track analysis is already running".into());
}
self.poll_mask_track();
if !wait && self.offloads() {
if !wait && self.offloads() && !work.method.is_face() {
let MaskWork { comp, layer, path, method, times, .. } = work;
return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times });
}
+106
View File
@@ -0,0 +1,106 @@
//! Face tracking end to end through the commands, on synthetic face footage generated in-test
//! (a moving, rotating face that opens its mouth and blinks).
use effectcraft_keyframe::{ShapePath, Value as KV};
use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks};
use serde_json::json;
use crate::mask_track::MaskMethod;
use crate::tests_track::{H, W, frame_time, setup};
fn face(f: u32) -> FaceParams {
let t = f as f64;
FaceParams {
center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()],
height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()),
roll: 12.0 * (t * 0.18).sin(),
yaw: 10.0 * (t * 0.12).sin(),
mouth_open: 0.5 + 0.45 * (t * 0.3).sin(),
eye_open: [0.85, 0.85],
..Default::default()
}
}
#[test]
fn face_tracking_keys_outline_points_and_measurements() {
let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5));
assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed));
assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline));
// A rough mask around the face.
let c = face(0).center;
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
let mask = r["mask"].as_u64().unwrap();
s.execute("time.set", json!({"time": 0})).unwrap();
let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap();
assert_eq!(r["frames"], 24, "{r}");
assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed);
let comp = s.active_comp().unwrap();
let layer = comp.layer(clip).unwrap();
// The mask follows the face outline.
let path = layer.props.find_group(mask).unwrap().get("path").unwrap();
assert_eq!(path.keys.len(), 25);
for f in [0, 12, 24] {
let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() };
let fp = face(f);
let (sn, cs) = fp.roll.to_radians().sin_cos();
for v in &p.vertices {
let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height];
let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]);
let r = ((u / 0.75).powi(2) + w * w).sqrt();
assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}");
}
}
// Face Track Points: one keyed point per landmark, on the face.
let fx = layer.effects().unwrap();
let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points");
let mut worst: f64 = 0.0;
for f in [0, 6, 12, 18, 24] {
let truth = synth_landmarks(&face(f));
for (i, (id, _, _)) in LANDMARKS.iter().enumerate() {
let pr = pts.get(id).unwrap();
assert_eq!(pr.keys.len(), 25, "{id}");
let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() };
worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height);
}
}
assert!(worst < 0.1, "worst landmark error {worst} face heights");
// Extract & Copy Face Measurements.
assert!(s.is_enabled("track.extractFaceMeasurements"));
let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap();
assert_eq!(r["frames"], 25);
assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness"));
assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len());
assert!(s.state.clip_is_keys);
let layer = s.active_comp().unwrap().layer(clip).unwrap();
let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements");
let mo = m.get("mouthOpenness").unwrap();
assert_eq!(mo.keys.len(), 25);
// Mouth openness follows the footage: widest open where the generator opens it most.
let open = |f: u32| mo.value_at(frame_time(f)).as_f64();
let (fmax, fmin) = (
(0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
(0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
);
assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin));
let roll = m.get("headOrientationZ").unwrap();
for f in [0, 10, 20] {
assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0);
}
// One undo step for the extraction.
assert!(s.undo());
assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID));
// Outline Only keys just the mask.
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
let m2 = r["mask"].as_u64().unwrap();
s.execute("time.set", json!({"time": 0})).unwrap();
s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap();
let layer = s.active_comp().unwrap().layer(clip).unwrap();
assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2);
// A mask without a face reports it.
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap();
let m3 = r["mask"].as_u64().unwrap();
s.execute("time.set", json!({"time": 0})).unwrap();
let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true}));
let layer = s.active_comp().unwrap().layer(clip).unwrap();
assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty());
}