mirror of
https://github.com/storytold/effectcraft.git
synced 2026-10-30 21:16:52 -06:00
Merge main and retain Cineon Settings in the extracted modules
This commit is contained in:
dotCooCoo
@@ -0,0 +1,668 @@
|
||||
//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping
|
||||
//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`).
|
||||
//!
|
||||
//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame
|
||||
//! between the layer's In and Out points on a background thread and tracks features through
|
||||
//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving
|
||||
//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method,
|
||||
//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into
|
||||
//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear
|
||||
//! them and queue the effect again (`invalidate`, run after every edit), like After Effects.
|
||||
//!
|
||||
//! Composition space: the solve's canonical frame maps to comp world by
|
||||
//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's
|
||||
//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created
|
||||
//! layers sit on the selected points.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY};
|
||||
use effectcraft_keyframe::{Keyframe, Value};
|
||||
use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid};
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track::camtrack::linalg::{self, Similarity, V3};
|
||||
use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::offload::JobKind;
|
||||
use crate::tracking::lock;
|
||||
use crate::warp::{analysis_times, signature};
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Live analysis progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CameraProgress {
|
||||
/// 1 = tracking features, 2 = solving the camera.
|
||||
pub step: u8,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: Option<String>,
|
||||
/// Re-solving stored tracks (no tracking step).
|
||||
pub solve_only: bool,
|
||||
}
|
||||
|
||||
impl CameraProgress {
|
||||
/// The viewer banner text, as After Effects words it.
|
||||
pub fn banner(&self) -> String {
|
||||
match self.step {
|
||||
2 => "Solving camera".into(),
|
||||
_ => {
|
||||
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
|
||||
format!("Analyzing in background (step 1 of 2): {pc}%")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CamResult {
|
||||
/// New tracks (None for a re-solve).
|
||||
tracks: Option<CameraTracks>,
|
||||
solve: Result<CameraSolve, String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CameraShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<CameraProgress>,
|
||||
result: Mutex<Option<CamResult>>,
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) camera analysis.
|
||||
pub struct CameraJob {
|
||||
pub shared: Arc<CameraShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub effect: Uid,
|
||||
/// The layer signature the tracks are made from, and the solve settings key.
|
||||
key: String,
|
||||
solve_key: String,
|
||||
}
|
||||
|
||||
impl CameraJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> CameraProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
}
|
||||
|
||||
struct CamWork {
|
||||
project: Arc<Project>,
|
||||
footage: Arc<dyn FootageSource>,
|
||||
expr: Option<Arc<dyn ExprHost>>,
|
||||
cache: Arc<LayerCache>,
|
||||
comp: ItemId,
|
||||
layer: LayerId,
|
||||
effect_index: usize,
|
||||
size: [f64; 2],
|
||||
detailed: bool,
|
||||
times: Vec<Tick>,
|
||||
/// Stored tracks to re-solve (skips step 1).
|
||||
tracks: Option<Arc<CameraTracks>>,
|
||||
settings: SolveSettings,
|
||||
}
|
||||
|
||||
pub(crate) fn is_tracker(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID)
|
||||
}
|
||||
|
||||
fn str_of(g: &PropGroup, m: &str) -> String {
|
||||
match g.prop(m).map(|p| &p.value) {
|
||||
Some(Value::Str(s)) => s.clone(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis.
|
||||
fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String {
|
||||
let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool);
|
||||
format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" })
|
||||
}
|
||||
|
||||
/// Parameters of an instance from static values (the tracker's settings are not animated).
|
||||
pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params {
|
||||
effectcraft_effects::flatten_params(g, &mut |p| p.value.clone())
|
||||
}
|
||||
|
||||
fn clear(g: &mut PropGroup, all: bool) {
|
||||
let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED];
|
||||
if all {
|
||||
ids.extend([TRACKS, TRACKS_KEY]);
|
||||
}
|
||||
for m in ids {
|
||||
if let Some(p) = g.prop_mut(m) {
|
||||
p.value = Value::Str(String::new());
|
||||
p.keys.clear();
|
||||
}
|
||||
}
|
||||
if let Some(p) = g.prop_mut(AVERAGE_ERROR) {
|
||||
p.value = Value::Scalar(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their
|
||||
/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis:
|
||||
/// cleared ones and newly applied, unanalysed ones.
|
||||
pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
|
||||
let mut todo = vec![];
|
||||
let mut clears = vec![];
|
||||
for (cid, it) in &after.items {
|
||||
let ItemKind::Comp(c) = &it.kind else { continue };
|
||||
for l in &c.layers {
|
||||
let Some(fx) = l.effects() else { continue };
|
||||
if !fx.groups().any(is_tracker) {
|
||||
continue;
|
||||
}
|
||||
let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id));
|
||||
for g in fx.groups().filter(|g| is_tracker(g)) {
|
||||
let tracked = !str_of(g, TRACKS).is_empty();
|
||||
if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) {
|
||||
clears.push((*cid, l.id, g.uid, true));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) {
|
||||
clears.push((*cid, l.id, g.uid, false));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() {
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (cid, lid, uid, all) in clears {
|
||||
if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) {
|
||||
clear(g, all);
|
||||
}
|
||||
}
|
||||
todo
|
||||
}
|
||||
|
||||
fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc<effectcraft_raster::Image>, [f64; 2])> {
|
||||
let buf = r.layer_input(ctx, layer, effect_index)?;
|
||||
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
|
||||
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
|
||||
}
|
||||
Some((Arc::new(buf.img.clone()), buf.offset))
|
||||
}
|
||||
|
||||
async fn run_work(w: CamWork, shared: &CameraShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
// A browser job worker's GPU (deferred readbacks): frames render in passes.
|
||||
let accel = crate::offload::job_accel();
|
||||
let accel = accel.as_deref();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let cancelled = || {
|
||||
let mut s = lock(&shared.state);
|
||||
s.cancelled = true;
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let lt0 = layer.layer_time(w.times[0]).seconds();
|
||||
let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() };
|
||||
let (tracks, fresh) = match w.tracks.clone() {
|
||||
Some(t) => (t, false),
|
||||
None => {
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel));
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO);
|
||||
let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 };
|
||||
let ctx0 = &ctx0;
|
||||
let frame_at = |t: Tick| crate::offload::in_passes(accel, move |a| input_frame(&r.with_accel(a), &ctx0.at(t), layer, w.effect_index));
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.step = 1;
|
||||
s.total = w.times.len() as u64;
|
||||
}
|
||||
let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed });
|
||||
let mut next = frame_at(w.times[0]).await;
|
||||
for k in 0..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
return cancelled();
|
||||
}
|
||||
let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) };
|
||||
let ((), nf) = crate::offload::join_fetch(
|
||||
accel.is_some(),
|
||||
|| an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }),
|
||||
|| async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } },
|
||||
)
|
||||
.await;
|
||||
next = nf;
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64 + 1;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
(Arc::new(an.finish(lt0, fd)), true)
|
||||
}
|
||||
};
|
||||
lock(&shared.state).step = 2;
|
||||
let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0);
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
return cancelled();
|
||||
}
|
||||
*lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve });
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn is_camera_analyzing(&self) -> bool {
|
||||
self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded(JobKind::Camera).is_some()
|
||||
}
|
||||
|
||||
/// Progress of the running (or just finished) analysis.
|
||||
pub fn camera_progress(&self) -> Option<CameraProgress> {
|
||||
self.camera_job.as_ref().map(|j| j.progress()).or_else(|| {
|
||||
self.offloaded(JobKind::Camera).map(|j| {
|
||||
let p = &j.progress;
|
||||
CameraProgress { step: 1, done: p.done, total: p.total, elapsed: p.elapsed, fps: crate::offload::rate(p.done, p.elapsed), ..Default::default() }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// The tracker being analysed: (comp, layer, effect uid).
|
||||
pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> {
|
||||
self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)).or_else(|| self.offloaded(JobKind::Camera).map(|j| (j.comp, j.layer, j.uid)))
|
||||
}
|
||||
|
||||
/// Cancel the running analysis (nothing is written).
|
||||
pub fn stop_camera(&mut self) -> bool {
|
||||
if self.cancel_offloaded(JobKind::Camera) {
|
||||
return true;
|
||||
}
|
||||
match &self.camera_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.shared.cancel.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when
|
||||
/// they are still valid). `wait`: block until done (wasm32 always does). Returns the number
|
||||
/// of frames.
|
||||
pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result<usize, String> {
|
||||
if self.is_camera_analyzing() {
|
||||
return Err("a 3D Camera Tracker analysis is already running".into());
|
||||
}
|
||||
self.poll_camera_job();
|
||||
self.camera_pending.retain(|x| *x != (comp, layer, effect));
|
||||
let c = self.project.comp(comp).ok_or("no composition")?;
|
||||
let l = c.layer(layer).ok_or("no layer")?;
|
||||
let fx = l.effects().ok_or("the layer has no effects")?;
|
||||
let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
|
||||
let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?;
|
||||
if l.switches.adjustment {
|
||||
return Err("3D Camera Tracker needs footage, not an adjustment layer".into());
|
||||
}
|
||||
let times = analysis_times(c, l);
|
||||
if times.len() < 2 {
|
||||
return Err("the layer must be at least two frames long".into());
|
||||
}
|
||||
let size = effectcraft_render::effect_bounds(&self.project, c, l).0;
|
||||
let params = static_params(g);
|
||||
let key = tracks_key(&self.project, comp, l, g);
|
||||
let settings = ct::settings(¶ms);
|
||||
let solve_key = ct::solve_key(&settings);
|
||||
let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key);
|
||||
let n = times.len();
|
||||
let work = CamWork {
|
||||
project: self.project.clone(),
|
||||
footage: self.footage.clone(),
|
||||
expr: self.expr.clone(),
|
||||
cache: self.layer_cache.clone(),
|
||||
comp,
|
||||
layer,
|
||||
effect_index: index,
|
||||
size,
|
||||
detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()),
|
||||
times,
|
||||
tracks: tracks.clone(),
|
||||
settings,
|
||||
};
|
||||
if !wait && self.offloads() {
|
||||
drop(work);
|
||||
self.offload_analysis(crate::offload::WorkerJob::Camera { comp, layer, effect })?;
|
||||
return Ok(n);
|
||||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let shared = Arc::new(CameraShared::default());
|
||||
{
|
||||
let mut st = lock(&shared.state);
|
||||
st.solve_only = tracks.is_some();
|
||||
st.step = if tracks.is_some() { 2 } else { 1 };
|
||||
st.total = n as u64;
|
||||
}
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
async move { run_work(work, &sh).await }
|
||||
});
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(
|
||||
std::thread::Builder::new()
|
||||
.name("camera-track".into())
|
||||
.spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh)))
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key });
|
||||
self.events.push(Event::ProjectChanged { revision: self.revision });
|
||||
self.poll_camera_job();
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Write a finished analysis into its effect (one undo step) and drop the job.
|
||||
fn poll_camera_job(&mut self) -> bool {
|
||||
let Some(job) = &self.camera_job else { return false };
|
||||
if !job.is_finished() {
|
||||
return false;
|
||||
}
|
||||
let Some(mut job) = self.camera_job.take() else { return false };
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let result = lock(&job.shared.result).take();
|
||||
match (result, &st.error) {
|
||||
(Some(res), None) if !st.cancelled => {
|
||||
let (cid, lid, uid) = (job.comp, job.layer, job.effect);
|
||||
let (key, skey) = (job.key.clone(), job.solve_key.clone());
|
||||
let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json);
|
||||
let (solve_json, method, avg, msg, failed) = match &res.solve {
|
||||
Ok(s) => (
|
||||
s.to_json(),
|
||||
s.method_used.label().to_string(),
|
||||
s.average_error,
|
||||
format!(
|
||||
"3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)",
|
||||
s.frames.len(),
|
||||
s.points.len(),
|
||||
s.average_error,
|
||||
st.elapsed
|
||||
),
|
||||
false,
|
||||
),
|
||||
Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true),
|
||||
};
|
||||
let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| {
|
||||
let g = p
|
||||
.comp_mut(cid)
|
||||
.and_then(|c| c.layer_mut(lid))
|
||||
.and_then(|l| l.props.find_group_mut(uid))
|
||||
.ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?;
|
||||
let mut set = |m: &str, v: Value| {
|
||||
if let Some(pr) = g.prop_mut(m) {
|
||||
pr.value = v;
|
||||
}
|
||||
};
|
||||
if let Some(t) = &tracks_json {
|
||||
set(TRACKS, Value::Str(t.clone()));
|
||||
set(TRACKS_KEY, Value::Str(key.clone()));
|
||||
}
|
||||
set(SOLVE, Value::Str(solve_json.clone()));
|
||||
set(SOLVE_KEY, Value::Str(skey.clone()));
|
||||
set(METHOD_USED, Value::Str(method.clone()));
|
||||
set(AVERAGE_ERROR, Value::Scalar(avg));
|
||||
Ok(())
|
||||
});
|
||||
let (msg, failed) = match r {
|
||||
Ok(()) => (msg, failed),
|
||||
Err(e) => (format!("3D Camera Tracker: {e}"), true),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: failed });
|
||||
}
|
||||
_ => {
|
||||
let msg = match &st.error {
|
||||
Some(e) => format!("3D Camera Tracker analysis failed: {e}"),
|
||||
None => "3D Camera Tracker analysis cancelled".into(),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Frontends call this every frame: finishes analyses and (with `auto`) starts queued
|
||||
/// (re-)analyses in the background, like After Effects.
|
||||
pub fn poll_camera(&mut self, auto: bool) -> bool {
|
||||
let changed = self.poll_camera_job();
|
||||
if auto && self.camera_job.is_none() && self.offloaded(JobKind::Camera).is_none() {
|
||||
while let Some((c, l, e)) = self.camera_pending.first().copied() {
|
||||
self.camera_pending.remove(0);
|
||||
if self.start_camera(c, l, e, false).is_ok() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- solve → composition
|
||||
|
||||
/// A tracker's solve placed in its composition.
|
||||
pub struct Placed {
|
||||
pub solve: Arc<CameraSolve>,
|
||||
/// Canonical → comp world.
|
||||
pub world: Similarity,
|
||||
/// Comp pixels per layer pixel.
|
||||
pub scale: f64,
|
||||
/// Comp times of the analysed frames.
|
||||
pub times: Vec<Tick>,
|
||||
pub comp_size: [f64; 2],
|
||||
}
|
||||
|
||||
impl Placed {
|
||||
/// Comp-world position of a canonical point.
|
||||
pub fn to_world(&self, x: V3) -> V3 {
|
||||
self.world.apply(x)
|
||||
}
|
||||
/// Canonical position of a comp-world point.
|
||||
pub fn from_world(&self, x: V3) -> V3 {
|
||||
let r = &self.world;
|
||||
linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s)
|
||||
}
|
||||
/// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom).
|
||||
pub fn camera(&self, k: usize) -> (V3, V3, f64) {
|
||||
let c = &self.solve.frames[k];
|
||||
let m = linalg::mm(&self.world.r, &linalg::mt(&c.r()));
|
||||
(self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale)
|
||||
}
|
||||
/// Analysed frame at comp time `t`.
|
||||
pub fn frame_at(&self, t: Tick) -> usize {
|
||||
self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1))
|
||||
}
|
||||
}
|
||||
|
||||
/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point).
|
||||
fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 {
|
||||
let Some(comp) = project.comp(cid) else { return 1.0 };
|
||||
let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO));
|
||||
let m = ctx.world_matrix(layer).0;
|
||||
let s = m[0][0].hypot(m[1][0]);
|
||||
if s.is_finite() && s > 1e-9 { s } else { 1.0 }
|
||||
}
|
||||
|
||||
/// The solve of tracker `uid` on `lid`, placed in the comp.
|
||||
pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option<Placed> {
|
||||
let comp = project.comp(cid)?;
|
||||
let layer = comp.layer(lid)?;
|
||||
let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?;
|
||||
let solve = ct::solve(&static_params(g))?;
|
||||
let scale = layer_scale(project, cid, layer);
|
||||
let k0 = solve.first_solved()?;
|
||||
let comp_size = [comp.width as f64, comp.height as f64];
|
||||
let world = solve.world(comp_size, solve.frames[k0].focal * scale);
|
||||
let mut times = analysis_times(comp, layer);
|
||||
times.truncate(solve.frames.len());
|
||||
Some(Placed { solve, world, scale, times, comp_size })
|
||||
}
|
||||
|
||||
/// What to create from a target (`camera.createFromSolve`).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum CreateKind {
|
||||
Text,
|
||||
Solid,
|
||||
Null,
|
||||
ShadowCatcher,
|
||||
/// Only the camera.
|
||||
Camera,
|
||||
}
|
||||
|
||||
impl CreateKind {
|
||||
pub fn from_name(s: &str) -> Option<CreateKind> {
|
||||
match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() {
|
||||
"text" => Some(CreateKind::Text),
|
||||
"solid" => Some(CreateKind::Solid),
|
||||
"null" => Some(CreateKind::Null),
|
||||
"shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher),
|
||||
"camera" => Some(CreateKind::Camera),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera),
|
||||
/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera.
|
||||
pub fn plane_orientation(normal: V3, right: V3) -> V3 {
|
||||
let z = linalg::scale(normal, -1.0);
|
||||
let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z)));
|
||||
if linalg::norm(x) < 1e-9 {
|
||||
x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] };
|
||||
x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z)));
|
||||
}
|
||||
let x = linalg::normalize(x);
|
||||
let y = linalg::cross(z, x);
|
||||
let m = linalg::from_cols(x, y, z);
|
||||
norm_angles(linalg::euler_xyz(&m))
|
||||
}
|
||||
|
||||
fn norm_angles(a: V3) -> V3 {
|
||||
a.map(|v| {
|
||||
let r = v.rem_euclid(360.0);
|
||||
if (r - 360.0).abs() < 1e-9 { 0.0 } else { r }
|
||||
})
|
||||
}
|
||||
|
||||
/// Key `path` on `l` at every `(time, value)`.
|
||||
pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) {
|
||||
if let Some(pr) = l.props.prop_mut(path) {
|
||||
pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect();
|
||||
if let Some((_, v)) = keys.first() {
|
||||
pr.value = v.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame;
|
||||
/// Zoom keyed for Variable Zoom).
|
||||
pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) {
|
||||
l.auto_orient = AutoOrient::Off;
|
||||
let n = pl.times.len().min(pl.solve.frames.len());
|
||||
let mut pos = Vec::with_capacity(n);
|
||||
let mut ori = Vec::with_capacity(n);
|
||||
let mut zoom = Vec::with_capacity(n);
|
||||
let mut prev: Option<V3> = None;
|
||||
for k in 0..n {
|
||||
let (p, o, z) = pl.camera(k);
|
||||
// Unwrap the angles so per-frame keys interpolate the short way.
|
||||
let o = match prev {
|
||||
None => norm_angles(o),
|
||||
Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0),
|
||||
};
|
||||
prev = Some(o);
|
||||
let t = l.layer_time(pl.times[k]);
|
||||
pos.push((t, Value::Vec3(p)));
|
||||
ori.push((t, Value::Vec3(o)));
|
||||
zoom.push((t, Value::Scalar(z)));
|
||||
}
|
||||
key_all(l, "transform/position", &pos);
|
||||
key_all(l, "transform/orientation", &ori);
|
||||
let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal);
|
||||
if variable {
|
||||
key_all(l, "cameraOptions/zoom", &zoom);
|
||||
} else if let Some((_, z)) = zoom.first()
|
||||
&& let Some(pr) = l.props.prop_mut("cameraOptions/zoom")
|
||||
{
|
||||
pr.keys.clear();
|
||||
pr.value = z.clone();
|
||||
}
|
||||
if let Some((_, z)) = zoom.first()
|
||||
&& let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance")
|
||||
{
|
||||
pr.value = z.clone();
|
||||
}
|
||||
// A one-node camera has no Point of Interest.
|
||||
if let Some(tr) = l.props.sub_mut("transform") {
|
||||
tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi"));
|
||||
}
|
||||
for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] {
|
||||
if let Some(pr) = l.props.prop_mut(rot) {
|
||||
pr.keys.clear();
|
||||
pr.value = Value::Scalar(0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest
|
||||
/// solved neighbours (in 3D) complete the plane.
|
||||
pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option<Target> {
|
||||
let mut use_ids: Vec<u32> = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect();
|
||||
if use_ids.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if use_ids.len() < 3 {
|
||||
let c = solve.point(use_ids[0])?.pos;
|
||||
let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect();
|
||||
others.sort_by(|a, b| a.0.total_cmp(&b.0));
|
||||
for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) {
|
||||
use_ids.push(id);
|
||||
}
|
||||
let mut t = solve.target(&use_ids, k)?;
|
||||
// The target sits on the chosen point(s).
|
||||
let pts: Vec<V3> = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect();
|
||||
t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64);
|
||||
return Some(t);
|
||||
}
|
||||
solve.target(&use_ids, k)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn plane_orientation_faces_the_normal() {
|
||||
let n = linalg::normalize([0.2, -0.9, -0.3]);
|
||||
let o = plane_orientation(n, [1.0, 0.0, 0.0]);
|
||||
let m = linalg::from_euler_xyz(o);
|
||||
// Layer −z (its front) is the normal.
|
||||
let front = linalg::mv(&m, [0.0, 0.0, -1.0]);
|
||||
assert!(linalg::norm(linalg::sub(front, n)) < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
//! Mask tracking runtime (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode): follow
|
||||
//! the pixels inside a mask from frame to frame and key its Mask Path on every analysed frame.
|
||||
//!
|
||||
//! Analysis runs on a background thread (progress, cancel; blocking with `wait` and on wasm32)
|
||||
//! like the point tracker, over the layer's source frames; one undo step covers it. The
|
||||
//! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the
|
||||
//! chosen motion model), and every frame's motion is applied to the mask's vertices and
|
||||
//! tangents, so the shape's Bezier structure is kept.
|
||||
//!
|
||||
//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn
|
||||
//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path
|
||||
//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track
|
||||
//! Points** effect on the layer (created on first use), from which Extract & Copy Face
|
||||
//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_keyframe::{Keyframe, ShapePath, Value};
|
||||
use effectcraft_project::{ItemId, LayerId, Project, Uid};
|
||||
use effectcraft_render::{ExprHost, FootageSource, LayerCache, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track as trk;
|
||||
use effectcraft_track::fit::Model;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::tracking::{Direction, TrackProgress, lock, source_frame};
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Tracker panel ▸ Method (mask tracking).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum MaskMethod {
|
||||
Position,
|
||||
PositionScale,
|
||||
#[default]
|
||||
PositionScaleRotation,
|
||||
PositionScaleRotationSkew,
|
||||
Perspective,
|
||||
/// Face Tracking (Outline Only).
|
||||
FaceOutline,
|
||||
/// Face Tracking (Detailed Features).
|
||||
FaceDetailed,
|
||||
}
|
||||
|
||||
impl MaskMethod {
|
||||
pub const ALL: [MaskMethod; 7] = [
|
||||
MaskMethod::Position,
|
||||
MaskMethod::PositionScale,
|
||||
MaskMethod::PositionScaleRotation,
|
||||
MaskMethod::PositionScaleRotationSkew,
|
||||
MaskMethod::Perspective,
|
||||
MaskMethod::FaceOutline,
|
||||
MaskMethod::FaceDetailed,
|
||||
];
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
MaskMethod::Position => "Position",
|
||||
MaskMethod::PositionScale => "Position & Scale",
|
||||
MaskMethod::PositionScaleRotation => "Position, Scale & Rotation",
|
||||
MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew",
|
||||
MaskMethod::Perspective => "Perspective",
|
||||
MaskMethod::FaceOutline => "Face Tracking (Outline Only)",
|
||||
MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)",
|
||||
}
|
||||
}
|
||||
/// One of the Face Tracking methods.
|
||||
pub fn is_face(self) -> bool {
|
||||
matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed)
|
||||
}
|
||||
pub fn id(self) -> &'static str {
|
||||
match self {
|
||||
MaskMethod::Position => "position",
|
||||
MaskMethod::PositionScale => "positionScale",
|
||||
MaskMethod::PositionScaleRotation => "positionScaleRotation",
|
||||
MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew",
|
||||
MaskMethod::Perspective => "perspective",
|
||||
MaskMethod::FaceOutline => "faceOutline",
|
||||
MaskMethod::FaceDetailed => "faceDetailed",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<MaskMethod> {
|
||||
let n: String = s.chars().filter(char::is_ascii_alphanumeric).collect::<String>().to_ascii_lowercase();
|
||||
MaskMethod::ALL
|
||||
.into_iter()
|
||||
.find(|m| m.id().to_ascii_lowercase() == n || m.label().chars().filter(char::is_ascii_alphanumeric).collect::<String>().to_ascii_lowercase() == n)
|
||||
}
|
||||
pub fn model(self) -> Model {
|
||||
match self {
|
||||
MaskMethod::Position => Model::Translation,
|
||||
MaskMethod::PositionScale => Model::TranslationScale,
|
||||
MaskMethod::PositionScaleRotation => Model::Similarity,
|
||||
MaskMethod::PositionScaleRotationSkew => Model::Affine,
|
||||
MaskMethod::Perspective => Model::Homography,
|
||||
MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A closed polyline through a Bezier path (`steps` samples per segment), in layer pixels.
|
||||
pub fn flatten(p: &ShapePath, steps: usize) -> Vec<[f64; 2]> {
|
||||
let n = p.len();
|
||||
if n == 0 {
|
||||
return vec![];
|
||||
}
|
||||
let segs = if p.closed { n } else { n.saturating_sub(1) };
|
||||
let mut out = Vec::with_capacity(segs * steps + 1);
|
||||
for i in 0..segs {
|
||||
let j = (i + 1) % n;
|
||||
let (a, b) = (p.vertices[i], p.vertices[j]);
|
||||
let o = p.out_tangents.get(i).copied().unwrap_or([0.0; 2]);
|
||||
let it = p.in_tangents.get(j).copied().unwrap_or([0.0; 2]);
|
||||
let c = [a, [a[0] + o[0], a[1] + o[1]], [b[0] + it[0], b[1] + it[1]], b];
|
||||
for k in 0..steps {
|
||||
let t = k as f64 / steps as f64;
|
||||
let u = 1.0 - t;
|
||||
let w = [u * u * u, 3.0 * u * u * t, 3.0 * u * t * t, t * t * t];
|
||||
out.push([0, 1].map(|d| w[0] * c[0][d] + w[1] * c[1][d] + w[2] * c[2][d] + w[3] * c[3][d]));
|
||||
}
|
||||
}
|
||||
if !p.closed {
|
||||
out.push(p.vertices[n - 1]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A path moved by a frame-to-frame motion (vertices and tangent handles).
|
||||
pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath {
|
||||
ShapePath {
|
||||
vertices: p.vertices.iter().map(|v| h.apply(*v)).collect(),
|
||||
in_tangents: p.vertices.iter().zip(&p.in_tangents).map(|(v, t)| h.apply_tangent(*v, *t)).collect(),
|
||||
out_tangents: p.vertices.iter().zip(&p.out_tangents).map(|(v, t)| h.apply_tangent(*v, *t)).collect(),
|
||||
closed: p.closed,
|
||||
feather: p.feather.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom
|
||||
/// tangents. `like` lends its feather points.
|
||||
pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath {
|
||||
let m = poly.len();
|
||||
let n = n.clamp(3, m.max(3));
|
||||
let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect();
|
||||
let tan: Vec<[f64; 2]> = (0..n)
|
||||
.map(|i| {
|
||||
let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]);
|
||||
[(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0]
|
||||
})
|
||||
.collect();
|
||||
ShapePath {
|
||||
vertices: v,
|
||||
in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(),
|
||||
out_tangents: tan,
|
||||
closed: true,
|
||||
feather: if like.len() == n { like.feather.clone() } else { vec![] },
|
||||
}
|
||||
}
|
||||
|
||||
/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features).
|
||||
type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>);
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MaskTrackShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<TrackProgress>,
|
||||
frames: Mutex<Vec<MaskFrame>>,
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) mask track.
|
||||
pub struct MaskTrackJob {
|
||||
pub shared: Arc<MaskTrackShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub mask: Uid,
|
||||
pub direction: Direction,
|
||||
}
|
||||
|
||||
impl MaskTrackJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> TrackProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
pub(crate) struct MaskWork {
|
||||
pub project: Arc<Project>,
|
||||
pub footage: Arc<dyn FootageSource>,
|
||||
pub expr: Option<Arc<dyn ExprHost>>,
|
||||
pub cache: Arc<LayerCache>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
/// The mask path on the first frame.
|
||||
pub path: ShapePath,
|
||||
pub method: MaskMethod,
|
||||
/// Comp times; the first is the start frame.
|
||||
pub times: Vec<Tick>,
|
||||
/// Face tracking's trained model (Settings ▸ Face Tracking), if one is in use.
|
||||
pub face_model: Option<Arc<dyn effectcraft_segment::face::FaceModel>>,
|
||||
}
|
||||
|
||||
async fn run_work(w: MaskWork, shared: &MaskTrackShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
// A browser job worker's GPU (deferred readbacks): frames render in passes.
|
||||
let accel = crate::offload::job_accel();
|
||||
let accel = accel.as_deref();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel));
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let wr = &w;
|
||||
let frame_at =
|
||||
|t: Tick| crate::offload::in_passes(accel, move |a| source_frame(&r.with_accel(a), &wr.project, wr.comp, comp, layer, t, wr.expr.as_deref()));
|
||||
lock(&shared.state).total = w.times.len().saturating_sub(1) as u64;
|
||||
let Some(first) = frame_at(w.times[0]).await else { return fail("the layer has no pixels to track") };
|
||||
if w.method.is_face() {
|
||||
return run_face(&w, shared, first, &frame_at, accel.is_some(), layer, t0).await;
|
||||
}
|
||||
let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 });
|
||||
let mut path = w.path.clone();
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None));
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
let (res, nf) = crate::offload::join_fetch(
|
||||
accel.is_some(),
|
||||
|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }),
|
||||
|| async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } },
|
||||
)
|
||||
.await;
|
||||
next = nf;
|
||||
let Some(step) = res else {
|
||||
lock(&shared.state).error = Some(format!("lost the mask's pixels at {:.3} s (too little texture inside the mask)", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
path = transform_path(&path, &step.motion);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None));
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks.
|
||||
/// `deferred`: frames render in passes (awaited one after the other).
|
||||
async fn run_face<F: std::future::Future<Output = Option<(Arc<effectcraft_raster::Image>, [f64; 2])>>>(
|
||||
w: &MaskWork,
|
||||
shared: &MaskTrackShared,
|
||||
first: (Arc<effectcraft_raster::Image>, [f64; 2]),
|
||||
frame_at: &(impl Fn(Tick) -> F + Sync),
|
||||
deferred: bool,
|
||||
layer: &effectcraft_project::Layer,
|
||||
t0: web_time::Instant,
|
||||
) {
|
||||
let detailed = w.method == MaskMethod::FaceDetailed;
|
||||
let b = trk::mask::bounds(&flatten(&w.path, 8));
|
||||
let Some((mut tracker, fit)) = trk::face::FaceTracker::new_with(&trk::Frame { img: &first.0, offset: first.1 }, b, w.face_model.clone()) else {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some("no face found inside the mask (draw the mask around a face)".into());
|
||||
s.finished = true;
|
||||
return;
|
||||
};
|
||||
let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks));
|
||||
let (path, pts) = key(&fit);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts));
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
let (res, nf) = crate::offload::join_fetch(
|
||||
deferred,
|
||||
|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }),
|
||||
|| async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } },
|
||||
)
|
||||
.await;
|
||||
next = nf;
|
||||
let Some(fit) = res else {
|
||||
lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
let (path, pts) = key(&fit);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts));
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// The layer's Face Track Points effect (created when missing). Returns its uid.
|
||||
pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option<Uid> {
|
||||
face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID)
|
||||
}
|
||||
|
||||
/// The layer's effect `id`, applied when missing (data effects of face tracking).
|
||||
pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option<Uid> {
|
||||
let spec = effectcraft_effects::find(id)?;
|
||||
let l = p.comp(comp)?.layer(layer)?;
|
||||
if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) {
|
||||
return Some(g.uid);
|
||||
}
|
||||
let (w, h) = effectcraft_render::source_size(p, l);
|
||||
let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] };
|
||||
let mut next = p.next_id;
|
||||
let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size);
|
||||
let uid = g.uid;
|
||||
let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?;
|
||||
fx.children.push(g.into());
|
||||
p.next_id = next;
|
||||
Some(uid)
|
||||
}
|
||||
|
||||
fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) {
|
||||
let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None };
|
||||
let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return };
|
||||
if let Some(g) = layer.props.find_group_mut(job.mask)
|
||||
&& let Some(pr) = g.get_mut("path")
|
||||
{
|
||||
for (lt, _, path, _) in frames {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone())));
|
||||
}
|
||||
}
|
||||
if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) {
|
||||
for (lt, _, _, pts) in frames {
|
||||
let Some(pts) = pts else { continue };
|
||||
for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() {
|
||||
if let Some(pr) = g.get_mut(id) {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i])));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Whether a mask track is running.
|
||||
pub fn is_mask_tracking(&self) -> bool {
|
||||
self.mask_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded(crate::offload::JobKind::MaskTrack).is_some()
|
||||
}
|
||||
|
||||
/// Live progress of the running (or just finished) mask track.
|
||||
pub fn mask_track_progress(&self) -> Option<TrackProgress> {
|
||||
self.mask_job.as_ref().map(|j| j.progress()).or_else(|| self.offloaded(crate::offload::JobKind::MaskTrack).map(|j| j.progress.track()))
|
||||
}
|
||||
|
||||
/// Cancel the running mask track (frames tracked so far are kept).
|
||||
pub fn stop_mask_track(&mut self) -> bool {
|
||||
if self.cancel_offloaded(crate::offload::JobKind::MaskTrack) {
|
||||
return true;
|
||||
}
|
||||
match &self.mask_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.shared.cancel.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start_mask_track(&mut self, work: MaskWork, mask: Uid, direction: Direction, wait: bool) -> Result<(), String> {
|
||||
if self.is_mask_tracking() || self.is_tracking() {
|
||||
return Err("a track analysis is already running".into());
|
||||
}
|
||||
self.poll_mask_track();
|
||||
if !wait && self.offloads() && !work.method.is_face() {
|
||||
let MaskWork { comp, layer, path, method, times, .. } = work;
|
||||
return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times });
|
||||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let levels = self.prefs.general.undo_levels as usize;
|
||||
self.history.record("Track Mask", self.project.clone(), levels);
|
||||
let shared = Arc::new(MaskTrackShared::default());
|
||||
let (comp, layer) = (work.comp, work.layer);
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
async move { run_work(work, &sh).await }
|
||||
});
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(
|
||||
std::thread::Builder::new()
|
||||
.name("mask-track".into())
|
||||
.spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh)))
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.mask_job = Some(MaskTrackJob { shared, thread, comp, layer, mask, direction });
|
||||
self.poll_mask_track();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Key newly tracked frames and move the CTI with them; drop the job once it finished.
|
||||
/// Frontends call this every frame while tracking.
|
||||
pub fn poll_mask_track(&mut self) -> bool {
|
||||
let Some(job) = &self.mask_job else { return false };
|
||||
let frames = std::mem::take(&mut *lock(&job.shared.frames));
|
||||
let finished = job.is_finished();
|
||||
let changed = !frames.is_empty();
|
||||
if changed {
|
||||
write_frames(Arc::make_mut(&mut self.project), job, &frames);
|
||||
if self.state.active_comp == Some(job.comp)
|
||||
&& let Some(last) = frames.last()
|
||||
{
|
||||
self.state.times.insert(job.comp, last.1);
|
||||
}
|
||||
self.bump();
|
||||
}
|
||||
if finished && let Some(mut job) = self.mask_job.take() {
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let msg = match (&st.error, st.cancelled) {
|
||||
(Some(e), _) => format!("Mask tracking stopped: {e}"),
|
||||
(None, true) => format!("Mask tracking stopped after {} frame(s)", st.done),
|
||||
_ => format!("Tracked the mask over {} frame(s) in {:.1} s", st.done, st.elapsed),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
}
|
||||
changed || finished
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn methods_parse_and_paths_transform() {
|
||||
assert_eq!(MaskMethod::from_name("positionScaleRotationSkew"), Some(MaskMethod::PositionScaleRotationSkew));
|
||||
assert_eq!(MaskMethod::from_name("Position & Scale"), Some(MaskMethod::PositionScale));
|
||||
assert_eq!(MaskMethod::from_name("perspective"), Some(MaskMethod::Perspective));
|
||||
let p = ShapePath::ellipse([50.0, 40.0], 40.0, 20.0);
|
||||
let poly = flatten(&p, 8);
|
||||
assert_eq!(poly.len(), p.len() * 8);
|
||||
assert!(poly.iter().all(|q| ((q[0] - 50.0) / 20.0).powi(2) + ((q[1] - 40.0) / 10.0).powi(2) < 1.01));
|
||||
let h = trk::Homography([[0.0, -2.0, 10.0], [2.0, 0.0, 0.0], [0.0, 0.0, 1.0]]);
|
||||
let q = transform_path(&p, &h);
|
||||
for i in 0..p.len() {
|
||||
let v = h.apply(p.vertices[i]);
|
||||
assert!((q.vertices[i][0] - v[0]).abs() < 1e-9);
|
||||
let t = p.out_tangents[i];
|
||||
assert!((q.out_tangents[i][0] + 2.0 * t[1]).abs() < 1e-9 && (q.out_tangents[i][1] - 2.0 * t[0]).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
//! Trained models (Settings ▸ Roto Brush and Settings ▸ Face Tracking; `roto.model*` and
|
||||
//! `face.model*` commands): which models the registry offers (`effectcraft_segment::MODELS`, all
|
||||
//! open source), which are installed (in the `models` folder), installing one from a file or
|
||||
//! downloading the official weights (verified against the registry's SHA-256 either way, and kept
|
||||
//! with a notice naming their authors and licence), choosing one per task, and loading it in the
|
||||
//! background: into the Roto Brush effect (`effects::roto::set_model`) or the face tracker
|
||||
//! ([`Models::face`]).
|
||||
//!
|
||||
//! Weights are never bundled: the install stays small and nothing is fetched unless asked for.
|
||||
//! Downloads use the system's `curl` (shipped with Windows 10+, macOS and most Linux systems),
|
||||
//! so no networking code is linked into EffectCraft; without it, the file can be downloaded in a
|
||||
//! browser and installed with Install from File. The browser build uses the built-in engines
|
||||
//! only, for now.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_segment::face::FaceModel;
|
||||
use effectcraft_segment::{self as seg, Loaded, ModelInfo, Task};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::commands::{CommandSpec, always, b_p, bad, str_p};
|
||||
use crate::{EngineError, Result, Session, cmd, query};
|
||||
|
||||
/// What a task's background worker reports.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct Status {
|
||||
/// What is happening now ("Downloading MobileSAM…").
|
||||
pub busy: Option<String>,
|
||||
/// The model in use.
|
||||
pub active: Option<String>,
|
||||
/// The last failure, for the Settings page.
|
||||
pub error: Option<String>,
|
||||
/// A finished job's message and whether it failed, for a toast (taken by
|
||||
/// [`Session::poll_models`]).
|
||||
pub done: Option<(String, bool)>,
|
||||
}
|
||||
|
||||
/// The session's trained models: per task, the worker's status; and the face model in use.
|
||||
#[derive(Default)]
|
||||
pub struct Models {
|
||||
roto: Arc<Mutex<Status>>,
|
||||
face: Arc<Mutex<Status>>,
|
||||
face_model: FaceSlot,
|
||||
}
|
||||
|
||||
impl Models {
|
||||
fn lane(&self, task: Task) -> &Arc<Mutex<Status>> {
|
||||
match task {
|
||||
Task::Mask => &self.roto,
|
||||
Task::Face => &self.face,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn status(&self, task: Task) -> Status {
|
||||
self.lane(task).lock().map(|s| s.clone()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The face model face tracking uses (`None`: the classic tracker).
|
||||
pub fn face(&self) -> Option<Arc<dyn FaceModel>> {
|
||||
self.face_model.lock().ok().and_then(|m| m.clone())
|
||||
}
|
||||
|
||||
fn set_active(&self, task: Task, model: Option<Loaded>) {
|
||||
activate(self.lane(task), &self.face_model, task, model);
|
||||
}
|
||||
}
|
||||
|
||||
type FaceSlot = Arc<Mutex<Option<Arc<dyn FaceModel>>>>;
|
||||
|
||||
/// Put a loaded model to use (or drop `task`'s): Roto Brush's lives in the effect, the face
|
||||
/// tracker's in `face`.
|
||||
fn activate(lane: &Arc<Mutex<Status>>, face: &FaceSlot, task: Task, model: Option<Loaded>) {
|
||||
let id = model.as_ref().map(|m| match m {
|
||||
Loaded::Mask(m) => m.info().id.to_string(),
|
||||
Loaded::Face(m) => m.info().id.to_string(),
|
||||
});
|
||||
match (task, model) {
|
||||
(Task::Mask, Some(Loaded::Mask(m))) => crate::effects::roto::set_model(Some(m)),
|
||||
(Task::Mask, _) => crate::effects::roto::set_model(None),
|
||||
(Task::Face, m) => {
|
||||
if let Ok(mut slot) = face.lock() {
|
||||
*slot = match m {
|
||||
Some(Loaded::Face(f)) => Some(f),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
set(lane, |s| s.active = id);
|
||||
}
|
||||
|
||||
fn set(status: &Arc<Mutex<Status>>, f: impl FnOnce(&mut Status)) {
|
||||
if let Ok(mut s) = status.lock() {
|
||||
f(&mut s);
|
||||
}
|
||||
}
|
||||
|
||||
/// How a task shows in Settings, toasts and the command list.
|
||||
struct Kind {
|
||||
/// Command prefix and settings key section.
|
||||
prefix: &'static str,
|
||||
label: &'static str,
|
||||
classic_name: &'static str,
|
||||
classic_description: &'static str,
|
||||
}
|
||||
|
||||
const ROTO: Kind = Kind {
|
||||
prefix: "roto",
|
||||
label: "Roto Brush",
|
||||
classic_name: "Classic (graph cut)",
|
||||
classic_description: "Built in: colour models and graph cuts, propagated by optical flow. Always available.",
|
||||
};
|
||||
|
||||
const FACE: Kind = Kind {
|
||||
prefix: "face",
|
||||
label: "Face Tracking",
|
||||
classic_name: "Classic (shape model)",
|
||||
classic_description: "Built in: a skin colour model, facial feature shapes and a fitted face shape model. Best on front-facing, evenly lit faces. Always available.",
|
||||
};
|
||||
|
||||
fn kind(task: Task) -> &'static Kind {
|
||||
match task {
|
||||
Task::Mask => &ROTO,
|
||||
Task::Face => &FACE,
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// The folder installed models live in: the host's, else `models` next to the settings.
|
||||
pub fn models_dir(&self) -> Option<PathBuf> {
|
||||
self.models_dir.clone().or_else(|| Some(self.config.as_ref()?.dir()?.join("models")))
|
||||
}
|
||||
|
||||
/// Where `m` is installed, if it is (a file of the published size).
|
||||
pub fn model_path(&self, m: &ModelInfo) -> Option<PathBuf> {
|
||||
let p = self.models_dir()?.join(m.file_name);
|
||||
std::fs::metadata(&p).ok().filter(|md| md.len() == m.size).map(|_| p)
|
||||
}
|
||||
|
||||
/// The model chosen for `task` (Settings).
|
||||
pub fn chosen_model(&self, task: Task) -> &str {
|
||||
match task {
|
||||
Task::Mask => &self.prefs.roto.model,
|
||||
Task::Face => &self.prefs.face.model,
|
||||
}
|
||||
}
|
||||
|
||||
fn choose_model(&mut self, task: Task, id: &str) {
|
||||
let slot = match task {
|
||||
Task::Mask => &mut self.prefs.roto.model,
|
||||
Task::Face => &mut self.prefs.face.model,
|
||||
};
|
||||
*slot = id.to_string();
|
||||
}
|
||||
|
||||
/// Load (or drop) the models Settings chooses; called whenever settings change.
|
||||
pub fn apply_models(&mut self) {
|
||||
for task in [Task::Mask, Task::Face] {
|
||||
self.apply_model(task, false);
|
||||
}
|
||||
}
|
||||
|
||||
/// Load (or drop) `task`'s chosen model; `wait`: before returning (scripts and agents).
|
||||
fn apply_model(&mut self, task: Task, wait: bool) {
|
||||
let want = self.chosen_model(task).to_string();
|
||||
let lane = self.models.lane(task).clone();
|
||||
let st = self.models.status(task);
|
||||
let info = seg::info(&want).filter(|m| m.task == task);
|
||||
let Some(info) = info else {
|
||||
if st.active.is_some() {
|
||||
self.models.set_active(task, None);
|
||||
}
|
||||
return;
|
||||
};
|
||||
if st.active.as_deref() == Some(info.id) || st.busy.is_some() {
|
||||
return;
|
||||
}
|
||||
let Some(path) = self.model_path(info) else {
|
||||
self.models.set_active(task, None);
|
||||
return;
|
||||
};
|
||||
let (face_slot, label) = (self.models.face_model.clone(), kind(task).label);
|
||||
spawn(&lane, label, format!("Loading {}…", info.name), wait, move |status| {
|
||||
let m = load_file(&path, info.id)?;
|
||||
activate(status, &face_slot, task, Some(m));
|
||||
Ok(format!("{label}: {} loaded", info.name))
|
||||
});
|
||||
}
|
||||
|
||||
/// Finish what the model workers did: a toast, and loading a model that just arrived.
|
||||
pub fn poll_models(&mut self) {
|
||||
for task in [Task::Mask, Task::Face] {
|
||||
let done = self.models.lane(task).lock().ok().and_then(|mut s| s.done.take());
|
||||
if let Some((message, error)) = done {
|
||||
self.events.push(crate::Event::Toast { message, error });
|
||||
self.apply_model(task, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn load_file(path: &Path, id: &str) -> std::result::Result<Loaded, String> {
|
||||
let bytes = std::fs::read(path).map_err(|e| format!("cannot read {}: {e}", path.display()))?;
|
||||
seg::load(id, &bytes)
|
||||
}
|
||||
|
||||
/// Run `job` on a worker thread (`wait`: right here), reporting through `status` (desktop; the
|
||||
/// browser build has no trained models yet).
|
||||
fn spawn(
|
||||
status: &Arc<Mutex<Status>>,
|
||||
label: &'static str,
|
||||
what: String,
|
||||
wait: bool,
|
||||
job: impl FnOnce(&Arc<Mutex<Status>>) -> std::result::Result<String, String> + Send + 'static,
|
||||
) {
|
||||
if cfg!(target_arch = "wasm32") {
|
||||
set(status, |s| s.error = Some("trained models are available in the desktop app".into()));
|
||||
return;
|
||||
}
|
||||
set(status, |s| {
|
||||
s.busy = Some(what);
|
||||
s.error = None;
|
||||
});
|
||||
let st = status.clone();
|
||||
let run = move || {
|
||||
let r = job(&st);
|
||||
set(&st, |s| {
|
||||
s.busy = None;
|
||||
match r {
|
||||
Ok(msg) => s.done = Some((msg, false)),
|
||||
Err(e) => {
|
||||
s.done = Some((format!("{label} model: {e}"), true));
|
||||
s.error = Some(e);
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
if wait {
|
||||
run();
|
||||
} else if let Err(e) = std::thread::Builder::new().name("ec-model".into()).spawn(run) {
|
||||
set(status, |s| {
|
||||
s.busy = None;
|
||||
s.error = Some(format!("cannot start: {e}"));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The notice kept next to `m`'s weights.
|
||||
fn notice_path(dir: &Path, m: &ModelInfo) -> PathBuf {
|
||||
dir.join(format!("{}.NOTICE.txt", m.file_name))
|
||||
}
|
||||
|
||||
/// Copy verified weights into the models folder (atomically), with their notice.
|
||||
fn install_bytes(dir: &Path, m: &ModelInfo, bytes: &[u8]) -> std::result::Result<PathBuf, String> {
|
||||
if bytes.len() as u64 != m.size || seg::sha256::hex(bytes) != m.sha256 {
|
||||
return Err(format!("this is not the published {} file (size or SHA-256 differs)", m.name));
|
||||
}
|
||||
use crate::config::FileOps;
|
||||
let path = dir.join(m.file_name);
|
||||
let files = crate::config::StdFiles;
|
||||
files.write(&path, bytes).map_err(|e| format!("cannot write {}: {e}", path.display()))?;
|
||||
files.write(¬ice_path(dir, m), seg::notice(m).as_bytes()).map_err(|e| format!("cannot write the notice: {e}"))?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn model_p(p: &Value, cmd: &str, task: Task) -> Result<&'static ModelInfo> {
|
||||
let id = str_p(p, "id").ok_or_else(|| bad(cmd, "missing `id`"))?;
|
||||
seg::info(id).filter(|m| m.task == task).ok_or_else(|| bad(cmd, format!("unknown model `{id}` (see {}.models)", kind(task).prefix)))
|
||||
}
|
||||
|
||||
fn list(s: &mut Session, task: Task) -> Result<Value> {
|
||||
let k = kind(task);
|
||||
let st = s.models.status(task);
|
||||
let chosen = s.chosen_model(task).to_string();
|
||||
let mut v = vec![json!({
|
||||
"id": seg::CLASSICAL,
|
||||
"name": k.classic_name,
|
||||
"description": k.classic_description,
|
||||
"licence": "MIT OR Apache-2.0",
|
||||
"installed": true,
|
||||
"selected": chosen == seg::CLASSICAL,
|
||||
"active": st.active.is_none(),
|
||||
})];
|
||||
for m in seg::models(task) {
|
||||
v.push(json!({
|
||||
"id": m.id,
|
||||
"name": m.name,
|
||||
"description": m.description,
|
||||
"authors": m.authors,
|
||||
"licence": m.licence,
|
||||
"licenceUrl": m.licence_url,
|
||||
"homepage": m.homepage,
|
||||
"url": m.url,
|
||||
"size": m.size,
|
||||
"sha256": m.sha256,
|
||||
"installed": s.model_path(m).is_some(),
|
||||
"selected": chosen == m.id,
|
||||
"active": st.active.as_deref() == Some(m.id),
|
||||
}));
|
||||
}
|
||||
Ok(json!({"models": v, "busy": st.busy, "error": st.error, "folder": s.models_dir().map(|d| d.display().to_string())}))
|
||||
}
|
||||
|
||||
/// After a `wait`: the job's failure, as the command's.
|
||||
fn waited(s: &Session, task: Task, cmd: &str) -> Result<()> {
|
||||
match s.models.status(task).error {
|
||||
Some(e) => Err(bad(cmd, e)),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn select(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
let cmd = format!("{}.model.select", kind(task).prefix);
|
||||
let id = str_p(p, "id").ok_or_else(|| bad(&cmd, "missing `id`"))?;
|
||||
if id != seg::CLASSICAL {
|
||||
model_p(p, &cmd, task)?;
|
||||
}
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
s.choose_model(task, id);
|
||||
if wait {
|
||||
s.apply_model(task, true);
|
||||
waited(s, task, &cmd)?;
|
||||
}
|
||||
s.save_prefs();
|
||||
s.prefs_changed();
|
||||
Ok(json!({"model": id, "active": s.models.status(task).active}))
|
||||
}
|
||||
|
||||
fn install(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
let cmd = format!("{}.model.install", kind(task).prefix);
|
||||
let path = str_p(p, "path").ok_or_else(|| bad(&cmd, "missing `path` (the downloaded weights file)"))?;
|
||||
let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?;
|
||||
let bytes = std::fs::read(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?;
|
||||
// Which model it is: the one asked for, or the one whose checksum it has.
|
||||
let m = match str_p(p, "id") {
|
||||
Some(_) => model_p(p, &cmd, task)?,
|
||||
None => {
|
||||
let h = seg::sha256::hex(&bytes);
|
||||
seg::models(task).find(|m| m.sha256 == h).ok_or_else(|| bad(&cmd, "this file is not a published model in the registry (SHA-256 differs)"))?
|
||||
}
|
||||
};
|
||||
let dest = install_bytes(&dir, m, &bytes).map_err(|e| bad(&cmd, e))?;
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
s.apply_model(task, wait);
|
||||
if wait {
|
||||
waited(s, task, &cmd)?;
|
||||
}
|
||||
Ok(json!({"id": m.id, "path": dest.display().to_string()}))
|
||||
}
|
||||
|
||||
fn download(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
let k = kind(task);
|
||||
let cmd = format!("{}.model.download", k.prefix);
|
||||
let m = model_p(p, &cmd, task)?;
|
||||
let dir = s.models_dir().ok_or_else(|| bad(&cmd, "no settings folder to install into"))?;
|
||||
if s.models.status(task).busy.is_some() {
|
||||
return Err(bad(&cmd, "a model is already downloading or loading"));
|
||||
}
|
||||
let size = if m.size >= 10_000_000 { format!("{} MB", m.size / 1_000_000) } else { format!("{:.1} MB", m.size as f64 / 1e6) };
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
spawn(s.models.lane(task), k.label, format!("Downloading {} ({size})…", m.name), wait, move |_| {
|
||||
std::fs::create_dir_all(&dir).map_err(|e| format!("cannot create {}: {e}", dir.display()))?;
|
||||
let tmp = dir.join(format!("{}.part", m.file_name));
|
||||
let out = std::process::Command::new("curl")
|
||||
.args(["-fsSL", "--retry", "2", "--connect-timeout", "20", "-o"])
|
||||
.arg(&tmp)
|
||||
.arg(m.url)
|
||||
.output()
|
||||
.map_err(|e| format!("downloading needs curl ({e}); or download {} in a browser and use Install from File", m.url))?;
|
||||
if !out.status.success() {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(format!("download failed: {}", String::from_utf8_lossy(&out.stderr).trim()));
|
||||
}
|
||||
let bytes = std::fs::read(&tmp).map_err(|e| format!("cannot read the download: {e}"))?;
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
install_bytes(&dir, m, &bytes)?;
|
||||
Ok(format!("{}: {} downloaded and verified", k.label, m.name))
|
||||
});
|
||||
if wait {
|
||||
waited(s, task, &cmd)?;
|
||||
s.apply_model(task, true);
|
||||
waited(s, task, &cmd)?;
|
||||
return Ok(json!({"id": m.id, "installed": true}));
|
||||
}
|
||||
Ok(json!({"id": m.id, "downloading": true}))
|
||||
}
|
||||
|
||||
fn remove(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
let m = model_p(p, &format!("{}.model.remove", kind(task).prefix), task)?;
|
||||
let Some(path) = s.model_path(m) else { return Ok(json!({"removed": false})) };
|
||||
if s.models.status(task).active.as_deref() == Some(m.id) {
|
||||
s.models.set_active(task, None);
|
||||
}
|
||||
std::fs::remove_file(&path).map_err(|e| EngineError::Other(format!("cannot remove {}: {e}", path.display())))?;
|
||||
if let Some(dir) = path.parent() {
|
||||
let _ = std::fs::remove_file(notice_path(dir, m));
|
||||
}
|
||||
Ok(json!({"removed": true}))
|
||||
}
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![
|
||||
query!("roto.models", "Roto Brush Models", "{}", |s, _| list(s, Task::Mask)),
|
||||
cmd!(
|
||||
"roto.model.select",
|
||||
"Use Roto Brush Model",
|
||||
[],
|
||||
None,
|
||||
"{id: classical|mobilesam, wait?} (Roto Brush 2.0 / 3.0 use it; wait: loaded before returning)",
|
||||
always,
|
||||
|s, p| select(s, p, Task::Mask)
|
||||
),
|
||||
cmd!(
|
||||
"roto.model.install",
|
||||
"Install Roto Brush Model",
|
||||
[],
|
||||
None,
|
||||
"{path: weights file, id?, wait?} — verified against the registry's SHA-256",
|
||||
always,
|
||||
|s, p| install(s, p, Task::Mask)
|
||||
),
|
||||
cmd!(
|
||||
"roto.model.download",
|
||||
"Download Roto Brush Model",
|
||||
[],
|
||||
None,
|
||||
"{id, wait?} — the official weights, verified (desktop; uses the system curl)",
|
||||
always,
|
||||
|s, p| download(s, p, Task::Mask)
|
||||
),
|
||||
cmd!("roto.model.remove", "Remove Roto Brush Model", [], None, "{id}", always, |s, p| remove(s, p, Task::Mask)),
|
||||
query!("face.models", "Face Tracking Models", "{}", |s, _| list(s, Task::Face)),
|
||||
cmd!(
|
||||
"face.model.select",
|
||||
"Use Face Tracking Model",
|
||||
[],
|
||||
None,
|
||||
"{id: classical|mediapipe-face, wait?} (face tracking uses it; wait: loaded before returning)",
|
||||
always,
|
||||
|s, p| select(s, p, Task::Face)
|
||||
),
|
||||
cmd!(
|
||||
"face.model.install",
|
||||
"Install Face Tracking Model",
|
||||
[],
|
||||
None,
|
||||
"{path: model file, id?, wait?} — verified against the registry's SHA-256",
|
||||
always,
|
||||
|s, p| install(s, p, Task::Face)
|
||||
),
|
||||
cmd!(
|
||||
"face.model.download",
|
||||
"Download Face Tracking Model",
|
||||
[],
|
||||
None,
|
||||
"{id, wait?} — the official model, verified (desktop; uses the system curl)",
|
||||
always,
|
||||
|s, p| download(s, p, Task::Face)
|
||||
),
|
||||
cmd!("face.model.remove", "Remove Face Tracking Model", [], None, "{id}", always, |s, p| remove(s, p, Task::Face)),
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
//! Roto Brush & Refine Edge runtime: propagation and Freeze jobs, Input Key maintenance and the
|
||||
//! tool options.
|
||||
//!
|
||||
//! Segmentations are derived data in `effectcraft_effects::roto`'s content-keyed cache (see that
|
||||
//! module). `roto.propagate` fills it at full resolution in the background (or blocking with
|
||||
//! `wait`; in the browser the background is a Web Worker whose segmentations stream back,
|
||||
//! [`crate::offload`]), frame by frame outward from the base frame, rendering the effect's input
|
||||
//! (`Renderer::layer_input`, source → masks → the effects above it). `roto.freeze` runs the same
|
||||
//! pass and stores the final mattes in the effect (one undo step). `sync`, run after every
|
||||
//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made
|
||||
//! of (the layer's source and timing, its masks and the effects above it) and drops frozen
|
||||
//! mattes made from other frames.
|
||||
//!
|
||||
//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the
|
||||
//! frames are [`om::job_frames`] around the reference frame, each segmented through
|
||||
//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit
|
||||
//! to an instance's frames or prompt queues it for background tracking like a Roto Brush.
|
||||
|
||||
use crate::offload::JobKind;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_effects::object_matte as om;
|
||||
use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY};
|
||||
use effectcraft_effects::{Params, flatten_params};
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::tracking::lock;
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Roto Brush / Refine Edge tool options (serde: `roto.options`).
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct RotoOptions {
|
||||
/// Roto Brush tip diameter (layer pixels).
|
||||
pub diameter: f64,
|
||||
/// Refine Edge tip diameter (layer pixels).
|
||||
pub refine_diameter: f64,
|
||||
/// Layer panel view: `alphaBoundary` (pink outline), `alpha`, `alphaOverlay`, `none`.
|
||||
pub view: String,
|
||||
/// Alpha Overlay colour (straight RGB 0–1) and opacity (percent).
|
||||
pub overlay_color: [f64; 3],
|
||||
pub overlay_opacity: f64,
|
||||
/// Alpha Boundary colour.
|
||||
pub boundary_color: [f64; 3],
|
||||
/// Start propagating in the background after each stroke.
|
||||
pub auto_propagate: bool,
|
||||
}
|
||||
|
||||
impl Default for RotoOptions {
|
||||
fn default() -> Self {
|
||||
RotoOptions {
|
||||
diameter: 20.0,
|
||||
refine_diameter: 10.0,
|
||||
view: "alphaBoundary".into(),
|
||||
overlay_color: [1.0, 0.0, 0.0],
|
||||
overlay_opacity: 50.0,
|
||||
boundary_color: [1.0, 0.25, 0.75],
|
||||
auto_propagate: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"];
|
||||
|
||||
/// What a job does.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum RotoTask {
|
||||
Propagate,
|
||||
Freeze,
|
||||
}
|
||||
|
||||
/// Which way to propagate.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
Forward,
|
||||
Backward,
|
||||
#[default]
|
||||
Both,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn from_name(s: &str) -> Option<Direction> {
|
||||
Some(match s.to_ascii_lowercase().as_str() {
|
||||
"forward" | "fwd" => Direction::Forward,
|
||||
"backward" | "back" => Direction::Backward,
|
||||
"both" | "all" => Direction::Both,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Live job progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RotoProgress {
|
||||
pub task: Option<RotoTask>,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: Option<String>,
|
||||
/// The job works on an Object Matte (else a Roto Brush & Refine Edge).
|
||||
pub object_matte: bool,
|
||||
}
|
||||
|
||||
impl RotoProgress {
|
||||
pub fn banner(&self) -> String {
|
||||
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
|
||||
match (self.task, self.object_matte) {
|
||||
(Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
|
||||
(Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"),
|
||||
(_, false) => format!("Propagating Roto Brush: {pc}%"),
|
||||
(_, true) => format!("Tracking Object Matte: {pc}%"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct RotoShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<RotoProgress>,
|
||||
frozen: Mutex<Option<FrozenCache>>,
|
||||
}
|
||||
|
||||
pub struct RotoJob {
|
||||
pub shared: Arc<RotoShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub effect: Uid,
|
||||
pub task: RotoTask,
|
||||
/// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a
|
||||
/// Freeze result is dropped when it changed.
|
||||
strokes: String,
|
||||
/// The job works on an Object Matte.
|
||||
pub object_matte: bool,
|
||||
}
|
||||
|
||||
impl RotoJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> RotoProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_roto(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID)
|
||||
}
|
||||
|
||||
/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on
|
||||
/// demand while rendering).
|
||||
pub(crate) fn is_object_matte(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID)
|
||||
}
|
||||
|
||||
/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking
|
||||
/// parameters (static values), for queueing it after edits and guarding a Freeze.
|
||||
pub(crate) fn om_prompt(g: &PropGroup) -> String {
|
||||
let ps = params_static(g);
|
||||
let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect();
|
||||
keys.sort();
|
||||
keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect()
|
||||
}
|
||||
|
||||
/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte).
|
||||
fn guard_of(g: &PropGroup) -> String {
|
||||
if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) }
|
||||
}
|
||||
|
||||
fn str_of(g: &PropGroup, m: &str) -> String {
|
||||
match g.get(m).map(|p| &p.value) {
|
||||
Some(Value::Str(s)) => s.clone(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_str(g: &mut PropGroup, m: &str, v: String) {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
p.value = Value::Str(v);
|
||||
p.keys.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn fnv(s: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in s.as_bytes() {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// What the effect's input frames are made of: the layer's source and timing, its masks and the
|
||||
/// effects above the Roto Brush.
|
||||
pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
|
||||
let mut s = crate::warp::signature(p, comp, layer);
|
||||
if let Some(m) = layer.masks() {
|
||||
s.push_str(&format!("{m:?}"));
|
||||
}
|
||||
if let Some(fx) = layer.effects() {
|
||||
for g in fx.groups() {
|
||||
if g.uid == uid {
|
||||
break;
|
||||
}
|
||||
s.push_str(&format!("{g:?}"));
|
||||
}
|
||||
}
|
||||
format!("{:016x}", fnv(&s))
|
||||
}
|
||||
|
||||
/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from
|
||||
/// other frames are dropped. Returns the instances whose segmentation changed (to propagate
|
||||
/// again): new frames, strokes (Roto Brush) or prompt (Object Matte).
|
||||
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
|
||||
let mut todo = vec![];
|
||||
let mut set = vec![];
|
||||
for (cid, it) in &after.items {
|
||||
let ItemKind::Comp(c) = &it.kind else { continue };
|
||||
for l in &c.layers {
|
||||
let Some(fxg) = l.effects() else { continue };
|
||||
for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) {
|
||||
let key = input_key(after, *cid, l, g.uid);
|
||||
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
|
||||
if str_of(g, INPUT_KEY) != key {
|
||||
set.push((*cid, l.id, g.uid, key));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if old.is_none_or(|o| guard_of(o) != guard_of(g)) {
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (cid, lid, uid, key) in set {
|
||||
if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) {
|
||||
let had = !str_of(g, INPUT_KEY).is_empty();
|
||||
set_str(g, INPUT_KEY, key);
|
||||
if had {
|
||||
set_str(g, FROZEN, String::new());
|
||||
}
|
||||
}
|
||||
}
|
||||
todo.retain(|(c, l, u)| {
|
||||
after
|
||||
.comp(*c)
|
||||
.and_then(|c| c.layer(*l))
|
||||
.and_then(|l| l.props.find_group(*u))
|
||||
.is_some_and(|g| is_object_matte(g) || !fx::data(¶ms_static(g)).is_empty())
|
||||
});
|
||||
todo
|
||||
}
|
||||
|
||||
/// Parameters of an instance with static values (strokes and keys are never animated).
|
||||
pub(crate) fn params_static(g: &PropGroup) -> Params {
|
||||
flatten_params(g, &mut |p| p.value.clone())
|
||||
}
|
||||
|
||||
/// Evaluated parameters of an instance at comp time `t`.
|
||||
pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup, t: Tick, expr: Option<&dyn ExprHost>) -> Params {
|
||||
match p.comp(cid) {
|
||||
Some(c) => {
|
||||
let ctx = EvalCtx { expr, ..EvalCtx::new(p, cid, c, t) };
|
||||
flatten_params(g, &mut |pr| ctx.value(layer, pr))
|
||||
}
|
||||
None => params_static(g),
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer's source size (layer pixels).
|
||||
pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] {
|
||||
effectcraft_render::effect_bounds(p, comp, layer).0
|
||||
}
|
||||
|
||||
/// First and last layer frame of the layer inside the comp (inclusive).
|
||||
pub fn frame_limits(comp: &effectcraft_project::Comp, layer: &Layer) -> [i64; 2] {
|
||||
let fps = comp.frame_rate.as_f64().max(1e-6);
|
||||
let lo = layer.layer_time(layer.in_point.max(Tick::ZERO)).seconds();
|
||||
let hi = layer.layer_time(layer.out_point.min(comp.duration)).seconds();
|
||||
let a = fx::frame_of(lo, fps);
|
||||
let b = (fx::frame_of(hi, fps) - 1).max(a);
|
||||
[a, b]
|
||||
}
|
||||
|
||||
/// Comp time of layer frame `f`.
|
||||
pub(crate) fn comp_time_of(comp: &effectcraft_project::Comp, layer: &Layer, f: i64) -> Tick {
|
||||
layer.comp_time(comp.frame_rate.tick_of(f))
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
struct Work {
|
||||
project: Arc<Project>,
|
||||
footage: Arc<dyn FootageSource>,
|
||||
expr: Option<Arc<dyn ExprHost>>,
|
||||
cache: Arc<LayerCache>,
|
||||
comp: ItemId,
|
||||
layer: LayerId,
|
||||
effect_index: usize,
|
||||
params: Params,
|
||||
size: [f64; 2],
|
||||
fps: f64,
|
||||
frames: Vec<i64>,
|
||||
task: RotoTask,
|
||||
object_matte: bool,
|
||||
}
|
||||
|
||||
/// Renders the effect's input frames for a worker, keeping the last few. With a browser job
|
||||
/// worker's GPU (deferred readbacks), [`FrameSource::prefetch`] renders the frames a step
|
||||
/// will read in passes beforehand; [`FrameSource::get`] renders on the CPU what it lacks.
|
||||
pub(crate) struct FrameSource<'a> {
|
||||
r: Renderer<'a>,
|
||||
accel: Option<&'a dyn effectcraft_render::Accelerator>,
|
||||
/// Frames in `memo`, oldest first.
|
||||
order: std::collections::VecDeque<i64>,
|
||||
ctx0: EvalCtx<'a>,
|
||||
comp: &'a effectcraft_project::Comp,
|
||||
layer: &'a Layer,
|
||||
index: usize,
|
||||
size: [f64; 2],
|
||||
memo: HashMap<i64, Arc<Image>>,
|
||||
}
|
||||
|
||||
impl<'a> FrameSource<'a> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn new(
|
||||
project: &'a Project,
|
||||
footage: &'a dyn FootageSource,
|
||||
expr: Option<&'a dyn ExprHost>,
|
||||
cache: &'a LayerCache,
|
||||
cid: ItemId,
|
||||
comp: &'a effectcraft_project::Comp,
|
||||
layer: &'a Layer,
|
||||
index: usize,
|
||||
size: [f64; 2],
|
||||
) -> FrameSource<'a> {
|
||||
let mut r = Renderer::new(project, footage, RenderOpts { scale: 1.0, ..Default::default() });
|
||||
r.expr = expr;
|
||||
r.cache = Some(cache);
|
||||
let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) };
|
||||
FrameSource { r, accel: None, order: Default::default(), ctx0, comp, layer, index, size, memo: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Render with `accel` (deferred readbacks: see [`FrameSource::prefetch`]).
|
||||
pub(crate) fn with_accel(mut self, accel: Option<&'a dyn effectcraft_render::Accelerator>) -> Self {
|
||||
self.accel = accel;
|
||||
self.r.opts = crate::offload::analysis_opts(accel);
|
||||
self
|
||||
}
|
||||
|
||||
/// Render frames `fs` (those not kept yet) in passes on the GPU, so the step that reads
|
||||
/// them finds them ([`FrameSource::get`] can't wait for the GPU).
|
||||
pub(crate) async fn prefetch(&mut self, fs: &[i64]) {
|
||||
let Some(a) = self.accel else { return };
|
||||
for &f in fs {
|
||||
if self.memo.contains_key(&f) {
|
||||
continue;
|
||||
}
|
||||
let this = &*self;
|
||||
let img = crate::offload::in_passes(Some(a), |acc| this.render(f, &this.r.with_accel(acc))).await;
|
||||
if let Some(img) = img {
|
||||
self.keep(f, img);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn keep(&mut self, f: i64, img: Arc<Image>) {
|
||||
while self.order.len() >= 8 {
|
||||
if let Some(old) = self.order.pop_front() {
|
||||
self.memo.remove(&old);
|
||||
}
|
||||
}
|
||||
self.order.push_back(f);
|
||||
self.memo.insert(f, img);
|
||||
}
|
||||
|
||||
pub(crate) fn get(&mut self, f: i64) -> Option<Arc<Image>> {
|
||||
if let Some(i) = self.memo.get(&f) {
|
||||
return Some(i.clone());
|
||||
}
|
||||
// Not prefetched: the CPU (a pass of the GPU could only return placeholders here).
|
||||
let img = self.render(f, &self.r.with_accel(None))?;
|
||||
self.keep(f, img.clone());
|
||||
Some(img)
|
||||
}
|
||||
|
||||
fn render(&self, f: i64, r: &Renderer) -> Option<Arc<Image>> {
|
||||
let t = comp_time_of(self.comp, self.layer, f);
|
||||
let buf = r.layer_input(&self.ctx0.at(t), self.layer, self.index)?;
|
||||
let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
effectcraft_effects::Buf {
|
||||
img: effectcraft_raster::resample(&buf.img, w, h),
|
||||
offset: [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale],
|
||||
scale: 1.0,
|
||||
}
|
||||
} else {
|
||||
(*buf).clone()
|
||||
};
|
||||
Some(Arc::new(fx::layer_grid(&buf, self.size)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_work(w: Work, shared: &RotoShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
// A browser job worker's GPU (deferred readbacks): input frames render in passes.
|
||||
let accel = crate::offload::job_accel();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut src =
|
||||
FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size).with_accel(accel.as_deref());
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.task = Some(w.task);
|
||||
s.total = w.frames.len() as u64;
|
||||
s.object_matte = w.object_matte;
|
||||
}
|
||||
let cancel = || shared.cancel.load(Ordering::Relaxed);
|
||||
let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0);
|
||||
let model = fx::installed_model();
|
||||
let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) };
|
||||
let mut frozen = FrozenCache { key, ..Default::default() };
|
||||
for (k, f) in w.frames.iter().enumerate() {
|
||||
if cancel() {
|
||||
let mut s = lock(&shared.state);
|
||||
s.cancelled = true;
|
||||
s.finished = true;
|
||||
return;
|
||||
}
|
||||
// The frame and its neighbours (what a step reads), on the GPU.
|
||||
src.prefetch(&[*f, f - 1, f + 1]).await;
|
||||
let mut get = |g: i64| src.get(g);
|
||||
let ok = match (w.task, w.object_matte) {
|
||||
(RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(),
|
||||
(RotoTask::Freeze, true) => {
|
||||
let img = get(*f);
|
||||
match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) {
|
||||
Some((om::FrameMatte::Alpha(a), img)) => {
|
||||
frozen.size = [img.width as usize, img.height as usize];
|
||||
frozen.frames.insert(*f, fx::freeze_frame(&a, &[]));
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
(RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(),
|
||||
(RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
|
||||
Some((a, b)) => {
|
||||
if let Some(img) = src.get(*f) {
|
||||
frozen.size = [img.width as usize, img.height as usize];
|
||||
}
|
||||
frozen.frames.insert(*f, fx::freeze_frame(&a, &b));
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
},
|
||||
};
|
||||
if !ok && !cancel() {
|
||||
return fail(&format!("frame {f} could not be segmented"));
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64 + 1;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
if w.task == RotoTask::Freeze {
|
||||
*lock(&shared.frozen) = Some(frozen);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Frames to compute, outward from the base frame.
|
||||
pub(crate) fn job_frames(d: &effectcraft_track::roto::RotoData, dir: Direction) -> Vec<i64> {
|
||||
let Some(base) = d.base else { return vec![] };
|
||||
let mut v = vec![base];
|
||||
if dir != Direction::Backward {
|
||||
v.extend(base + 1..=d.span[1]);
|
||||
}
|
||||
if dir != Direction::Forward {
|
||||
v.extend((d.span[0]..base).rev());
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// The Roto Brush job running in a worker (Freeze or propagation), with its task.
|
||||
fn offloaded_roto(&self) -> Option<(&crate::offload::OffloadedJob, RotoTask)> {
|
||||
self.offloaded(JobKind::RotoFreeze).map(|j| (j, RotoTask::Freeze)).or_else(|| self.offloaded(JobKind::RotoPropagate).map(|j| (j, RotoTask::Propagate)))
|
||||
}
|
||||
|
||||
pub fn is_roto_running(&self) -> bool {
|
||||
self.roto_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded_roto().is_some()
|
||||
}
|
||||
|
||||
pub fn roto_progress(&self) -> Option<RotoProgress> {
|
||||
self.roto_job.as_ref().map(|j| j.progress()).or_else(|| {
|
||||
self.offloaded_roto().map(|(j, task)| {
|
||||
let p = &j.progress;
|
||||
let object_matte =
|
||||
self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte);
|
||||
RotoProgress {
|
||||
task: Some(task),
|
||||
done: p.done,
|
||||
total: p.total,
|
||||
elapsed: p.elapsed,
|
||||
fps: crate::offload::rate(p.done, p.elapsed),
|
||||
object_matte,
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// (comp, layer, effect uid, task) of the running job.
|
||||
pub fn roto_target(&self) -> Option<(ItemId, LayerId, Uid, RotoTask)> {
|
||||
self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task)))
|
||||
}
|
||||
|
||||
/// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache
|
||||
/// keys).
|
||||
pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<fx::Chain> {
|
||||
let c = self.project.comp(comp)?;
|
||||
let l = c.layer(layer)?;
|
||||
let g = l.effects()?.groups().find(|g| g.uid == effect && is_roto(g))?;
|
||||
let d = fx::data(¶ms_static(g));
|
||||
let t = comp_time_of(c, l, d.base?);
|
||||
let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
|
||||
Some(fx::Chain::new(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0))
|
||||
}
|
||||
|
||||
/// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object
|
||||
/// Matte) computes, by layer frame.
|
||||
pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<Vec<(i64, u64)>> {
|
||||
if let Some(ch) = self.roto_chain(comp, layer, effect) {
|
||||
return Some(ch.keys.into_iter().collect());
|
||||
}
|
||||
let c = self.project.comp(comp)?;
|
||||
let l = c.layer(layer)?;
|
||||
let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?;
|
||||
let params = self.om_params(comp, l, g)?;
|
||||
let frames = om::job_frames(¶ms, frame_limits(c, l));
|
||||
Some(om::frame_keys(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model()))
|
||||
}
|
||||
|
||||
/// An Object Matte's parameters at its reference frame (what its jobs segment with).
|
||||
fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option<Params> {
|
||||
let c = self.project.comp(comp)?;
|
||||
let t = comp_time_of(c, l, om::reference_frame(¶ms_static(g)));
|
||||
Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref()))
|
||||
}
|
||||
|
||||
pub fn stop_roto(&mut self) -> bool {
|
||||
if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) {
|
||||
return true;
|
||||
}
|
||||
match &self.roto_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.shared.cancel.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start propagating (or freezing) Roto Brush `effect` on `layer`. `wait`: block until done
|
||||
/// (wasm32 always does). Returns the number of frames.
|
||||
pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result<usize, String> {
|
||||
if self.is_roto_running() {
|
||||
if !wait {
|
||||
return Err("a Roto Brush job is already running".into());
|
||||
}
|
||||
self.stop_roto();
|
||||
if let Some(mut j) = self.roto_job.take()
|
||||
&& let Some(t) = j.thread.take()
|
||||
{
|
||||
let _ = t.join();
|
||||
}
|
||||
}
|
||||
self.poll_roto_job();
|
||||
self.roto_pending.retain(|x| *x != (comp, layer, effect));
|
||||
let c = self.project.comp(comp).ok_or("no composition")?;
|
||||
let l = c.layer(layer).ok_or("no layer")?;
|
||||
let fxg = l.effects().ok_or("the layer has no effects")?;
|
||||
let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
|
||||
let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?;
|
||||
let object_matte = is_object_matte(g);
|
||||
let size = layer_size(&self.project, c, l);
|
||||
let fps = c.frame_rate.as_f64();
|
||||
let (params, frames) = if object_matte {
|
||||
let params = self.om_params(comp, l, g).ok_or("no composition")?;
|
||||
let base = om::reference_frame(¶ms);
|
||||
if om::roto_data(¶ms, size, base, fx::installed_model().is_some()).is_none() {
|
||||
return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into());
|
||||
}
|
||||
let all = om::job_frames(¶ms, frame_limits(c, l));
|
||||
if all.is_empty() {
|
||||
return Err(format!("the reference frame {base} is outside the layer"));
|
||||
}
|
||||
// Forward / backward from the reference frame, as for a Roto Brush.
|
||||
let frames = all
|
||||
.into_iter()
|
||||
.filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward))
|
||||
.collect();
|
||||
(params, frames)
|
||||
} else {
|
||||
let pstat = params_static(g);
|
||||
let d = fx::data(&pstat);
|
||||
if d.is_empty() {
|
||||
return Err("draw a foreground stroke with the Roto Brush tool first".into());
|
||||
}
|
||||
let t = comp_time_of(c, l, d.base.unwrap_or(0));
|
||||
let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
|
||||
(params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }))
|
||||
};
|
||||
let n = frames.len();
|
||||
let work = Work {
|
||||
project: self.project.clone(),
|
||||
footage: self.footage.clone(),
|
||||
expr: self.expr.clone(),
|
||||
cache: self.layer_cache.clone(),
|
||||
comp,
|
||||
layer,
|
||||
effect_index: index,
|
||||
params,
|
||||
size,
|
||||
fps,
|
||||
frames,
|
||||
task,
|
||||
object_matte,
|
||||
};
|
||||
let strokes = guard_of(g);
|
||||
if !wait && self.offloads() {
|
||||
drop(work);
|
||||
self.offload_analysis(match task {
|
||||
RotoTask::Freeze => crate::offload::WorkerJob::RotoFreeze { comp, layer, effect },
|
||||
RotoTask::Propagate => crate::offload::WorkerJob::RotoPropagate { comp, layer, effect, direction: dir },
|
||||
})?;
|
||||
return Ok(n);
|
||||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let shared = Arc::new(RotoShared::default());
|
||||
{
|
||||
// Known before the worker starts (the progress shows at once).
|
||||
let mut st = lock(&shared.state);
|
||||
st.task = Some(task);
|
||||
st.total = n as u64;
|
||||
st.object_matte = object_matte;
|
||||
}
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
async move { run_work(work, &sh).await }
|
||||
});
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(
|
||||
std::thread::Builder::new()
|
||||
.name("roto".into())
|
||||
.spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh)))
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte });
|
||||
self.events.push(Event::ProjectChanged { revision: self.revision });
|
||||
self.poll_roto_job();
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Finish a done job: Freeze writes the frozen mattes (one undo step).
|
||||
fn poll_roto_job(&mut self) -> bool {
|
||||
let Some(job) = &self.roto_job else { return false };
|
||||
if !job.is_finished() {
|
||||
return false;
|
||||
}
|
||||
let Some(mut job) = self.roto_job.take() else { return false };
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let name = if job.object_matte { "Object Matte" } else { "Roto Brush" };
|
||||
if let Some(e) = &st.error {
|
||||
self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
if st.cancelled {
|
||||
self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
match job.task {
|
||||
RotoTask::Propagate => {
|
||||
let verb = if job.object_matte { "tracked" } else { "propagated" };
|
||||
self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
|
||||
// Rendered frames depend on the cache: redraw.
|
||||
self.bump();
|
||||
}
|
||||
RotoTask::Freeze => {
|
||||
let Some(fc) = lock(&job.shared.frozen).take() else { return true };
|
||||
let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone());
|
||||
let what = if job.object_matte { "prompt" } else { "strokes" };
|
||||
let r = self.edit("Freeze", None, |p, _| {
|
||||
let g = p
|
||||
.comp_mut(cid)
|
||||
.and_then(|c| c.layer_mut(lid))
|
||||
.and_then(|l| l.props.find_group_mut(uid))
|
||||
.ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?;
|
||||
if guard_of(g) != strokes {
|
||||
return Err(crate::EngineError::Other(format!("the {what} changed while freezing")));
|
||||
}
|
||||
set_str(g, FROZEN, fc.to_json());
|
||||
Ok(())
|
||||
});
|
||||
let msg = match r {
|
||||
Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
|
||||
Err(e) => format!("{name}: {e}"),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: false });
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Frontends call this every frame: finishes jobs and (with `auto`) propagates edited
|
||||
/// instances in the background.
|
||||
pub fn poll_roto(&mut self, auto: bool) -> bool {
|
||||
self.poll_models();
|
||||
let changed = self.poll_roto_job();
|
||||
if auto && self.roto_job.is_none() && self.offloaded_roto().is_none() && self.state.roto.auto_propagate {
|
||||
while let Some((c, l, e)) = self.roto_pending.first().copied() {
|
||||
self.roto_pending.remove(0);
|
||||
if self.start_roto(c, l, e, RotoTask::Propagate, Direction::Both, false).is_ok() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
/// The computed frames of an instance (full resolution), by layer frame.
|
||||
pub fn roto_computed(&self, comp: ItemId, layer: LayerId, effect: Uid) -> BTreeMap<i64, bool> {
|
||||
let mut out = BTreeMap::new();
|
||||
let Some(c) = self.project.comp(comp) else { return out };
|
||||
let Some(l) = c.layer(layer) else { return out };
|
||||
let Some(g) = l.props.find_group(effect) else { return out };
|
||||
if is_object_matte(g) {
|
||||
for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() {
|
||||
out.insert(f, fx::cached(k, 1.0).is_some());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
let ps = params_static(g);
|
||||
let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0);
|
||||
for f in ch.keys.keys() {
|
||||
out.insert(*f, ch.is_cached(*f, 1.0));
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
//! Motion tracking runtime: the analysis job (background thread with progress and cancel, or
|
||||
//! blocking for the CLI / agents / wasm), writing track results into the tracker's Track Point
|
||||
//! properties, and applying tracks (Transform, Stabilize, Corner Pin) to layers.
|
||||
//!
|
||||
//! The algorithms are in `effectcraft-track`; the property layout in
|
||||
//! `effectcraft_project::tracking`.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_geom::{Mat3, vec2};
|
||||
use effectcraft_project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions, TrackerSettings};
|
||||
use effectcraft_project::{Comp, ItemId, Keyframe, Layer, LayerId, Project, PropGroup, Uid, Value};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track as trk;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Tracker panel ▸ Analyze buttons.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
#[default]
|
||||
Forward,
|
||||
Backward,
|
||||
FrameForward,
|
||||
FrameBackward,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn from_name(s: &str) -> Option<Direction> {
|
||||
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
|
||||
Some(match n.as_str() {
|
||||
"forward" | "fwd" => Direction::Forward,
|
||||
"backward" | "back" | "bwd" => Direction::Backward,
|
||||
"frameforward" | "oneframeforward" | "1frameforward" => Direction::FrameForward,
|
||||
"framebackward" | "oneframebackward" | "1framebackward" => Direction::FrameBackward,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
pub fn forward(self) -> bool {
|
||||
matches!(self, Direction::Forward | Direction::FrameForward)
|
||||
}
|
||||
}
|
||||
|
||||
/// Track options for the algorithms.
|
||||
pub fn track_options(o: &TrackerOptions) -> trk::TrackOptions {
|
||||
trk::TrackOptions {
|
||||
channel: match o.channel {
|
||||
TrackChannel::Rgb => trk::Channel::Rgb,
|
||||
TrackChannel::Luminance => trk::Channel::Luminance,
|
||||
TrackChannel::Saturation => trk::Channel::Saturation,
|
||||
},
|
||||
blur: o.blur,
|
||||
enhance: o.enhance,
|
||||
subpixel: o.subpixel,
|
||||
adapt_every_frame: o.adapt_every_frame,
|
||||
threshold: o.threshold,
|
||||
action: match o.action {
|
||||
LowConfidence::Continue => trk::ConfidenceAction::Continue,
|
||||
LowConfidence::Stop => trk::ConfidenceAction::Stop,
|
||||
LowConfidence::Extrapolate => trk::ConfidenceAction::Extrapolate,
|
||||
LowConfidence::Adapt => trk::ConfidenceAction::Adapt,
|
||||
},
|
||||
track_shape: o.track_shape,
|
||||
}
|
||||
}
|
||||
|
||||
/// A track point's regions and attach offset read from its properties at a layer time.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PointValues {
|
||||
pub uid: Uid,
|
||||
pub spec: trk::PointSpec,
|
||||
pub attach_offset: [f64; 2],
|
||||
}
|
||||
|
||||
pub fn point_values(tp: &PropGroup, lt: Tick) -> PointValues {
|
||||
let v = |m: &str, d: [f64; 2]| tp.get(m).map(|p| p.value_at(lt).as_vec2()).unwrap_or(d);
|
||||
PointValues {
|
||||
uid: tp.uid,
|
||||
spec: trk::PointSpec {
|
||||
center: v("featureCenter", [0.0; 2]),
|
||||
feature_size: v("featureSize", [32.0; 2]),
|
||||
search_offset: v("searchOffset", [0.0; 2]),
|
||||
search_size: v("searchSize", [64.0; 2]),
|
||||
},
|
||||
attach_offset: v("attachPointOffset", [0.0; 2]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach points of a frame from the feature centres, given the start frame's centres and attach
|
||||
/// points: two-point transforms rotate and scale the offsets with the track, Parallel Corner Pin
|
||||
/// maps them through the 3-point affine, Perspective Corner Pin through the 4-point homography.
|
||||
pub fn attach_points(kind: TrackKind, rot_scale: bool, ref_c: &[[f64; 2]], ref_a: &[[f64; 2]], cur: &[[f64; 2]]) -> Vec<[f64; 2]> {
|
||||
let translate = || cur.iter().zip(ref_c.iter().zip(ref_a)).map(|(c, (r, a))| [c[0] + a[0] - r[0], c[1] + a[1] - r[1]]).collect::<Vec<_>>();
|
||||
let through = |h: Option<trk::Homography>| match h {
|
||||
Some(h) => ref_a.iter().map(|a| h.apply(*a)).collect(),
|
||||
None => translate(),
|
||||
};
|
||||
match kind {
|
||||
TrackKind::Perspective if cur.len() >= 4 => {
|
||||
let h = trk::Homography::from_points([ref_c[0], ref_c[1], ref_c[2], ref_c[3]], [cur[0], cur[1], cur[2], cur[3]]);
|
||||
through(h)
|
||||
}
|
||||
TrackKind::Affine if cur.len() >= 3 => through(trk::Homography::affine_from_points([ref_c[0], ref_c[1], ref_c[2]], [cur[0], cur[1], cur[2]])),
|
||||
_ if rot_scale && cur.len() >= 2 => {
|
||||
let Some(s) = trk::Homography::similarity_from_points([ref_c[0], ref_c[1]], [cur[0], cur[1]]) else { return translate() };
|
||||
let m = s.0;
|
||||
cur.iter()
|
||||
.zip(ref_c.iter().zip(ref_a))
|
||||
.map(|(c, (r, a))| {
|
||||
let d = [a[0] - r[0], a[1] - r[1]];
|
||||
[c[0] + m[0][0] * d[0] + m[0][1] * d[1], c[1] + m[1][0] * d[0] + m[1][1] * d[1]]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => translate(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Live analysis progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrackProgress {
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
/// Analysed frames per second.
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
/// Stopped by "If Confidence is Below … ▸ Stop Tracking".
|
||||
pub low_confidence_stop: bool,
|
||||
pub error: Option<String>,
|
||||
/// Comp time (seconds) of the last analysed frame.
|
||||
pub time: f64,
|
||||
}
|
||||
|
||||
/// One analysed frame.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FrameResult {
|
||||
pub comp_time: Tick,
|
||||
pub layer_time: Tick,
|
||||
/// Per point: (point group uid, result, attach point).
|
||||
pub points: Vec<(Uid, trk::PointResult, [f64; 2])>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TrackShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<TrackProgress>,
|
||||
pub frames: Mutex<Vec<FrameResult>>,
|
||||
}
|
||||
|
||||
pub(crate) fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) analysis.
|
||||
pub struct TrackJob {
|
||||
pub shared: Arc<TrackShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub tracker: Uid,
|
||||
pub direction: Direction,
|
||||
}
|
||||
|
||||
impl TrackJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> TrackProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
pub fn cancel(&self) {
|
||||
self.shared.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
pub(crate) struct Work {
|
||||
pub project: Arc<Project>,
|
||||
pub footage: Arc<dyn FootageSource>,
|
||||
pub expr: Option<Arc<dyn ExprHost>>,
|
||||
pub cache: Arc<LayerCache>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub settings: TrackerSettings,
|
||||
pub points: Vec<PointValues>,
|
||||
/// Comp times to analyse; the first is the start frame.
|
||||
pub times: Vec<Tick>,
|
||||
}
|
||||
|
||||
/// The layer's source frame at comp time `t` at 100 % (layer pixels + offset = image pixels).
|
||||
/// Footage at its native size is used straight from the footage source (no copy unless its
|
||||
/// Cineon Settings convert it).
|
||||
pub(crate) fn source_frame(
|
||||
r: &Renderer,
|
||||
project: &Project,
|
||||
cid: ItemId,
|
||||
comp: &Comp,
|
||||
layer: &Layer,
|
||||
t: Tick,
|
||||
expr: Option<&dyn ExprHost>,
|
||||
) -> Option<(Arc<Image>, [f64; 2])> {
|
||||
let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr, footage: None };
|
||||
if let effectcraft_project::LayerSource::Footage { item } = &layer.source
|
||||
&& let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind)
|
||||
&& f.has_video
|
||||
&& let Some(img) = r.footage.frame(*item, f, ctx.source_time(layer))
|
||||
&& img.width == f.width
|
||||
&& img.height == f.height
|
||||
{
|
||||
return Some((effectcraft_render::cineon_frame(img, f), [0.0; 2]));
|
||||
}
|
||||
let buf = r.layer_source(&ctx, layer)?;
|
||||
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
// Footage decoded at another size: bring it to layer pixels.
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
|
||||
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
|
||||
}
|
||||
Some((Arc::new(buf.img), buf.offset))
|
||||
}
|
||||
|
||||
pub(crate) async fn run_work(w: Work, shared: &TrackShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
// A browser job worker's GPU (deferred readbacks): frames render in passes.
|
||||
let accel = crate::offload::job_accel();
|
||||
let accel = accel.as_deref();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel));
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let wr = &w;
|
||||
let frame_at =
|
||||
|t: Tick| crate::offload::in_passes(accel, move |a| source_frame(&r.with_accel(a), &wr.project, wr.comp, comp, layer, t, wr.expr.as_deref()));
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.total = w.times.len().saturating_sub(1) as u64;
|
||||
}
|
||||
let Some(first) = frame_at(w.times[0]).await else { return fail("the layer has no pixels to track") };
|
||||
let specs: Vec<trk::PointSpec> = w.points.iter().map(|p| p.spec).collect();
|
||||
let mut tracker = trk::Tracker::new(track_options(&w.settings.options), &specs, &trk::Frame { img: &first.0, offset: first.1 });
|
||||
let ref_c: Vec<[f64; 2]> = specs.iter().map(|s| s.center).collect();
|
||||
let ref_a: Vec<[f64; 2]> = w.points.iter().map(|p| [p.spec.center[0] + p.attach_offset[0], p.spec.center[1] + p.attach_offset[1]]).collect();
|
||||
let rot_scale = w.settings.rotation || w.settings.scale;
|
||||
// The start frame keeps its regions and gets keys too.
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[0],
|
||||
layer_time: layer.layer_time(w.times[0]),
|
||||
points: w
|
||||
.points
|
||||
.iter()
|
||||
.zip(&ref_a)
|
||||
.map(|(p, a)| (p.uid, trk::PointResult { center: p.spec.center, confidence: 100.0, rotation: 0.0, scale: 1.0, status: trk::Status::Tracked }, *a))
|
||||
.collect(),
|
||||
});
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]).await } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
// Track this frame while the next one renders.
|
||||
let (res, nf) = crate::offload::join_fetch(
|
||||
accel.is_some(),
|
||||
|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }),
|
||||
|| async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } },
|
||||
)
|
||||
.await;
|
||||
next = nf;
|
||||
if res.iter().any(|p| p.status == trk::Status::Stopped) {
|
||||
lock(&shared.state).low_confidence_stop = true;
|
||||
break;
|
||||
}
|
||||
let centers: Vec<[f64; 2]> = res.iter().map(|p| p.center).collect();
|
||||
let attach = attach_points(w.settings.kind, rot_scale, &ref_c, &ref_a, ¢ers);
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[k],
|
||||
layer_time: layer.layer_time(w.times[k]),
|
||||
points: w.points.iter().zip(res).zip(attach).map(|((p, r), a)| (p.uid, r, a)).collect(),
|
||||
});
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Set (or add) a key at layer time `t`.
|
||||
pub(crate) fn key(g: &mut PropGroup, m: &str, t: Tick, v: Value) {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
effectcraft_keyframe::set_key(&mut p.keys, Keyframe::new(t, v));
|
||||
}
|
||||
}
|
||||
|
||||
/// Write analysed frames into the tracker's points.
|
||||
fn write_frames(p: &mut Project, job: &TrackJob, frames: &[FrameResult]) {
|
||||
let Some(comp) = p.comp_mut(job.comp) else { return };
|
||||
let Some(layer) = comp.layer_mut(job.layer) else { return };
|
||||
for f in frames {
|
||||
for (uid, r, a) in &f.points {
|
||||
let Some(tp) = layer.props.find_group_mut(*uid) else { continue };
|
||||
key(tp, "featureCenter", f.layer_time, Value::Vec2(r.center));
|
||||
key(tp, "confidence", f.layer_time, Value::Scalar(r.confidence));
|
||||
key(tp, "attachPoint", f.layer_time, Value::Vec2(*a));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Whether a track analysis is running.
|
||||
pub fn is_tracking(&self) -> bool {
|
||||
self.track_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded(crate::offload::JobKind::Track).is_some()
|
||||
}
|
||||
|
||||
/// Live progress of the running (or just finished) analysis.
|
||||
pub fn track_progress(&self) -> Option<TrackProgress> {
|
||||
self.track_job.as_ref().map(|j| j.progress()).or_else(|| self.offloaded(crate::offload::JobKind::Track).map(|j| j.progress.track()))
|
||||
}
|
||||
|
||||
/// Cancel the running analysis (results so far are kept).
|
||||
pub fn stop_track(&mut self) -> bool {
|
||||
if self.cancel_offloaded(crate::offload::JobKind::Track) {
|
||||
return true;
|
||||
}
|
||||
match &self.track_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.cancel();
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start analysing. `wait`: block until done (wasm32 always does). One undo step covers the
|
||||
/// whole analysis.
|
||||
pub(crate) fn start_track(&mut self, work: Work, tracker: Uid, direction: Direction, wait: bool) -> Result<(), String> {
|
||||
if self.is_tracking() {
|
||||
return Err("a track analysis is already running".into());
|
||||
}
|
||||
self.poll_track();
|
||||
if !wait && self.offloads() {
|
||||
let Work { comp, layer, settings, points, times, .. } = work;
|
||||
return self.offload_analysis(crate::offload::WorkerJob::Track { comp, layer, tracker, direction, settings, points, times });
|
||||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let levels = self.prefs.general.undo_levels as usize;
|
||||
self.history.record("Analyze Track", self.project.clone(), levels);
|
||||
let shared = Arc::new(TrackShared::default());
|
||||
let (comp, layer) = (work.comp, work.layer);
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
async move { run_work(work, &sh).await }
|
||||
});
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(
|
||||
std::thread::Builder::new()
|
||||
.name("track-analyze".into())
|
||||
.spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh)))
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.track_job = Some(TrackJob { shared, thread, comp, layer, tracker, direction });
|
||||
self.poll_track();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write new results into the project and move the CTI to the last analysed frame; drop the
|
||||
/// job once it finished. Frontends call this every frame while tracking.
|
||||
pub fn poll_track(&mut self) -> bool {
|
||||
let Some(job) = &self.track_job else { return false };
|
||||
let frames = std::mem::take(&mut *lock(&job.shared.frames));
|
||||
let finished = job.is_finished();
|
||||
let changed = !frames.is_empty();
|
||||
if changed {
|
||||
write_frames(Arc::make_mut(&mut self.project), job, &frames);
|
||||
if self.state.active_comp == Some(job.comp)
|
||||
&& let Some(last) = frames.last()
|
||||
{
|
||||
self.state.times.insert(job.comp, last.comp_time);
|
||||
}
|
||||
self.bump();
|
||||
}
|
||||
if finished && let Some(mut job) = self.track_job.take() {
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let msg = match (&st.error, st.cancelled, st.low_confidence_stop) {
|
||||
(Some(e), _, _) => format!("Tracking stopped: {e}"),
|
||||
(None, true, _) => format!("Tracking stopped after {} frame(s)", st.done),
|
||||
(None, false, true) => format!("Tracking stopped: confidence below threshold after {} frame(s)", st.done),
|
||||
_ => format!("Tracked {} frame(s) in {:.1} s", st.done, st.elapsed),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
}
|
||||
changed || finished
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- apply
|
||||
|
||||
/// Apply Dimensions.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Dims {
|
||||
#[default]
|
||||
XY,
|
||||
X,
|
||||
Y,
|
||||
}
|
||||
|
||||
impl Dims {
|
||||
pub fn from_name(s: &str) -> Option<Dims> {
|
||||
match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() {
|
||||
"xy" | "xandy" | "both" => Some(Dims::XY),
|
||||
"x" | "xonly" => Some(Dims::X),
|
||||
"y" | "yonly" => Some(Dims::Y),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn mix(self, old: [f64; 3], new: [f64; 2]) -> [f64; 3] {
|
||||
match self {
|
||||
Dims::XY => [new[0], new[1], old[2]],
|
||||
Dims::X => [new[0], old[1], old[2]],
|
||||
Dims::Y => [old[0], new[1], old[2]],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_m(m: &Mat3, p: [f64; 2]) -> [f64; 2] {
|
||||
let q = m.apply(vec2(p[0], p[1]));
|
||||
[q.x, q.y]
|
||||
}
|
||||
|
||||
/// Parent space of a layer (comp space when unparented), as a 2D matrix from comp space.
|
||||
fn comp_to_parent(ctx: &EvalCtx, layer: &Layer) -> Mat3 {
|
||||
match layer.parent.and_then(|p| ctx.comp.layer(p)) {
|
||||
Some(p) => ctx.layer_to_comp(p).0.inverse().unwrap_or(Mat3::IDENTITY),
|
||||
None => Mat3::IDENTITY,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per analysed time: (source layer time, attach points in source layer space).
|
||||
pub fn attach_track(tracker: &PropGroup) -> Vec<(Tick, Vec<[f64; 2]>)> {
|
||||
let pts: Vec<&PropGroup> = tracker.track_points().collect();
|
||||
let Some(first) = pts.first().and_then(|g| g.get("attachPoint")) else { return vec![] };
|
||||
first.keys.iter().map(|k| (k.time, pts.iter().map(|g| g.get("attachPoint").map(|p| p.value_at(k.time).as_vec2()).unwrap_or([0.0; 2])).collect())).collect()
|
||||
}
|
||||
|
||||
/// Apply a tracker (Tracker panel ▸ Apply). Returns the number of keyframed frames.
|
||||
pub fn apply(p: &mut Project, cid: ItemId, src: LayerId, tracker: Uid, dims: Dims) -> Result<usize, String> {
|
||||
let comp = p.comp(cid).ok_or("no composition")?.clone();
|
||||
let source = comp.layer(src).ok_or("no source layer")?.clone();
|
||||
let (tg, settings) = source.tracker(tracker).ok_or("no such tracker")?;
|
||||
let settings = settings.clone();
|
||||
let track = attach_track(tg);
|
||||
if track.is_empty() {
|
||||
return Err("analyze the track first".into());
|
||||
}
|
||||
let target_id = match settings.kind {
|
||||
TrackKind::Stabilize => src,
|
||||
TrackKind::Raw => return Err("a Raw track has nothing to apply".into()),
|
||||
_ => settings.target.ok_or("set a Motion Target first (Edit Target…)")?,
|
||||
};
|
||||
let target = comp.layer(target_id).ok_or("the motion target layer is gone")?.clone();
|
||||
let ctx0 = EvalCtx::new(p, cid, &comp, Tick::ZERO);
|
||||
let t_first = source.comp_time(track[0].0);
|
||||
let rot_scale = settings.point_count() >= 2 && track[0].1.len() >= 2;
|
||||
let (ang0, len0) = if rot_scale { effectcraft_track::angle_and_length(track[0].1[0], track[0].1[1]) } else { (0.0, 1.0) };
|
||||
// (target layer time, values) to write.
|
||||
let mut pos_keys = vec![];
|
||||
let mut anchor_keys = vec![];
|
||||
let mut rot_keys = vec![];
|
||||
let mut scale_keys = vec![];
|
||||
let mut corner_keys = vec![];
|
||||
let ttr = target.transform().ok_or("the target has no transform")?;
|
||||
let c0 = ctx0.at(t_first);
|
||||
let rot0 = c0.f(&target, ttr, "rotation", 0.0);
|
||||
let scale0 = c0.v3(&target, ttr, "scale", [100.0; 3]);
|
||||
let mut prev_ang = 0.0;
|
||||
// Stabilize: the feature stays where it was on the first tracked frame.
|
||||
let stab_pos = {
|
||||
let (m, _) = c0.layer_to_comp(&source);
|
||||
apply_m(&comp_to_parent(&c0, &source), apply_m(&m, track[0].1[0]))
|
||||
};
|
||||
for (lt, pts) in &track {
|
||||
let ct = source.comp_time(*lt);
|
||||
let ctx = ctx0.at(ct);
|
||||
let tt = target.layer_time(ct);
|
||||
let (ang, len) = if rot_scale { effectcraft_track::angle_and_length(pts[0], pts[1]) } else { (ang0, len0) };
|
||||
let dang = effectcraft_track::unwrap_degrees(prev_ang, ang - ang0);
|
||||
prev_ang = dang;
|
||||
let ratio = if len0 > 1e-9 { len / len0 } else { 1.0 };
|
||||
match settings.kind {
|
||||
TrackKind::Stabilize => {
|
||||
if settings.position {
|
||||
let old = ctx.v3(&target, ttr, "anchor", [0.0; 3]);
|
||||
anchor_keys.push((tt, dims.mix(old, pts[0])));
|
||||
let oldp = ctx.position(&target, ttr);
|
||||
pos_keys.push((tt, dims.mix(oldp, stab_pos)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 - dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] / ratio, scale0[1] / ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Transform => {
|
||||
let (m, _) = ctx.layer_to_comp(&source);
|
||||
if settings.position {
|
||||
let c = apply_m(&m, pts[0]);
|
||||
let pp = apply_m(&comp_to_parent(&ctx, &target), c);
|
||||
pos_keys.push((tt, dims.mix(ctx.position(&target, ttr), pp)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 + dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] * ratio, scale0[1] * ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Affine | TrackKind::Perspective => {
|
||||
let (ms, _) = ctx.layer_to_comp(&source);
|
||||
// Corner Pin's corners are effect points: in the target's effect space.
|
||||
let inv = ctx.effect_to_comp(&target).inverse().ok_or("the target layer's transform is not invertible")?;
|
||||
let n = pts.len();
|
||||
if n < 3 {
|
||||
return Err("corner pin tracks need 3 or 4 track points".into());
|
||||
}
|
||||
let lr =
|
||||
if settings.kind == TrackKind::Affine || n < 4 { [pts[1][0] + pts[2][0] - pts[0][0], pts[1][1] + pts[2][1] - pts[0][1]] } else { pts[3] };
|
||||
// Track Points 1–4: upper left, upper right, lower left, lower right.
|
||||
let map = |q: [f64; 2]| apply_m(&inv, apply_m(&ms, q));
|
||||
corner_keys.push((tt, [map(pts[0]), map(pts[1]), map(pts[2]), map(lr)]));
|
||||
}
|
||||
TrackKind::Raw => {}
|
||||
}
|
||||
}
|
||||
let n = track.len();
|
||||
let size = effectcraft_render::effect_bounds(p, &comp, &target).0;
|
||||
let mut ids_next = p.next_id;
|
||||
let layer = p.comp_mut(cid).and_then(|c| c.layer_mut(target_id)).ok_or("no target layer")?;
|
||||
if !corner_keys.is_empty() {
|
||||
let spec = effectcraft_effects::find("ec.distort.cornerpin").ok_or("Corner Pin is not available")?;
|
||||
let fx = layer.props.sub_mut("effects").ok_or("the target layer can't have effects")?;
|
||||
let same = fx.groups().filter(|g| g.match_id == spec.id).count();
|
||||
let name = if same == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, same + 1) };
|
||||
let mut g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut ids_next), &name, size);
|
||||
for (t, c) in &corner_keys {
|
||||
for (m, v) in ["ul", "ur", "ll", "lr"].iter().zip(c) {
|
||||
key(&mut g, m, *t, Value::Vec2(*v));
|
||||
}
|
||||
}
|
||||
fx.children.push(g.into());
|
||||
}
|
||||
let tr = layer.transform_mut().ok_or("no transform")?;
|
||||
for (t, v) in anchor_keys {
|
||||
key(tr, "anchor", t, Value::Vec3(v));
|
||||
}
|
||||
let separated = tr.get("positionX").is_some();
|
||||
for (t, v) in pos_keys {
|
||||
if separated {
|
||||
key(tr, "positionX", t, Value::Scalar(v[0]));
|
||||
key(tr, "positionY", t, Value::Scalar(v[1]));
|
||||
} else {
|
||||
key(tr, "position", t, Value::Vec3(v));
|
||||
}
|
||||
}
|
||||
for (t, v) in rot_keys {
|
||||
key(tr, "rotation", t, Value::Scalar(v));
|
||||
}
|
||||
for (t, v) in scale_keys {
|
||||
key(tr, "scale", t, Value::Vec3(v));
|
||||
}
|
||||
p.next_id = ids_next;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attach_points_follow_the_track() {
|
||||
let rc = [[0.0, 0.0], [10.0, 0.0]];
|
||||
let ra = [[0.0, 5.0], [10.0, 5.0]];
|
||||
// Rotated 90° and doubled.
|
||||
let cur = [[100.0, 100.0], [100.0, 120.0]];
|
||||
let a = attach_points(TrackKind::Transform, true, &rc, &ra, &cur);
|
||||
assert!((a[0][0] - 90.0).abs() < 1e-9 && (a[0][1] - 100.0).abs() < 1e-9, "{a:?}");
|
||||
let a = attach_points(TrackKind::Transform, false, &rc, &ra, &cur);
|
||||
assert_eq!(a[0], [100.0, 105.0]);
|
||||
let h = trk::Homography([[1.1, 0.1, 5.0], [0.05, 0.9, -3.0], [0.001, 0.0005, 1.0]]);
|
||||
let rc4 = [[0.0, 0.0], [100.0, 0.0], [0.0, 100.0], [100.0, 100.0]];
|
||||
let ra4 = [[10.0, 10.0], [90.0, 10.0], [10.0, 90.0], [90.0, 90.0]];
|
||||
let cur4: Vec<[f64; 2]> = rc4.iter().map(|p| h.apply(*p)).collect();
|
||||
let a = attach_points(TrackKind::Perspective, false, &rc4, &ra4, &cur4);
|
||||
for (got, want) in a.iter().zip(ra4.iter().map(|p| h.apply(*p))) {
|
||||
assert!((got[0] - want[0]).abs() < 1e-6 && (got[1] - want[1]).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
//! Warp Stabilizer runtime: the analysis job (`warp.analyze`) and keeping analyses valid.
|
||||
//!
|
||||
//! The analysis renders the layer's input to the effect (source → masks → the effects above
|
||||
//! the Warp Stabilizer, `Renderer::layer_input`) for every frame between the layer's In and Out
|
||||
//! points on a background thread, and feeds the frames to `effectcraft_track::stabilize`'s
|
||||
//! analyzer (step 1 of 2, "Analyzing in background"); step 2 ("Stabilizing") derives the
|
||||
//! stabilization plan once to validate the result. When the job finishes, the analysis is
|
||||
//! written into the effect's hidden Analysis parameter as one undo step, together with a key of
|
||||
//! what it was made from (the layer's source, In/Out, start, stretch and Time Remap). Edits that
|
||||
//! change those clear the analysis (`invalidate`, run after every edit) and queue the effect
|
||||
//! for re-analysis, which frontends start with [`Session::poll_warp`] (After Effects re-analyses
|
||||
//! automatically too).
|
||||
|
||||
use crate::offload::{JobKind, rate};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_effects::warp_stab::{self, ANALYSIS, ANALYSIS_KEY};
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, LayerSource, Project, PropGroup, Uid};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track::stabilize::{AnalyzeOpts, Analyzer, StabSettings, WarpAnalysis};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::tracking::lock;
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Live analysis progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct WarpProgress {
|
||||
/// 1 = analyzing, 2 = stabilizing.
|
||||
pub step: u8,
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl WarpProgress {
|
||||
/// The viewer banner text, as After Effects words it.
|
||||
pub fn banner(&self) -> String {
|
||||
match self.step {
|
||||
2 => "Stabilizing...".into(),
|
||||
_ => {
|
||||
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
|
||||
format!("Analyzing in background (step 1 of 2): {pc}%")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct WarpShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<WarpProgress>,
|
||||
result: Mutex<Option<WarpAnalysis>>,
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) Warp Stabilizer analysis.
|
||||
pub struct WarpJob {
|
||||
pub shared: Arc<WarpShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub effect: Uid,
|
||||
/// The layer signature the analysis is made from.
|
||||
key: String,
|
||||
}
|
||||
|
||||
impl WarpJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> WarpProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
pub(crate) struct WarpWork {
|
||||
pub project: Arc<Project>,
|
||||
pub footage: Arc<dyn FootageSource>,
|
||||
pub expr: Option<Arc<dyn ExprHost>>,
|
||||
pub cache: Arc<LayerCache>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
/// Position of the effect in the layer's stack (the effects before it are analysed).
|
||||
pub effect_index: usize,
|
||||
pub size: [f64; 2],
|
||||
pub detailed: bool,
|
||||
/// Comp times of the analysed frames.
|
||||
pub times: Vec<Tick>,
|
||||
}
|
||||
|
||||
fn fnv(s: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
|
||||
for b in s.as_bytes() {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// What an analysis of `layer` depends on (its frames): source (and the footage item's
|
||||
/// interpretation), In/Out, start time, stretch, Time Remap and the comp frame rate.
|
||||
pub fn signature(p: &Project, comp: ItemId, layer: &Layer) -> String {
|
||||
let src = match &layer.source {
|
||||
LayerSource::Footage { item } | LayerSource::Solid { item } | LayerSource::Comp { item } => match p.item(*item).map(|i| &i.kind) {
|
||||
Some(ItemKind::Footage(f)) => format!("{f:?}"),
|
||||
Some(ItemKind::Solid(s)) => format!("{s:?}"),
|
||||
Some(ItemKind::Comp(c)) => format!("{}x{}@{:?}/{:?}", c.width, c.height, c.frame_rate, c.duration),
|
||||
_ => String::new(),
|
||||
},
|
||||
_ => String::new(),
|
||||
};
|
||||
let rate = p.comp(comp).map(|c| format!("{:?}", c.frame_rate)).unwrap_or_default();
|
||||
let s = format!(
|
||||
"{:?}|{src}|{:?}|{:?}|{:?}|{}|{:?}|{rate}",
|
||||
layer.source,
|
||||
layer.in_point,
|
||||
layer.out_point,
|
||||
layer.start_time,
|
||||
layer.stretch,
|
||||
layer.props.get("timeRemap")
|
||||
);
|
||||
format!("{:016x}", fnv(&s))
|
||||
}
|
||||
|
||||
fn is_warp(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == warp_stab::ID)
|
||||
}
|
||||
|
||||
fn str_of(g: &PropGroup, m: &str) -> String {
|
||||
match g.get(m).map(|p| &p.value) {
|
||||
Some(Value::Str(s)) => s.clone(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear analyses that no longer match their layer, after an edit. Returns the Warp
|
||||
/// Stabilizers that need (re-)analysis: cleared ones and newly applied, unanalysed ones.
|
||||
pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
|
||||
let mut todo = vec![];
|
||||
let mut clear = vec![];
|
||||
for (cid, it) in &after.items {
|
||||
let ItemKind::Comp(c) = &it.kind else { continue };
|
||||
for l in &c.layers {
|
||||
let Some(fx) = l.effects() else { continue };
|
||||
if !fx.groups().any(is_warp) {
|
||||
continue;
|
||||
}
|
||||
let sig = signature(after, *cid, l);
|
||||
let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id));
|
||||
for g in fx.groups().filter(|g| is_warp(g)) {
|
||||
let analysed = !str_of(g, ANALYSIS).is_empty();
|
||||
if analysed && str_of(g, ANALYSIS_KEY) != sig {
|
||||
clear.push((*cid, l.id, g.uid));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if !analysed && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() {
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (cid, lid, uid) in clear {
|
||||
if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) {
|
||||
for m in [ANALYSIS, ANALYSIS_KEY] {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
p.value = Value::Str(String::new());
|
||||
p.keys.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
todo
|
||||
}
|
||||
|
||||
/// The layer's input to effect `effect_index` at comp time `t`, in layer pixels at 100 %.
|
||||
fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc<Image>, [f64; 2])> {
|
||||
let buf = r.layer_input(ctx, layer, effect_index)?;
|
||||
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
|
||||
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
|
||||
}
|
||||
Some((Arc::new(buf.img.clone()), buf.offset))
|
||||
}
|
||||
|
||||
async fn run_work(w: WarpWork, shared: &WarpShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
// A browser job worker's GPU (deferred readbacks): frames render in passes.
|
||||
let accel = crate::offload::job_accel();
|
||||
let accel = accel.as_deref();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), crate::offload::analysis_opts(accel));
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO);
|
||||
let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 };
|
||||
let ctx0 = &ctx0;
|
||||
let frame_at = |t: Tick| crate::offload::in_passes(accel, move |a| input_frame(&r.with_accel(a), &ctx0.at(t), layer, w.effect_index));
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.step = 1;
|
||||
s.total = w.times.len() as u64;
|
||||
}
|
||||
let mut an = Analyzer::new(w.size, AnalyzeOpts { detailed: w.detailed });
|
||||
let mut next = frame_at(w.times[0]).await;
|
||||
for k in 0..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
let mut s = lock(&shared.state);
|
||||
s.cancelled = true;
|
||||
s.finished = true;
|
||||
return;
|
||||
}
|
||||
let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) };
|
||||
let ((), nf) = crate::offload::join_fetch(
|
||||
accel.is_some(),
|
||||
|| an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }),
|
||||
|| async { if k + 1 < w.times.len() { frame_at(w.times[k + 1]).await } else { None } },
|
||||
)
|
||||
.await;
|
||||
next = nf;
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64 + 1;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
lock(&shared.state).step = 2;
|
||||
let lt0 = layer.layer_time(w.times[0]).seconds();
|
||||
let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() };
|
||||
let a = an.finish(lt0, fd);
|
||||
// Step 2: derive a plan once (validates the motion and warms nothing else up).
|
||||
let plan = effectcraft_track::stabilize::plan(&a, &StabSettings { fps: if fd > 0.0 { 1.0 / fd } else { 30.0 }, ..Default::default() });
|
||||
if plan.warps.len() != a.frames.len() || plan.warps.iter().any(|h| h.0.iter().flatten().any(|v| !v.is_finite())) {
|
||||
return fail("the stabilization could not be computed");
|
||||
}
|
||||
*lock(&shared.result) = Some(a);
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Frames to analyse: every comp frame from the layer's In to its Out point (inside the comp).
|
||||
pub(crate) fn analysis_times(comp: &effectcraft_project::Comp, layer: &Layer) -> Vec<Tick> {
|
||||
let fd = comp.frame_duration();
|
||||
let lo = comp.frame_rate.snap_nearest(layer.in_point.max(Tick::ZERO));
|
||||
let hi = layer.out_point.min(comp.duration);
|
||||
let mut t = lo;
|
||||
let mut v = vec![];
|
||||
while t < hi {
|
||||
v.push(t);
|
||||
t += fd;
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
impl Session {
|
||||
pub fn is_warp_analyzing(&self) -> bool {
|
||||
self.warp_job.as_ref().is_some_and(|j| !j.is_finished()) || self.offloaded(JobKind::Warp).is_some()
|
||||
}
|
||||
|
||||
/// Progress of the running (or just finished) analysis.
|
||||
pub fn warp_progress(&self) -> Option<WarpProgress> {
|
||||
self.warp_job.as_ref().map(|j| j.progress()).or_else(|| {
|
||||
self.offloaded(JobKind::Warp).map(|j| {
|
||||
let p = &j.progress;
|
||||
WarpProgress { step: 1, done: p.done, total: p.total, elapsed: p.elapsed, fps: rate(p.done, p.elapsed), ..Default::default() }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// The effect being analysed: (comp, layer, effect uid).
|
||||
pub fn warp_target(&self) -> Option<(ItemId, LayerId, Uid)> {
|
||||
self.warp_job.as_ref().map(|j| (j.comp, j.layer, j.effect)).or_else(|| self.offloaded(JobKind::Warp).map(|j| (j.comp, j.layer, j.uid)))
|
||||
}
|
||||
|
||||
/// Cancel the running analysis (nothing is written).
|
||||
pub fn stop_warp(&mut self) -> bool {
|
||||
if self.cancel_offloaded(JobKind::Warp) {
|
||||
return true;
|
||||
}
|
||||
match &self.warp_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.shared.cancel.store(true, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start analysing Warp Stabilizer `effect` on `layer`. `wait`: block until done (wasm32
|
||||
/// always does). Returns the number of frames.
|
||||
pub fn start_warp(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result<usize, String> {
|
||||
if self.is_warp_analyzing() {
|
||||
return Err("a Warp Stabilizer analysis is already running".into());
|
||||
}
|
||||
self.poll_warp_job();
|
||||
self.warp_pending.retain(|x| *x != (comp, layer, effect));
|
||||
let c = self.project.comp(comp).ok_or("no composition")?;
|
||||
let l = c.layer(layer).ok_or("no layer")?;
|
||||
let fx = l.effects().ok_or("the layer has no effects")?;
|
||||
let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
|
||||
let g = fx.groups().nth(index).filter(|g| is_warp(g)).ok_or("not a Warp Stabilizer")?;
|
||||
if l.switches.adjustment {
|
||||
return Err("Warp Stabilizer needs footage, not an adjustment layer".into());
|
||||
}
|
||||
let times = analysis_times(c, l);
|
||||
if times.len() < 2 {
|
||||
return Err("the layer must be at least two frames long".into());
|
||||
}
|
||||
let size = effectcraft_render::effect_bounds(&self.project, c, l).0;
|
||||
let detailed = g.prop("advanced/detailedAnalysis").is_some_and(|p| p.value.as_bool());
|
||||
let n = times.len();
|
||||
let work = WarpWork {
|
||||
project: self.project.clone(),
|
||||
footage: self.footage.clone(),
|
||||
expr: self.expr.clone(),
|
||||
cache: self.layer_cache.clone(),
|
||||
comp,
|
||||
layer,
|
||||
effect_index: index,
|
||||
size,
|
||||
detailed,
|
||||
times,
|
||||
};
|
||||
let key = signature(&self.project, comp, l);
|
||||
if !wait && self.offloads() {
|
||||
drop(work);
|
||||
self.offload_analysis(crate::offload::WorkerJob::Warp { comp, layer, effect })?;
|
||||
return Ok(n);
|
||||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let shared = Arc::new(WarpShared::default());
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
async move { run_work(work, &sh).await }
|
||||
});
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(
|
||||
std::thread::Builder::new()
|
||||
.name("warp-analyze".into())
|
||||
.spawn(move || effectcraft_render::passes::block_on(run_work(work, &sh)))
|
||||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.warp_job = Some(WarpJob { shared, thread, comp, layer, effect, key });
|
||||
self.events.push(Event::ProjectChanged { revision: self.revision });
|
||||
self.poll_warp_job();
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Write a finished analysis into its effect (one undo step) and drop the job.
|
||||
fn poll_warp_job(&mut self) -> bool {
|
||||
let Some(job) = &self.warp_job else { return false };
|
||||
if !job.is_finished() {
|
||||
return false;
|
||||
}
|
||||
let Some(mut job) = self.warp_job.take() else { return false };
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let result = lock(&job.shared.result).take();
|
||||
match (result, &st.error) {
|
||||
(Some(a), None) if !st.cancelled => {
|
||||
let json = a.to_json();
|
||||
let (cid, lid, uid, key) = (job.comp, job.layer, job.effect, job.key.clone());
|
||||
let r = self.edit("Warp Stabilizer Analysis", None, |p, _| {
|
||||
let g = p
|
||||
.comp_mut(cid)
|
||||
.and_then(|c| c.layer_mut(lid))
|
||||
.and_then(|l| l.props.find_group_mut(uid))
|
||||
.ok_or_else(|| crate::EngineError::Other("the Warp Stabilizer was removed during analysis".into()))?;
|
||||
for (m, v) in [(ANALYSIS, json), (ANALYSIS_KEY, key)] {
|
||||
if let Some(pr) = g.get_mut(m) {
|
||||
pr.value = Value::Str(v);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
let msg = match r {
|
||||
Ok(()) => format!("Warp Stabilizer: analysed {} frame(s) in {:.1} s", a.frames.len(), st.elapsed),
|
||||
Err(e) => format!("Warp Stabilizer: {e}"),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: false });
|
||||
}
|
||||
_ => {
|
||||
let msg = match &st.error {
|
||||
Some(e) => format!("Warp Stabilizer analysis failed: {e}"),
|
||||
None => "Warp Stabilizer analysis cancelled".into(),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Frontends call this every frame: finishes analyses and (with `auto`) starts queued
|
||||
/// re-analyses in the background, like After Effects.
|
||||
pub fn poll_warp(&mut self, auto: bool) -> bool {
|
||||
let changed = self.poll_warp_job();
|
||||
if auto && self.warp_job.is_none() && self.offloaded(JobKind::Warp).is_none() {
|
||||
while let Some((c, l, e)) = self.warp_pending.first().copied() {
|
||||
self.warp_pending.remove(0);
|
||||
if self.start_warp(c, l, e, false).is_ok() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user