mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-02 06:27:11 -06:00
S2.41: Roto Brush follows the object with SAM 2
When the chosen model has a tracker, roto.propagate runs one tracking pass over the span: prompts from the strokes (a handful of points, which SAM 2 trusts), the clip tracked both ways, each frame's probability map refined by the same graph cut with the strokes as hard constraints. Results are cached under their own keys with a per-frame confidence and an occluded flag; roto.status reports them (source: tracker|classical). Without a tracker nothing changes. Real weights: IoU 0.98 on all 21 frames, confidence 0.99. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
a62d59da4b
commit
c1f5ff9f7d
@@ -188,6 +188,11 @@ impl Session {
|
||||
});
|
||||
}
|
||||
|
||||
/// The Roto Brush model in use (`None`: the classic engine).
|
||||
pub fn models_roto_active(&self) -> Option<String> {
|
||||
self.models.status(Task::Mask).active
|
||||
}
|
||||
|
||||
/// Finish what the model workers did: a toast, and loading a model that just arrived.
|
||||
pub fn poll_models(&mut self) {
|
||||
for task in [Task::Mask, Task::Face] {
|
||||
|
||||
@@ -112,6 +112,10 @@ pub struct RotoProgress {
|
||||
pub error: Option<String>,
|
||||
/// The job works on an Object Matte (else a Roto Brush & Refine Edge).
|
||||
pub object_matte: bool,
|
||||
/// What the model's tracker found, when one followed the object (SAM 2).
|
||||
pub tracked: Option<fx::TrackSummary>,
|
||||
/// Why the tracker was not used, when a tracking model was chosen but the classic engine ran.
|
||||
pub tracker_note: Option<String>,
|
||||
}
|
||||
|
||||
impl RotoProgress {
|
||||
@@ -436,6 +440,31 @@ async fn run_work(w: Work, shared: &RotoShared) {
|
||||
let model = fx::installed_model();
|
||||
let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) };
|
||||
let mut frozen = FrozenCache { key, ..Default::default() };
|
||||
// A tracking model (SAM 2) follows the object through the whole span in one pass; the loop
|
||||
// below then finds every frame cached. Without one (or if it fails) nothing changes: the
|
||||
// classic engine runs frame by frame.
|
||||
if w.task == RotoTask::Propagate && !w.object_matte && chain.has_tracker() {
|
||||
let mut get = |g: i64| src.get(g);
|
||||
let mut report = |n: usize| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = n as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 };
|
||||
crate::offload::report(s.done, s.total);
|
||||
};
|
||||
let result = chain.track(&mut get, &cancel, &mut report);
|
||||
if cancel() {
|
||||
let mut s = lock(&shared.state);
|
||||
s.cancelled = true;
|
||||
s.finished = true;
|
||||
return;
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
match result {
|
||||
Ok(summary) => s.tracked = summary,
|
||||
Err(e) => s.tracker_note = Some(format!("tracker unavailable, classic engine used: {e}")),
|
||||
}
|
||||
}
|
||||
for (k, f) in w.frames.iter().enumerate() {
|
||||
if cancel() {
|
||||
let mut s = lock(&shared.state);
|
||||
|
||||
Reference in New Issue
Block a user