aleleba/effectcraft
1
0
Fork 0
mirror of https://github.com/storytold/effectcraft.git synced 2026-11-10 02:11:41 -06:00
Code Issues Packages Projects Releases Wiki Activity

S2.40: SAM 2 as the Roto Brush object tracker

Browse Source 
ObjectTracker interface in the segment crate (prompts on some frames, the
clip tracked both ways, each frame streamed with confidence and an occluded
flag), implemented by SAM 2. MaskModel::tracker() is None for other models.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kristian Myram
2026-10-30 08:43:30 +01:00
co-authored by Claude Sonnet 5.5
parent 92ac189a49
commit a62d59da4b
6 changed files with 249 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/sam2/src/session.rs
+5 -5
View File
@@ -33,7 +33,7 @@ const MAX_FRAMES: usize = 1_000_000;
const GRID: usize = 64;
/// Frames of the clip, already resized to the model input and normalised: `[1, 3, 1024, 1024]`.
pub trait FrameSource: Send {
pub trait FrameSource {
fn num_frames(&self) -> usize;
fn frame(&self, index: usize) -> Result<Tensor>;
}
@@ -129,9 +129,9 @@ struct FrameFeatures {
high: (Tensor, Tensor),
}
pub struct VideoTracker {
pub struct VideoTracker<'a> {
model: Arc<VideoModel>,
frames: Box<dyn FrameSource>,
frames: Box<dyn FrameSource + 'a>,
objects: BTreeMap<ObjectId, ObjectState>,
cache: VecDeque<(usize, Arc<FrameFeatures>)>,
}
@@ -151,8 +151,8 @@ fn scalar(t: &Tensor) -> Result<f32> {
Ok(t.flatten_all()?.to_vec1::<f32>()?.first().copied().unwrap_or(0.0))
}
impl VideoTracker {
pub fn new(model: Arc<VideoModel>, frames: Box<dyn FrameSource>) -> Result<VideoTracker> {
impl<'a> VideoTracker<'a> {
pub fn new(model: Arc<VideoModel>, frames: Box<dyn FrameSource + 'a>) -> Result<VideoTracker<'a>> {
let n = frames.num_frames();
if n == 0 || n > MAX_FRAMES {
return Err(Error::Input(format!("a clip needs between 1 and {MAX_FRAMES} frames, got {n}")));