// Modified work: wraps the ported SAM 2 image path behind EffectCraft's `MaskModel` interface // (Apache-2.0, see NOTICE). //! SAM 2 as a drop-in [`MaskModel`]: a frame and a prompt in, a foreground probability per pixel //! out. The frame's encoding is cached, so further strokes on the same frame cost one decode. use std::sync::{Arc, Mutex, PoisonError}; use candle_core::{Device, Tensor}; use effectcraft_segment::{MaskModel, ModelInfo, SAM2_TINY}; use crate::model::Encoded; use crate::prompt::{INPUT_SIZE, Prompt}; use crate::session::VideoModel; use crate::{Error, Result}; /// Largest frame accepted (pixels per side and in total). const MAX_SIDE: usize = 16_384; const MAX_PIXELS: usize = 100_000_000; /// Resize `w x h` RGB (0..1) to the model input and normalise it: `[1, 3, 1024, 1024]`. pub fn preprocess(rgb: &[[f32; 3]], w: usize, h: usize) -> Result { let pixels = validate_frame_size(w, h, rgb.len())?; const MEAN: [f32; 3] = [0.485, 0.456, 0.406]; const STD: [f32; 3] = [0.229, 0.224, 0.225]; let mut planar = vec![0f32; 3 * pixels]; for (i, px) in rgb.iter().enumerate() { for c in 0..3 { let v = px.get(c).copied().unwrap_or(0.0); if let Some(o) = planar.get_mut(c * pixels + i) { *o = if v.is_finite() { v.clamp(0.0, 1.0) } else { 0.0 }; } } } let x = Tensor::from_vec(planar, (1, 3, h, w), &Device::Cpu)?.upsample_bilinear2d(INPUT_SIZE, INPUT_SIZE, false)?; let mean = Tensor::from_slice(&MEAN, (1, 3, 1, 1), &Device::Cpu)?; let std = Tensor::from_slice(&STD, (1, 3, 1, 1), &Device::Cpu)?; Ok(x.broadcast_sub(&mean)?.broadcast_div(&std)?) } pub(crate) fn validate_frame_size(w: usize, h: usize, len: usize) -> Result { w.checked_mul(h) .filter(|n| *n > 0 && *n == len && w <= MAX_SIDE && h <= MAX_SIDE && *n <= MAX_PIXELS) .ok_or_else(|| Error::Input(format!("a {w} x {h} frame needs {} pixels (at most {MAX_PIXELS}), got {len}", w.saturating_mul(h)))) } fn fingerprint(rgb: &[[f32; 3]], w: usize, h: usize) -> u64 { let mut hash = 0xcbf2_9ce4_8422_2325u64 ^ ((w as u64) << 32 | h as u64); for px in rgb { for v in px { hash = (hash ^ u64::from(v.to_bits())).wrapping_mul(0x0100_0000_01b3); } } hash } /// SAM 2 for single frames. pub struct Sam2Mask { model: Arc, info: &'static ModelInfo, last: Mutex)>>, used: Mutex, pub(crate) peak: std::sync::atomic::AtomicU64, } impl Sam2Mask { pub fn new(model: Arc) -> Sam2Mask { Sam2Mask::with_info(model, &SAM2_TINY) } /// A model of a given registry entry (its size). pub fn with_info(model: Arc, info: &'static ModelInfo) -> Sam2Mask { Sam2Mask { model, info, last: Mutex::new(None), used: Mutex::new(std::time::Instant::now()), peak: std::sync::atomic::AtomicU64::new(0) } } /// Note that the model is being used (for [`MaskModel::idle_secs`]). pub(crate) fn touch(&self) { *self.used.lock().unwrap_or_else(PoisonError::into_inner) = std::time::Instant::now(); } /// The video model, for trackers built on the same weights. pub fn video_model(&self) -> &Arc { &self.model } fn encoded(&self, rgb: &[[f32; 3]], w: usize, h: usize) -> Result> { let key = fingerprint(rgb, w, h); if let Some((k, e)) = self.last.lock().unwrap_or_else(PoisonError::into_inner).as_ref() && *k == key { return Ok(e.clone()); } let enc = Arc::new(self.model.encode(&preprocess(rgb, w, h)?)?); *self.last.lock().unwrap_or_else(PoisonError::into_inner) = Some((key, enc.clone())); Ok(enc) } fn probability(&self, rgb: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> Result> { let enc = self.encoded(rgb, w, h)?; let (sx, sy) = (INPUT_SIZE as f32 / w as f32, INPUT_SIZE as f32 / h as f32); let p = Prompt { points: prompt.points.iter().map(|(c, fg)| ([c[0] * sx, c[1] * sy], *fg)).collect(), bbox: prompt.bbox.map(|b| [b[0] * sx, b[1] * sy, b[2] * sx, b[3] * sy]), }; if p.points.is_empty() && p.bbox.is_none() { return Err(Error::Input("SAM 2 needs at least one point or a box".into())); } let multimask = p.points.len() + 2 * usize::from(p.bbox.is_some()) <= 1; let out = self.model.image.segment(&enc, &p, multimask)?; let ious = out.iou.flatten_all()?.to_vec1::()?; let best = ious.iter().enumerate().fold((0usize, f32::NEG_INFINITY), |b, (i, v)| if *v > b.1 { (i, *v) } else { b }).0; let logits = out.masks.narrow(1, best, 1)?.upsample_bilinear2d(h, w, false)?; Ok(candle_nn::ops::sigmoid(&logits)?.flatten_all()?.to_vec1::()?) } } impl MaskModel for Sam2Mask { fn info(&self) -> &'static ModelInfo { self.info } fn segment(&self, rgb: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { self.touch(); self.probability(rgb, w, h, prompt).map_err(|e| e.to_string()) } fn tracker(&self) -> Option<&dyn effectcraft_segment::ObjectTracker> { Some(self) } fn device_name(&self) -> Option { Some(self.model.device_name()) } fn peak_state_bytes(&self) -> Option { Some(self.peak.load(std::sync::atomic::Ordering::Relaxed)) } fn idle_secs(&self) -> Option { Some(self.used.lock().unwrap_or_else(PoisonError::into_inner).elapsed().as_secs_f64()) } } /// Verify the installed file against the registry and build the model. pub fn load(bytes: &[u8]) -> std::result::Result, String> { let got = effectcraft_segment::sha256::hex(bytes); let Some(info) = SIZES.iter().find(|m| m.sha256 == got) else { return Err(format!("SAM 2.1: the file does not match any published size (SHA-256 {got})")); }; let w = crate::weights::Weights::from_bytes(bytes).map_err(|e| e.to_string())?; let model = VideoModel::load_auto(&w).map_err(|e| e.to_string())?; Ok(Arc::new(Sam2Mask::with_info(Arc::new(model), info))) } /// The size to suggest: Tiny. On the benchmark clips (`bench --roto`, 24 frames, a GPU) no larger size tracked better, they /// cost 1.0x (Small), 1.15x (Base+) and 1.7x (Large) the time, and Large lost the object after it re-entered the frame. pub fn recommended() -> &'static ModelInfo { &SAM2_TINY } /// A one-line note for the Settings list: what was measured for this size, and what the bigger ones cost without a GPU. pub fn hint(id: &str) -> Option { hint_for(id, crate::gpu::Gpu::probe()) } /// [`hint`] given whether a GPU is known (`None`: not looked for yet). The list is built on the /// Settings path, so it never creates the GPU device itself: that happens when a model is loaded, /// and after that the notes say whether one was found. fn hint_for(id: &str, gpu: Option) -> Option { Some( match id { "sam2.1-hiera-tiny" => "Recommended: the smallest download, and no larger size tracked better on the benchmark clips.", "sam2.1-hiera-small" => "As fast as Tiny on a GPU and no better on the benchmark clips; worth a try on difficult footage.", "sam2.1-hiera-base-plus" => match gpu { Some(false) => "Slow without a GPU (several seconds a frame); none was found.", Some(true) => "About 15% slower than Tiny on a GPU and no better on the benchmark clips.", None => "About 15% slower than Tiny on a GPU and no better on the benchmark clips; several seconds a frame without one.", }, "sam2.1-hiera-large" => match gpu { Some(false) => "Slow without a GPU (many seconds a frame); none was found.", Some(true) => { "About 1.7 times Tiny's time on a GPU; it lost the object after it re-entered in one benchmark clip. Try it on difficult footage." } None => "About 1.7 times Tiny's time on a GPU, many seconds a frame without one; it lost the object after it re-entered in one benchmark clip.", }, _ => return None, } .to_string(), ) } /// The registry entries this crate loads. pub const SIZES: [&ModelInfo; 4] = [&SAM2_TINY, &effectcraft_segment::SAM2_SMALL, &effectcraft_segment::SAM2_BASE_PLUS, &effectcraft_segment::SAM2_LARGE]; #[cfg(test)] mod tests { use super::*; #[test] fn preprocess_rejects_bad_frames_and_cleans_pixels() { assert!(preprocess(&[], 0, 0).is_err()); assert!(preprocess(&[[0.0; 3]; 3], 2, 2).is_err()); assert!(preprocess(&[[0.5; 3]; 4], usize::MAX, 2).is_err()); let t = preprocess(&[[f32::NAN, 2.0, -1.0]; 4], 2, 2).unwrap(); assert_eq!(t.dims(), [1, 3, INPUT_SIZE, INPUT_SIZE]); let v = t.flatten_all().unwrap().to_vec1::().unwrap(); assert!(v.iter().all(|x| x.is_finite())); } /// The Settings list asks for hints without creating the GPU device: before anything has loaded a /// model there is no answer yet, and the notes say so rather than guess. #[test] fn hints_do_not_need_the_gpu_to_have_been_created() { for id in ["sam2.1-hiera-base-plus", "sam2.1-hiera-large"] { let (unknown, none, found) = (hint_for(id, None), hint_for(id, Some(false)), hint_for(id, Some(true))); assert!(unknown.is_some() && none.is_some() && found.is_some()); assert!(none.as_deref().is_some_and(|h| h.contains("none was found")), "{id}: {none:?}"); assert!(unknown.as_deref().is_some_and(|h| h.contains("without one") && !h.contains("none was found")), "{id}: {unknown:?}"); } assert!(hint_for("mobilesam", None).is_none()); } #[test] fn a_wrong_file_is_refused_before_parsing() { let e = load(b"not the weights").err().unwrap_or_default(); assert!(e.contains("does not match"), "{e}"); } }