diff --git a/Cargo.lock b/Cargo.lock index 6a694299d..abf1f898b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1765,6 +1765,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-sam2" +version = "0.9.2" + [[package]] name = "effectcraft-script" version = "0.18.0" diff --git a/Cargo.toml b/Cargo.toml index afc59257b..28a011f24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } effectcraft-track = { path = "crates/track" } effectcraft-segment = { path = "crates/segment" } +effectcraft-sam2 = { path = "crates/sam2" } effectcraft-model = { path = "crates/model" } effectcraft-render = { path = "crates/render" } effectcraft-gpu = { path = "crates/gpu" } diff --git a/crates/sam2/Cargo.toml b/crates/sam2/Cargo.toml new file mode 100644 index 000000000..24c317bc1 --- /dev/null +++ b/crates/sam2/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-sam2" +description = "SAM 2.1 (Hiera) video object tracker for the Roto Brush: a Rust port of the Hugging Face transformers implementation, native only, weights downloaded by the user" +version.workspace = true +edition.workspace = true +license = "Apache-2.0" +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/sam2/NOTICE b/crates/sam2/NOTICE new file mode 100644 index 000000000..2dcb613fa --- /dev/null +++ b/crates/sam2/NOTICE @@ -0,0 +1,11 @@ +EffectCraft SAM 2 (effectcraft-sam2) +Copyright the EffectCraft contributors. + +This crate is a Rust port of the SAM 2 implementation in Hugging Face `transformers` +(`models/sam2` and `models/sam2_video`), Copyright 2025 The HuggingFace Inc. team, which in turn +implements "SAM 2: Segment Anything in Images and Videos" by Meta AI (facebookresearch/sam2). +Both are licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0). + +Each ported source file states that it is a modified work (Apache-2.0 section 4(b)). +Model weights are not part of this crate; they are downloaded by the user from the official +`facebook/sam2.1-hiera-*` repositories (Apache-2.0). diff --git a/crates/sam2/src/lib.rs b/crates/sam2/src/lib.rs new file mode 100644 index 000000000..d41de59fa --- /dev/null +++ b/crates/sam2/src/lib.rs @@ -0,0 +1,78 @@ +//! SAM 2.1 video object tracker (Hiera backbone) for the Roto Brush. +//! +//! A Rust port of the Hugging Face `transformers` SAM 2 implementation (Apache-2.0; see +//! `NOTICE`). Native only: the crate is empty on wasm32 so L0–L4 still build for the web. +//! Nothing here loads weights unless asked to; weights are never bundled. +//! +//! Modules are added stage by stage, each checked against reference tensors produced by +//! `tools/sam2_reference.py`. +#![cfg(not(target_arch = "wasm32"))] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::fmt; + +/// Everything that can go wrong in this crate. Never a panic. +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + /// The weights file is missing a tensor, has the wrong shape, or contains non-finite values. + Weights(String), + /// The input (frame size, prompt, frame index) is unusable. + Input(String), + /// The work was cancelled by the caller. + Cancelled, + /// A time or memory budget was exceeded. + Budget(String), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Weights(m) => write!(f, "SAM 2 weights: {m}"), + Error::Input(m) => write!(f, "SAM 2 input: {m}"), + Error::Cancelled => write!(f, "SAM 2: cancelled"), + Error::Budget(m) => write!(f, "SAM 2 budget: {m}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// The released SAM 2.1 sizes. Only Tiny is wired up first; the others are configuration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Size { + Tiny, + Small, + BasePlus, + Large, +} + +impl Size { + /// The Hugging Face repository the official weights come from. + pub fn repo(self) -> &'static str { + match self { + Size::Tiny => "facebook/sam2.1-hiera-tiny", + Size::Small => "facebook/sam2.1-hiera-small", + Size::BasePlus => "facebook/sam2.1-hiera-base-plus", + Size::Large => "facebook/sam2.1-hiera-large", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sizes_name_their_official_repositories() { + assert_eq!(Size::Tiny.repo(), "facebook/sam2.1-hiera-tiny"); + assert!(Size::Large.repo().ends_with("large")); + } + + #[test] + fn errors_say_what_failed() { + assert!(Error::Input("empty frame".into()).to_string().contains("empty frame")); + assert_eq!(Error::Cancelled.to_string(), "SAM 2: cancelled"); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ad03cada7..401ed861e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -50,6 +50,7 @@ const LAYERS: &[(&str, u8)] = &[ ("expr", 3), ("export", 3), ("gpu", 3), + ("sam2", 3), ("lottie", 3), ("format", 3), ("plugin", 3),