From e5842c0c923621d5b1a2d06408624ef380fc1923 Mon Sep 17 00:00:00 2001 From: Kristian Myram Date: Thu, 15 Oct 2026 01:37:17 +0200 Subject: [PATCH] S2.30: SAM 2.1 Tiny in the model registry Apache-2.0 entry pinned to a Hugging Face commit with its SHA-256 and size, listed in Settings > Roto Brush and installable with the existing verified install flow. No weights are bundled. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/tests/models.rs | 4 +++- crates/segment/README.md | 1 + crates/segment/src/lib.rs | 23 ++++++++++++++++++++++- docs/agents.md | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/tests/models.rs b/crates/engine/src/tests/models.rs index 21d316e75..786770ec5 100644 --- a/crates/engine/src/tests/models.rs +++ b/crates/engine/src/tests/models.rs @@ -36,7 +36,9 @@ fn models_are_listed_chosen_and_verified() { for (prefix, model, task) in [("roto", "mobilesam", Task::Mask), ("face", "mediapipe-face", Task::Face)] { let list = s.execute_checked(&format!("{prefix}.models"), json!({})).unwrap(); let ids: Vec<&str> = list["models"].as_array().unwrap().iter().map(|m| m["id"].as_str().unwrap()).collect(); - assert_eq!(ids, ["classical", model]); + // Roto Brush also offers SAM 2.1 Tiny (a video tracker; see crates/sam2). + let expect: Vec<&str> = if task == Task::Mask { vec!["classical", model, "sam2.1-hiera-tiny"] } else { vec!["classical", model] }; + assert_eq!(ids, expect); let m = &list["models"][1]; assert_eq!((m["licence"].as_str(), m["installed"].as_bool()), (Some("Apache-2.0"), Some(false))); assert!(m["url"].as_str().unwrap().starts_with("https://") && !m["authors"].as_str().unwrap().is_empty()); diff --git a/crates/segment/README.md b/crates/segment/README.md index 79b1d996b..4962f8ef8 100644 --- a/crates/segment/README.md +++ b/crates/segment/README.md @@ -36,6 +36,7 @@ kernels at run time). No ML framework, no unsafe code. | id | Task | Model | Authors | Licence | File | Size | |---|---|---|---|---|---|---| | `mobilesam` | Roto Brush | [MobileSAM](https://github.com/ChaoningZhang/MobileSAM): Segment Anything with a 5M-parameter TinyViT encoder | Chaoning Zhang et al., building on Meta AI's Segment Anything and Microsoft's TinyViT | Apache-2.0 | `mobile_sam.pt` from the official repository | 40.7 MB | +| `sam2.1-hiera-tiny` | Roto Brush | [SAM 2.1 Tiny](https://github.com/facebookresearch/sam2): Hiera-Tiny video object tracker with streaming memory | Meta AI (Nikhila Ravi et al.) | Apache-2.0 | `model.safetensors` of `facebook/sam2.1-hiera-tiny` (Hugging Face, pinned to a commit); loaded by `effectcraft-sam2` | 155.9 MB | | `mediapipe-face` | Face tracking | [MediaPipe Face Landmarker](https://developers.google.com/edge/mediapipe/solutions/vision/face_landmarker): BlazeFace detector and Face Mesh V2 | Google LLC | Apache-2.0 ([model card](https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Face%20Mesh%20V2.pdf)) | `face_landmarker.task` (float16, version 1) from Google's model storage | 3.8 MB | How they were checked: diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index adeb44881..dbfe07c1f 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -96,6 +96,24 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo { sha256: "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f", }; +/// SAM 2.1 Tiny (Meta AI, Apache-2.0): a video object tracker with streaming memory. The weights are +/// the Hugging Face `transformers` checkpoint, pinned to a commit so the SHA-256 stays valid; they +/// are loaded by `effectcraft-sam2` (native only), not by [`load`]. +pub const SAM2_TINY: ModelInfo = ModelInfo { + id: "sam2.1-hiera-tiny", + task: Task::Mask, + name: "SAM 2.1 Tiny", + description: "Meta's Segment Anything Model 2 (Hiera-Tiny, 39M parameters). Follows the object through the clip with a streaming memory: it keeps tracking through occlusion and when the object leaves and re-enters the frame. Slower than MobileSAM on a CPU.", + authors: "Meta AI (Nikhila Ravi et al., SAM 2), Hugging Face transformers checkpoint", + licence: "Apache-2.0", + licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", + homepage: "https://github.com/facebookresearch/sam2", + url: "https://huggingface.co/facebook/sam2.1-hiera-tiny/resolve/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors", + file_name: "sam2.1-hiera-tiny.safetensors", + size: 155_908_064, + sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7", +}; + pub const FACE_LANDMARKER: ModelInfo = ModelInfo { id: "mediapipe-face", task: Task::Face, @@ -112,7 +130,7 @@ pub const FACE_LANDMARKER: ModelInfo = ModelInfo { }; /// The trained models, besides the built-in classical engines. -pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, FACE_LANDMARKER]; +pub const MODELS: &[ModelInfo] = &[MOBILE_SAM, SAM2_TINY, FACE_LANDMARKER]; /// The id of the built-in classical engines (graph cut segmenter, shape-model face tracker): no /// weights, always available. @@ -150,6 +168,7 @@ pub fn load(id: &str, bytes: &[u8]) -> Result { Ok(Loaded::Mask(Arc::new(Sam { net: mobilesam::MobileSam::from_weights(&w)? }))) } "mediapipe-face" => Ok(Loaded::Face(Arc::new(mediapipe::FaceLandmarker::from_task(bytes)?))), + "sam2.1-hiera-tiny" => Err(format!("{} is loaded by the desktop application (effectcraft-sam2)", info.name)), _ => Err(format!("no loader for `{id}`")), } } @@ -263,6 +282,8 @@ mod tests { assert!(m.url.starts_with("https://") && m.licence_url.starts_with("https://") && m.homepage.starts_with("https://")); assert!(!m.authors.is_empty() && notice(m).contains(m.authors)); } + assert_eq!(models(Task::Mask).map(|m| m.id).collect::>(), ["mobilesam", "sam2.1-hiera-tiny"]); + assert!(SAM2_TINY.url.contains("/resolve/") && SAM2_TINY.file_name.ends_with(".safetensors"), "pinned to a commit, not a branch"); assert_eq!(models(Task::Face).map(|m| m.id).collect::>(), ["mediapipe-face"]); assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match"))); // A file that isn't the published one is refused before anything is parsed. diff --git a/docs/agents.md b/docs/agents.md index d801fcec6..864e7cf59 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -506,7 +506,7 @@ Trained model (Roto Brush 2.0 / 3.0): licence, size, URL, SHA-256 and whether it is installed, selected and active. - `roto.model.download {id}` fetches and verifies the official weights in the background; `roto.model.install {path, id?}` installs a file you already have. -- `roto.model.select {id: classical|mobilesam}` chooses which one Roto Brush uses. Loading runs +- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny}` chooses which one Roto Brush uses. Loading runs in the background; `"wait": true` (also on download and install) returns once it is in use. - `roto.model.remove {id}` deletes the installed weights. - The `version` property (`effects/#1/version`: 0 = 1.0, 1 = 2.0, 2 = 3.0) picks between the