diff --git a/crates/engine/src/analysis/models.rs b/crates/engine/src/analysis/models.rs index 5d5cf436d..63550545a 100644 --- a/crates/engine/src/analysis/models.rs +++ b/crates/engine/src/analysis/models.rs @@ -575,6 +575,8 @@ fn list(s: &mut Session, task: Task) -> Result { "licenceUrl": m.licence_url, "homepage": m.homepage, "url": m.url, + "source": m.source, + "sourceFile": m.source.rsplit('/').next().unwrap_or(m.file_name), "size": m.size, "sha256": m.sha256, "installed": s.model_path(m).is_some(), @@ -600,6 +602,9 @@ fn select(s: &mut Session, p: &Value, task: Task) -> Result { model_p(p, &cmd, task)?; } let wait = b_p(p, "wait").unwrap_or(false); + if wait && id != seg::CLASSICAL && s.models.status(task).active.as_deref() != Some(id) { + s.wait_model_worker(task, id).map_err(|error| bad(&cmd, error))?; + } s.choose_model(task, id); if wait { s.apply_model(task, true); @@ -756,11 +761,38 @@ mod availability_tests { #[test] fn sam2_is_install_from_file_only() { let mut session = Session::default(); + let listed = session.execute("roto.models", json!({})).unwrap(); for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] { let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err(); assert!(error.to_string().contains("Install from File")); assert!(session.models.status(Task::Mask).busy.is_none()); assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs")); + let row = listed["models"].as_array().unwrap().iter().find(|row| row["id"] == model.id).unwrap(); + assert_eq!(row["source"], model.source); + assert_eq!(row["sourceFile"], "model.safetensors"); + } + } + + /// A waiting selection cannot claim completion while unrelated work prevents its model loading. + #[test] + fn waiting_selection_refuses_an_unrelated_worker_unless_the_model_is_active() { + crate::tests::roto::hold_roto_cache_test_lock(); + for active in [false, true] { + let mut session = Session::default(); + session.choose_model(Task::Mask, if active { "mobilesam" } else { seg::CLASSICAL }); + set(session.models.lane(Task::Mask), |status| { + status.active = active.then(|| "mobilesam".into()); + status.busy = Some("Downloading SAM 2".into()); + status.busy_model = Some("sam2.1-hiera-tiny"); + }); + let result = session.execute("roto.model.select", json!({"id": "mobilesam", "wait": true})); + if active { + assert_eq!(result.unwrap()["active"], "mobilesam"); + } else { + assert!(result.unwrap_err().to_string().contains("another model operation")); + assert_eq!(session.chosen_model(Task::Mask), seg::CLASSICAL, "a refused selection must not change settings"); + } + assert!(session.models.status(Task::Mask).busy.is_some()); } } diff --git a/crates/segment/README.md b/crates/segment/README.md index 06ffce7ad..a95c92f93 100644 --- a/crates/segment/README.md +++ b/crates/segment/README.md @@ -14,6 +14,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappable EffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence URL, an app download URL when available, size and SHA-256. `load` refuses anything else. SAM 2 is Install from File only until a storytold-controlled mirror is available. + The checkpoint `source` stays separate from the app download URL, so file-install guidance and + licence notices identify the verified artifact even when app downloads are unavailable. - **Weights are never bundled.** They are downloaded on demand (or installed from a file) by the engine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verified against the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming their diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 0d9cc3134..7fc0e4ff6 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -143,6 +143,8 @@ pub struct ModelInfo { pub homepage: &'static str, /// App download URL; empty when only Install from File is available. pub url: &'static str, + /// Checkpoint source for manual installation and notices; never used by the app downloader. + pub source: &'static str, pub file_name: &'static str, pub size: u64, /// SHA-256 of the weights file (lower-case hex). @@ -162,6 +164,7 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo { licence_url: "https://github.com/ChaoningZhang/MobileSAM/blob/master/LICENSE", homepage: "https://github.com/ChaoningZhang/MobileSAM", url: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt", + source: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt", file_name: "mobile_sam.pt", size: 40_728_226, sha256: "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f", @@ -180,6 +183,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo { licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", url: "", + source: "https://huggingface.co/facebook/sam2.1-hiera-tiny/blob/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors", file_name: "sam2.1-hiera-tiny.safetensors", size: 155_908_064, sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7", @@ -196,6 +200,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo { licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", url: "", + source: "https://huggingface.co/facebook/sam2.1-hiera-small/blob/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors", file_name: "sam2.1-hiera-small.safetensors", size: 184_305_280, sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60", @@ -212,6 +217,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo { licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", url: "", + source: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/blob/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors", file_name: "sam2.1-hiera-base-plus.safetensors", size: 323_476_296, sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780", @@ -228,6 +234,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo { licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE", homepage: "https://github.com/facebookresearch/sam2", url: "", + source: "https://huggingface.co/facebook/sam2.1-hiera-large/blob/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors", file_name: "sam2.1-hiera-large.safetensors", size: 897_897_416, sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230", @@ -243,6 +250,7 @@ pub const FACE_LANDMARKER: ModelInfo = ModelInfo { licence_url: "https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Face%20Mesh%20V2.pdf", homepage: "https://developers.google.com/edge/mediapipe/solutions/vision/face_landmarker", url: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", + source: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", file_name: "face_landmarker.task", size: 3_758_596, sha256: "64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff", @@ -314,10 +322,9 @@ fn loader(id: &str) -> Option { /// The notice kept next to installed weights: what they are, who made them, their licence. pub fn notice(m: &ModelInfo) -> String { - let source = if m.url.is_empty() { m.homepage } else { m.url }; format!( - "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n", - m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage + "{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nSHA-256: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n", + m.name, m.authors, m.licence, m.licence, m.licence_url, m.source, m.sha256, m.homepage ) } @@ -420,6 +427,7 @@ mod tests { assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id); assert_eq!(m.sha256.len(), 64); assert!(m.url.is_empty() || m.url.starts_with("https://")); + assert!(m.source.starts_with("https://")); assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://")); assert!(!m.authors.is_empty() && notice(m).contains(m.authors)); } @@ -430,7 +438,9 @@ mod tests { for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] { assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available"); assert!(model.file_name.ends_with(".safetensors")); - assert!(notice(&model).contains(&format!("Source: {}", model.homepage))); + assert!(model.source.contains("/blob/") && model.source.ends_with("/model.safetensors")); + assert!(notice(&model).contains(&format!("Source: {}", model.source))); + assert!(notice(&model).contains(model.sha256)); } assert_eq!(models(Task::Face).map(|m| m.id).collect::>(), ["mediapipe-face"]); assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match"))); diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index 5fdbf1c50..69f7623dd 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -508,6 +508,13 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mut } } } + if let (Some(source), Some(file)) = (m["source"].as_str(), m["sourceFile"].as_str()) { + let response = ui.link(file).on_hover_text(crate::i18n::tr("Source")); + reg(app, &format!("settings.{prefix}.source.{id}"), &response, file); + if response.clicked() { + ui.ctx().open_url(egui::OpenUrl::new_tab(source)); + } + } }); if classic { return; @@ -944,6 +951,7 @@ mod tests { for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] { assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none()); assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some()); + assert_eq!(app.auto.find(&format!("settings.roto.source.{id}")).unwrap().label, "model.safetensors"); } assert!(actions.is_empty()); } diff --git a/docs/agents.md b/docs/agents.md index 1a5035f8f..d43cbd689 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -395,8 +395,9 @@ Face tracking's trained model works like Roto Brush's (below): `face.models` lis tracker and MediaPipe Face Landmarker with its authors and licence; `face.model.download {id}` / `face.model.install {path, id?}` install it (verified); `face.model.select {id: classical|mediapipe-face}` chooses it; `face.model.remove {id}` deletes it. Downloads and loading -run in the background; `"wait": true` on download, install or select returns once the model is in -use (or with its error), which scripts need before tracking. `track.mask` with a +run in the background; `"wait": true` waits for the requested operation (or returns its error). +Download/install do not change the selected model. After installing, use +`face.model.select {id, wait:true}` to select and load it before tracking. `track.mask` with a face method returns `faceModel`, the engine it tries first (the classical one when the model finds no face in the mask). @@ -503,12 +504,16 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/se Trained model (Roto Brush 2.0 / 3.0): - `roto.models` lists the built-in engine and every registered model, with its authors, - licence, size, URL, SHA-256 and whether it is installed, selected and active. + licence, size, app download URL (empty for install-only models), checkpoint `source` and `sourceFile`, + SHA-256 and whether it is installed, selected and active. - `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL; `roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until a storytold-controlled mirror is available; its size and SHA-256 are verified before installation. - `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runs - in the background; `"wait": true` (also on download and install) returns once it is in use. + in the background. `"wait": true` waits for the requested operation; download/install load the model + only if it is already selected. They do not activate an unselected model: after installation, use + `roto.model.select {id, wait:true}` to select and load it. Unrelated model work can return a retryable + busy error rather than reporting that a waiting selection has finished. - `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed to load, is an error, and without `"wait": true` so is a model still loading. `roto.status` reports `model` (`selected`, `installed`, `active`, `loading`, `error`) and a `notice` while the chosen model is not the one in use. diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index 5b8038552..7078fa103 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -8,7 +8,10 @@ Apache-2.0). It is optional: nothing changes until you install the model and cho (155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it. Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`. SAM 2 has no in-app download until the weights are mirrored under storytold control. -2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`). + Each model card links its pinned Hugging Face `model.safetensors` checkpoint, rather than the + project's different `.pt` files. Its checkpoint source and SHA-256 are retained in the notice. +2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny", wait: true}` for agents). Installing an + unselected model does not activate it, even with `wait: true` on installation. 3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through the whole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the object leaves the frame and returns. Each frame's probability map is then refined by the same graph cut as every other