mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-01 05:58:19 -06:00
#1539: retain checkpoint provenance and clarify model waits
This commit is contained in:
Kristian Myram
@@ -575,6 +575,8 @@ fn list(s: &mut Session, task: Task) -> Result<Value> {
|
||||
"licenceUrl": m.licence_url,
|
||||
"homepage": m.homepage,
|
||||
"url": m.url,
|
||||
"source": m.source,
|
||||
"sourceFile": m.source.rsplit('/').next().unwrap_or(m.file_name),
|
||||
"size": m.size,
|
||||
"sha256": m.sha256,
|
||||
"installed": s.model_path(m).is_some(),
|
||||
@@ -600,6 +602,9 @@ fn select(s: &mut Session, p: &Value, task: Task) -> Result<Value> {
|
||||
model_p(p, &cmd, task)?;
|
||||
}
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
if wait && id != seg::CLASSICAL && s.models.status(task).active.as_deref() != Some(id) {
|
||||
s.wait_model_worker(task, id).map_err(|error| bad(&cmd, error))?;
|
||||
}
|
||||
s.choose_model(task, id);
|
||||
if wait {
|
||||
s.apply_model(task, true);
|
||||
@@ -756,11 +761,38 @@ mod availability_tests {
|
||||
#[test]
|
||||
fn sam2_is_install_from_file_only() {
|
||||
let mut session = Session::default();
|
||||
let listed = session.execute("roto.models", json!({})).unwrap();
|
||||
for model in [seg::SAM2_TINY, seg::SAM2_SMALL, seg::SAM2_BASE_PLUS, seg::SAM2_LARGE] {
|
||||
let error = session.execute("roto.model.download", json!({"id": model.id})).unwrap_err();
|
||||
assert!(error.to_string().contains("Install from File"));
|
||||
assert!(session.models.status(Task::Mask).busy.is_none());
|
||||
assert!(install_bytes(Path::new("unused"), &model, b"invalid weights").unwrap_err().contains("SHA-256 differs"));
|
||||
let row = listed["models"].as_array().unwrap().iter().find(|row| row["id"] == model.id).unwrap();
|
||||
assert_eq!(row["source"], model.source);
|
||||
assert_eq!(row["sourceFile"], "model.safetensors");
|
||||
}
|
||||
}
|
||||
|
||||
/// A waiting selection cannot claim completion while unrelated work prevents its model loading.
|
||||
#[test]
|
||||
fn waiting_selection_refuses_an_unrelated_worker_unless_the_model_is_active() {
|
||||
crate::tests::roto::hold_roto_cache_test_lock();
|
||||
for active in [false, true] {
|
||||
let mut session = Session::default();
|
||||
session.choose_model(Task::Mask, if active { "mobilesam" } else { seg::CLASSICAL });
|
||||
set(session.models.lane(Task::Mask), |status| {
|
||||
status.active = active.then(|| "mobilesam".into());
|
||||
status.busy = Some("Downloading SAM 2".into());
|
||||
status.busy_model = Some("sam2.1-hiera-tiny");
|
||||
});
|
||||
let result = session.execute("roto.model.select", json!({"id": "mobilesam", "wait": true}));
|
||||
if active {
|
||||
assert_eq!(result.unwrap()["active"], "mobilesam");
|
||||
} else {
|
||||
assert!(result.unwrap_err().to_string().contains("another model operation"));
|
||||
assert_eq!(session.chosen_model(Task::Mask), seg::CLASSICAL, "a refused selection must not change settings");
|
||||
}
|
||||
assert!(session.models.status(Task::Mask).busy.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ Trained models for Roto Brush and face tracking, each kind behind one swappable
|
||||
EffectCraft's MIT OR Apache-2.0 (`ALLOWED_LICENCES`), with its task, authors, homepage, licence
|
||||
URL, an app download URL when available, size and SHA-256. `load` refuses anything else.
|
||||
SAM 2 is Install from File only until a storytold-controlled mirror is available.
|
||||
The checkpoint `source` stays separate from the app download URL, so file-install guidance and
|
||||
licence notices identify the verified artifact even when app downloads are unavailable.
|
||||
- **Weights are never bundled.** They are downloaded on demand (or installed from a file) by the
|
||||
engine (`roto.model.*` / `face.model.*`, Settings ▸ Roto Brush / Face Tracking), verified
|
||||
against the registry's SHA-256 (`sha256`, dependency-free) and kept with a notice naming their
|
||||
|
||||
@@ -143,6 +143,8 @@ pub struct ModelInfo {
|
||||
pub homepage: &'static str,
|
||||
/// App download URL; empty when only Install from File is available.
|
||||
pub url: &'static str,
|
||||
/// Checkpoint source for manual installation and notices; never used by the app downloader.
|
||||
pub source: &'static str,
|
||||
pub file_name: &'static str,
|
||||
pub size: u64,
|
||||
/// SHA-256 of the weights file (lower-case hex).
|
||||
@@ -162,6 +164,7 @@ pub const MOBILE_SAM: ModelInfo = ModelInfo {
|
||||
licence_url: "https://github.com/ChaoningZhang/MobileSAM/blob/master/LICENSE",
|
||||
homepage: "https://github.com/ChaoningZhang/MobileSAM",
|
||||
url: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt",
|
||||
source: "https://raw.githubusercontent.com/ChaoningZhang/MobileSAM/master/weights/mobile_sam.pt",
|
||||
file_name: "mobile_sam.pt",
|
||||
size: 40_728_226,
|
||||
sha256: "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f",
|
||||
@@ -180,6 +183,7 @@ pub const SAM2_TINY: ModelInfo = ModelInfo {
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "",
|
||||
source: "https://huggingface.co/facebook/sam2.1-hiera-tiny/blob/de431c4043854a71d8101e17995dfe596bf101a5/model.safetensors",
|
||||
file_name: "sam2.1-hiera-tiny.safetensors",
|
||||
size: 155_908_064,
|
||||
sha256: "48c14467e5cf9e51870511feb72c89688e82dd74523142c0538b663e193ac2a7",
|
||||
@@ -196,6 +200,7 @@ pub const SAM2_SMALL: ModelInfo = ModelInfo {
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "",
|
||||
source: "https://huggingface.co/facebook/sam2.1-hiera-small/blob/ee5bba1d82bb8749febdf90f45e84b687142ba03/model.safetensors",
|
||||
file_name: "sam2.1-hiera-small.safetensors",
|
||||
size: 184_305_280,
|
||||
sha256: "0a4067b11ce1e23d5229203f11c718a823060d15a4b23fa2372a7d4b77cbbc60",
|
||||
@@ -212,6 +217,7 @@ pub const SAM2_BASE_PLUS: ModelInfo = ModelInfo {
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "",
|
||||
source: "https://huggingface.co/facebook/sam2.1-hiera-base-plus/blob/b7320756a13354e7530a63935656d35b2f91a290/model.safetensors",
|
||||
file_name: "sam2.1-hiera-base-plus.safetensors",
|
||||
size: 323_476_296,
|
||||
sha256: "2012733a0de5d03efd1bba550a2847c4551be9ef2e0d497c83074df66189f780",
|
||||
@@ -228,6 +234,7 @@ pub const SAM2_LARGE: ModelInfo = ModelInfo {
|
||||
licence_url: "https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
||||
homepage: "https://github.com/facebookresearch/sam2",
|
||||
url: "",
|
||||
source: "https://huggingface.co/facebook/sam2.1-hiera-large/blob/665f8e2ad61cf5f53d65644ff27c8ee525124610/model.safetensors",
|
||||
file_name: "sam2.1-hiera-large.safetensors",
|
||||
size: 897_897_416,
|
||||
sha256: "dc407dce21301fd94abb395c5099b4f2c455fdc8a8f261ac3d0ea6d4cd197230",
|
||||
@@ -243,6 +250,7 @@ pub const FACE_LANDMARKER: ModelInfo = ModelInfo {
|
||||
licence_url: "https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Face%20Mesh%20V2.pdf",
|
||||
homepage: "https://developers.google.com/edge/mediapipe/solutions/vision/face_landmarker",
|
||||
url: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
|
||||
source: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
|
||||
file_name: "face_landmarker.task",
|
||||
size: 3_758_596,
|
||||
sha256: "64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff",
|
||||
@@ -314,10 +322,9 @@ fn loader(id: &str) -> Option<Loader> {
|
||||
|
||||
/// The notice kept next to installed weights: what they are, who made them, their licence.
|
||||
pub fn notice(m: &ModelInfo) -> String {
|
||||
let source = if m.url.is_empty() { m.homepage } else { m.url };
|
||||
format!(
|
||||
"{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
|
||||
m.name, m.authors, m.licence, m.licence, m.licence_url, source, m.homepage
|
||||
"{}\n\nAuthors: {}\nLicence: {} (full text: https://spdx.org/licenses/{}.html; as stated by the authors: {})\nSource: {}\nSHA-256: {}\nProject: {}\n\nInstalled unmodified. EffectCraft runs it with its own implementation\nand is not affiliated with or endorsed by its authors.\n",
|
||||
m.name, m.authors, m.licence, m.licence, m.licence_url, m.source, m.sha256, m.homepage
|
||||
)
|
||||
}
|
||||
|
||||
@@ -420,6 +427,7 @@ mod tests {
|
||||
assert!(ALLOWED_LICENCES.contains(&m.licence), "{}", m.id);
|
||||
assert_eq!(m.sha256.len(), 64);
|
||||
assert!(m.url.is_empty() || m.url.starts_with("https://"));
|
||||
assert!(m.source.starts_with("https://"));
|
||||
assert!(m.licence_url.starts_with("https://") && m.homepage.starts_with("https://"));
|
||||
assert!(!m.authors.is_empty() && notice(m).contains(m.authors));
|
||||
}
|
||||
@@ -430,7 +438,9 @@ mod tests {
|
||||
for model in [SAM2_TINY, SAM2_SMALL, SAM2_BASE_PLUS, SAM2_LARGE] {
|
||||
assert!(model.url.is_empty(), "SAM 2 has no app download until a storytold mirror is available");
|
||||
assert!(model.file_name.ends_with(".safetensors"));
|
||||
assert!(notice(&model).contains(&format!("Source: {}", model.homepage)));
|
||||
assert!(model.source.contains("/blob/") && model.source.ends_with("/model.safetensors"));
|
||||
assert!(notice(&model).contains(&format!("Source: {}", model.source)));
|
||||
assert!(notice(&model).contains(model.sha256));
|
||||
}
|
||||
assert_eq!(models(Task::Face).map(|m| m.id).collect::<Vec<_>>(), ["mediapipe-face"]);
|
||||
assert!(load("mediapipe-face", b"nope").err().is_some_and(|e| e.contains("does not match")));
|
||||
|
||||
@@ -508,6 +508,13 @@ fn models_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, acts: &mut
|
||||
}
|
||||
}
|
||||
}
|
||||
if let (Some(source), Some(file)) = (m["source"].as_str(), m["sourceFile"].as_str()) {
|
||||
let response = ui.link(file).on_hover_text(crate::i18n::tr("Source"));
|
||||
reg(app, &format!("settings.{prefix}.source.{id}"), &response, file);
|
||||
if response.clicked() {
|
||||
ui.ctx().open_url(egui::OpenUrl::new_tab(source));
|
||||
}
|
||||
}
|
||||
});
|
||||
if classic {
|
||||
return;
|
||||
@@ -944,6 +951,7 @@ mod tests {
|
||||
for id in ["sam2.1-hiera-tiny", "sam2.1-hiera-small", "sam2.1-hiera-base-plus", "sam2.1-hiera-large"] {
|
||||
assert!(app.auto.find(&format!("settings.roto.download.{id}")).is_none());
|
||||
assert!(app.auto.find(&format!("settings.roto.install.{id}")).is_some());
|
||||
assert_eq!(app.auto.find(&format!("settings.roto.source.{id}")).unwrap().label, "model.safetensors");
|
||||
}
|
||||
assert!(actions.is_empty());
|
||||
}
|
||||
|
||||
+9
-4
@@ -395,8 +395,9 @@ Face tracking's trained model works like Roto Brush's (below): `face.models` lis
|
||||
tracker and MediaPipe Face Landmarker with its authors and licence; `face.model.download {id}` /
|
||||
`face.model.install {path, id?}` install it (verified); `face.model.select {id:
|
||||
classical|mediapipe-face}` chooses it; `face.model.remove {id}` deletes it. Downloads and loading
|
||||
run in the background; `"wait": true` on download, install or select returns once the model is in
|
||||
use (or with its error), which scripts need before tracking. `track.mask` with a
|
||||
run in the background; `"wait": true` waits for the requested operation (or returns its error).
|
||||
Download/install do not change the selected model. After installing, use
|
||||
`face.model.select {id, wait:true}` to select and load it before tracking. `track.mask` with a
|
||||
face method returns `faceModel`, the engine it tries first (the classical one when the model finds
|
||||
no face in the mask).
|
||||
|
||||
@@ -503,12 +504,16 @@ reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/se
|
||||
|
||||
Trained model (Roto Brush 2.0 / 3.0):
|
||||
- `roto.models` lists the built-in engine and every registered model, with its authors,
|
||||
licence, size, URL, SHA-256 and whether it is installed, selected and active.
|
||||
licence, size, app download URL (empty for install-only models), checkpoint `source` and `sourceFile`,
|
||||
SHA-256 and whether it is installed, selected and active.
|
||||
- `roto.model.download {id}` fetches and verifies weights when the registry provides an app download URL;
|
||||
`roto.model.install {path, id?}` installs a file you already have. SAM 2 is Install from File only until
|
||||
a storytold-controlled mirror is available; its size and SHA-256 are verified before installation.
|
||||
- `roto.model.select {id: classical|mobilesam|sam2.1-hiera-tiny|sam2.1-hiera-small|sam2.1-hiera-base-plus|sam2.1-hiera-large}` chooses which one Roto Brush uses (SAM 2 also tracks the object through the clip, see [roto-brush-sam2.md](roto-brush-sam2.md); `roto.status` then reports `source` and per-frame `tracker.frames` with `confidence` and `occluded`). Loading runs
|
||||
in the background; `"wait": true` (also on download and install) returns once it is in use.
|
||||
in the background. `"wait": true` waits for the requested operation; download/install load the model
|
||||
only if it is already selected. They do not activate an unselected model: after installation, use
|
||||
`roto.model.select {id, wait:true}` to select and load it. Unrelated model work can return a retryable
|
||||
busy error rather than reporting that a waiting selection has finished.
|
||||
- `roto.propagate` never falls back to the classic engine silently: a chosen model that is not installed, or failed
|
||||
to load, is an error, and without `"wait": true` so is a model still loading. `roto.status` reports `model`
|
||||
(`selected`, `installed`, `active`, `loading`, `error`) and a `notice` while the chosen model is not the one in use.
|
||||
|
||||
@@ -8,7 +8,10 @@ Apache-2.0). It is optional: nothing changes until you install the model and cho
|
||||
(155.9 MB). The file's size and SHA-256 are checked and the licence notice is kept next to it.
|
||||
Agents: `roto.model.install {id: "sam2.1-hiera-tiny", path: "model.safetensors"}`.
|
||||
SAM 2 has no in-app download until the weights are mirrored under storytold control.
|
||||
2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny"}`).
|
||||
Each model card links its pinned Hugging Face `model.safetensors` checkpoint, rather than the
|
||||
project's different `.pt` files. Its checkpoint source and SHA-256 are retained in the notice.
|
||||
2. Choose it (`roto.model.select {id: "sam2.1-hiera-tiny", wait: true}` for agents). Installing an
|
||||
unselected model does not activate it, even with `wait: true` on installation.
|
||||
3. Paint strokes on the base frame as usual, then **Propagate**. One tracking pass follows the object through the
|
||||
whole span in both directions, using SAM 2's streaming memory, so it keeps tracking through occlusion and when the
|
||||
object leaves the frame and returns. Each frame's probability map is then refined by the same graph cut as every other
|
||||
|
||||
Reference in New Issue
Block a user