diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 6c24d0836f..6fd077ff96 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -149,6 +149,7 @@ impl Sam3 { return Err(Error::Missing(dir.to_path_buf())); } let weights = weights::Weights::open(&dir.join(WEIGHTS_FILE))?; + hugging_face_format(&weights)?; let backbone = vit::Vit::new(Self::vb(&weights, &device).pp("detector_model.vision_encoder.backbone"))?; Ok(Self { dir: dir.to_path_buf(), device, weights, backbone, tracker_neck: None, tracker: None, detector: None, tokenizer: None }) } @@ -260,6 +261,20 @@ impl Encoded { } } +#[cfg(not(target_arch = "wasm32"))] +/// Meta's original checkpoint (`sam3.pt`, or a `sam3.safetensors` conversion of it as on community +/// mirrors) names its tensors differently: say so, rather than name the first tensor it lacks (#3378). +fn hugging_face_format(weights: &weights::Weights) -> Result<()> { + let first = "detector_model.vision_encoder.backbone.embeddings.patch_embeddings.projection.weight"; + if !weights.contains(first) && (weights.has_prefix("detector.backbone.") || weights.has_prefix("tracker.")) { + return Err(Error::Model(format!( + "{WEIGHTS_FILE} is Meta's original checkpoint (sam3.pt, or a sam3.safetensors conversion of it as on community mirrors); \ + LightCraft needs the Hugging Face format: {WEIGHTS_FILE} from facebook/sam3" + ))); + } + Ok(()) +} + #[cfg(not(target_arch = "wasm32"))] fn sigmoid(x: f32) -> f32 { 1.0 / (1.0 + (-x).exp()) @@ -273,3 +288,22 @@ pub use detector::Output as DetectorOutput; pub fn sync(model: &Sam3) -> Result<()> { Ok(model.device.synchronize()?) } + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + + #[test] + fn meta_original_checkpoint_is_named_as_such() { + // #3378: a community mirror's `sam3.safetensors` (Meta's key names, C64 RoPE tables) saved as model.safetensors + let dir = std::env::temp_dir().join(format!("lc-sam3-original-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let header = r#"{"detector.backbone.vision_backbone.trunk.blocks.0.attn.freqs_cis":{"dtype":"C64","shape":[2],"data_offsets":[0,16]},"tracker.sam_mask_decoder.iou_token.weight":{"dtype":"F32","shape":[2],"data_offsets":[16,24]}}"#; + std::fs::rename(weights::tests::write("original", header, 24), dir.join(WEIGHTS_FILE)).unwrap(); + std::fs::write(dir.join("vocab.json"), "{}").unwrap(); + std::fs::write(dir.join("merges.txt"), "").unwrap(); + let err = Sam3::load_cpu(&dir).err().unwrap().to_string(); + assert!(err.contains("Meta's original checkpoint") && err.contains("facebook/sam3"), "{err}"); + let _ = std::fs::remove_dir_all(dir); + } +} diff --git a/crates/segment/src/weights.rs b/crates/segment/src/weights.rs index 48ec2ef65f..ef47a7230a 100644 --- a/crates/segment/src/weights.rs +++ b/crates/segment/src/weights.rs @@ -16,6 +16,9 @@ use candle_nn::var_builder::SimpleBackend; use crate::{Error, Result}; struct Entry { + /// `Err`: the header's name for a dtype LightCraft doesn't read (Meta's original checkpoint + /// stores its RoPE tables as C64); the tensor is listed, and reading it fails + dtype: std::result::Result, shape: Vec, start: u64, end: u64, @@ -48,8 +51,8 @@ impl Weights { continue; } let dtype = match v.get("dtype").and_then(|d| d.as_str()) { - Some("F32") => DType::F32, - other => return Err(Error::Model(format!("{name}: unsupported dtype {other:?} (the SAM 3 checkpoint is F32)"))), + Some("F32") => Ok(DType::F32), + other => Err(format!("{other:?}")), }; let shape: Vec = v .get("shape") @@ -59,15 +62,18 @@ impl Weights { let offsets: Vec = v.get("data_offsets").and_then(|s| s.as_array()).map(|a| a.iter().filter_map(|x| x.as_u64()).collect()).unwrap_or_default(); let [start, end] = offsets[..] else { return Err(Error::Model(format!("{name}: bad data_offsets"))) }; - let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d)); - if end.checked_sub(start) != want.map(|w| w as u64) { - return Err(Error::Model(format!("{name}: size does not match its shape"))); + // an unreadable tensor's size isn't checked: its bytes are never read + if let Ok(dtype) = dtype { + let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d)); + if end.checked_sub(start) != want.map(|w| w as u64) { + return Err(Error::Model(format!("{name}: size does not match its shape"))); + } } // the bytes must be in the file (a damaged or truncated download fails here, not later) if (8 + n).checked_add(end).is_none_or(|e| e > file_len) { return Err(Error::Model(format!("{}: tensor {name} lies outside the file (incomplete or damaged download?)", path.display()))); } - entries.insert(name, Entry { shape, start, end }); + entries.insert(name, Entry { dtype, shape, start, end }); } Ok(Arc::new(Self { file: Mutex::new(file), entries, data_start: 8 + n })) } @@ -76,6 +82,11 @@ impl Weights { self.entries.contains_key(name) } + /// Whether any tensor's name starts with `prefix`. + pub fn has_prefix(&self, prefix: &str) -> bool { + self.entries.keys().any(|k| k.starts_with(prefix)) + } + /// The shape stored in the header for `name`, without reading its data. pub fn shape(&self, name: &str) -> Option<&[usize]> { self.entries.get(name).map(|e| e.shape.as_slice()) @@ -84,6 +95,9 @@ impl Weights { /// Tensor `name` as stored, on the CPU. pub fn read(&self, name: &str) -> Result { let e = self.entries.get(name).ok_or_else(|| Error::Model(format!("missing tensor `{name}`")))?; + if let Err(dtype) = &e.dtype { + return Err(Error::Model(format!("{name}: unsupported dtype {dtype} (the SAM 3 checkpoint is F32)"))); + } // checked against the file's length in `open` let len = usize::try_from(e.end.saturating_sub(e.start)).map_err(|_| Error::Model(format!("{name}: too large")))?; let mut bytes = vec![0u8; len]; @@ -122,10 +136,10 @@ impl SimpleBackend for Backend { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - fn write(name: &str, header: &str, data_len: usize) -> std::path::PathBuf { + pub(crate) fn write(name: &str, header: &str, data_len: usize) -> std::path::PathBuf { let p = std::env::temp_dir().join(format!("lc-weights-{name}-{}.safetensors", std::process::id())); let mut b = (header.len() as u64).to_le_bytes().to_vec(); b.extend_from_slice(header.as_bytes()); @@ -147,6 +161,22 @@ mod tests { let _ = std::fs::remove_file(p); } + #[test] + fn a_tensor_of_another_dtype_fails_when_read_not_when_opened() { + // Meta's original checkpoint stores RoPE tables as C64 next to its F32 weights (#3378) + let p = write( + "c64", + r#"{"freqs_cis":{"dtype":"C64","shape":[2],"data_offsets":[0,16]},"a":{"dtype":"F32","shape":[2],"data_offsets":[16,24]}}"#, + 24, + ); + let w = Weights::open(&p).unwrap(); + assert!(w.contains("freqs_cis")); + let err = w.read("freqs_cis").unwrap_err().to_string(); + assert!(err.contains("unsupported dtype") && err.contains("C64"), "{err}"); + assert_eq!(w.read("a").unwrap().dims(), &[2]); + let _ = std::fs::remove_file(p); + } + #[test] fn tensors_must_lie_inside_the_file() { let ok = write("ok", r#"{"a":{"dtype":"F32","shape":[2,2],"data_offsets":[0,16]}}"#, 16); diff --git a/docs/ai-masks.md b/docs/ai-masks.md index 14737ef520..41873cb5a6 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -153,7 +153,9 @@ the Mozilla root certificates; no OpenSSL, `ring` or `aws-lc`, nothing compiled ### By hand Put `model.safetensors`, `vocab.json` and `merges.txt` from `facebook/sam3` in the model folder: -`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. Developers +`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. It must be +the Hugging Face format (3,439,938,512 bytes): `sam3.safetensors` from community mirrors, renamed, is +Meta's original checkpoint, and LightCraft says so instead of loading it. Developers can use the installer scripts, which download from Hugging Face with your token and verify the weights: diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 9479846ff6..8a4d098f2f 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts). | LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) | | LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) | | LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) | -| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); the stored grid is enlarged along the photo's edges (a guided filter of its logits by log luminance, so no outline spills onto the background under a raised Exposure, #3272). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) | +| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); the stored grid is enlarged along the photo's edges (a guided filter of its logits by log luminance, so no outline spills onto the background under a raised Exposure, #3272). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md; Meta's original checkpoint installed in place of the Hugging Face one is named as such, #3378). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) | | LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic | | LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty | | LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); with the mask overlay hidden (O / the eye, or after a mask slider hid it) or a colour overlay at opacity 0 it is not tinted while painting either, so dodging and burning is judged on the photo, and its effect shows once the stroke is committed (#2549); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); oversized path segments skip intermediate dabs instead of allocating a million samples; Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt | diff --git a/docs/whats-new.md b/docs/whats-new.md index 013220cd0c..7d43b2ec8d 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -9,6 +9,12 @@ edit history now keep the first 64 too, in order, and the extra ones are skipped as they are read. Masks made in the app (up to 5 samples) are unchanged. +### A SAM 3 model in the wrong format says so +- Object and Describe need `model.safetensors` from `facebook/sam3` in the Hugging Face format. The + `sam3.safetensors` on community mirrors is Meta's original checkpoint, with other tensor names; installed in its + place, AI masks failed with "unsupported dtype C64". LightCraft now says that the file is Meta's original + checkpoint and which file it needs instead. + ## 2026-10-29 ### Object and Describe masks show their pins