Merge PR #3384: LR-MASK-OBJECTS: a SAM 3 checkpoint in Meta's original format is named as such, not as an unsupported dtype (#3378)

# Conflicts:
#	docs/parity-checklist.md
#	docs/whats-new.md

crates/segment/src/lib.rs

@@ -149,6 +149,7 @@ impl Sam3 {
             return Err(Error::Missing(dir.to_path_buf()));
         }
         let weights = weights::Weights::open(&dir.join(WEIGHTS_FILE))?;
+        hugging_face_format(&weights)?;
         let backbone = vit::Vit::new(Self::vb(&weights, &device).pp("detector_model.vision_encoder.backbone"))?;
         Ok(Self { dir: dir.to_path_buf(), device, weights, backbone, tracker_neck: None, tracker: None, detector: None, tokenizer: None })
     }
@@ -260,6 +261,20 @@ impl Encoded {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+/// Meta's original checkpoint (`sam3.pt`, or a `sam3.safetensors` conversion of it as on community
+/// mirrors) names its tensors differently: say so, rather than name the first tensor it lacks (#3378).
+fn hugging_face_format(weights: &weights::Weights) -> Result<()> {
+    let first = "detector_model.vision_encoder.backbone.embeddings.patch_embeddings.projection.weight";
+    if !weights.contains(first) && (weights.has_prefix("detector.backbone.") || weights.has_prefix("tracker.")) {
+        return Err(Error::Model(format!(
+            "{WEIGHTS_FILE} is Meta's original checkpoint (sam3.pt, or a sam3.safetensors conversion of it as on community mirrors); \
+             LightCraft needs the Hugging Face format: {WEIGHTS_FILE} from facebook/sam3"
+        )));
+    }
+    Ok(())
+}
+
 #[cfg(not(target_arch = "wasm32"))]
 fn sigmoid(x: f32) -> f32 {
     1.0 / (1.0 + (-x).exp())
@@ -273,3 +288,22 @@ pub use detector::Output as DetectorOutput;
 pub fn sync(model: &Sam3) -> Result<()> {
     Ok(model.device.synchronize()?)
 }
+
+#[cfg(all(test, not(target_arch = "wasm32")))]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn meta_original_checkpoint_is_named_as_such() {
+        // #3378: a community mirror's `sam3.safetensors` (Meta's key names, C64 RoPE tables) saved as model.safetensors
+        let dir = std::env::temp_dir().join(format!("lc-sam3-original-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let header = r#"{"detector.backbone.vision_backbone.trunk.blocks.0.attn.freqs_cis":{"dtype":"C64","shape":[2],"data_offsets":[0,16]},"tracker.sam_mask_decoder.iou_token.weight":{"dtype":"F32","shape":[2],"data_offsets":[16,24]}}"#;
+        std::fs::rename(weights::tests::write("original", header, 24), dir.join(WEIGHTS_FILE)).unwrap();
+        std::fs::write(dir.join("vocab.json"), "{}").unwrap();
+        std::fs::write(dir.join("merges.txt"), "").unwrap();
+        let err = Sam3::load_cpu(&dir).err().unwrap().to_string();
+        assert!(err.contains("Meta's original checkpoint") && err.contains("facebook/sam3"), "{err}");
+        let _ = std::fs::remove_dir_all(dir);
+    }
+}

crates/segment/src/weights.rs

@@ -16,6 +16,9 @@ use candle_nn::var_builder::SimpleBackend;
 use crate::{Error, Result};
 
 struct Entry {
+    /// `Err`: the header's name for a dtype LightCraft doesn't read (Meta's original checkpoint
+    /// stores its RoPE tables as C64); the tensor is listed, and reading it fails
+    dtype: std::result::Result<DType, String>,
     shape: Vec<usize>,
     start: u64,
     end: u64,
@@ -48,8 +51,8 @@ impl Weights {
                 continue;
             }
             let dtype = match v.get("dtype").and_then(|d| d.as_str()) {
-                Some("F32") => DType::F32,
-                other => return Err(Error::Model(format!("{name}: unsupported dtype {other:?} (the SAM 3 checkpoint is F32)"))),
+                Some("F32") => Ok(DType::F32),
+                other => Err(format!("{other:?}")),
             };
             let shape: Vec<usize> = v
                 .get("shape")
@@ -59,15 +62,18 @@ impl Weights {
             let offsets: Vec<u64> =
                 v.get("data_offsets").and_then(|s| s.as_array()).map(|a| a.iter().filter_map(|x| x.as_u64()).collect()).unwrap_or_default();
             let [start, end] = offsets[..] else { return Err(Error::Model(format!("{name}: bad data_offsets"))) };
-            let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d));
-            if end.checked_sub(start) != want.map(|w| w as u64) {
-                return Err(Error::Model(format!("{name}: size does not match its shape")));
+            // an unreadable tensor's size isn't checked: its bytes are never read
+            if let Ok(dtype) = dtype {
+                let want = shape.iter().try_fold(dtype.size_in_bytes(), |a, d| a.checked_mul(*d));
+                if end.checked_sub(start) != want.map(|w| w as u64) {
+                    return Err(Error::Model(format!("{name}: size does not match its shape")));
+                }
             }
             // the bytes must be in the file (a damaged or truncated download fails here, not later)
             if (8 + n).checked_add(end).is_none_or(|e| e > file_len) {
                 return Err(Error::Model(format!("{}: tensor {name} lies outside the file (incomplete or damaged download?)", path.display())));
             }
-            entries.insert(name, Entry { shape, start, end });
+            entries.insert(name, Entry { dtype, shape, start, end });
         }
         Ok(Arc::new(Self { file: Mutex::new(file), entries, data_start: 8 + n }))
     }
@@ -76,6 +82,11 @@ impl Weights {
         self.entries.contains_key(name)
     }
 
+    /// Whether any tensor's name starts with `prefix`.
+    pub fn has_prefix(&self, prefix: &str) -> bool {
+        self.entries.keys().any(|k| k.starts_with(prefix))
+    }
+
     /// The shape stored in the header for `name`, without reading its data.
     pub fn shape(&self, name: &str) -> Option<&[usize]> {
         self.entries.get(name).map(|e| e.shape.as_slice())
@@ -84,6 +95,9 @@ impl Weights {
     /// Tensor `name` as stored, on the CPU.
     pub fn read(&self, name: &str) -> Result<Tensor> {
         let e = self.entries.get(name).ok_or_else(|| Error::Model(format!("missing tensor `{name}`")))?;
+        if let Err(dtype) = &e.dtype {
+            return Err(Error::Model(format!("{name}: unsupported dtype {dtype} (the SAM 3 checkpoint is F32)")));
+        }
         // checked against the file's length in `open`
         let len = usize::try_from(e.end.saturating_sub(e.start)).map_err(|_| Error::Model(format!("{name}: too large")))?;
         let mut bytes = vec![0u8; len];
@@ -122,10 +136,10 @@ impl SimpleBackend for Backend {
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use super::*;
 
-    fn write(name: &str, header: &str, data_len: usize) -> std::path::PathBuf {
+    pub(crate) fn write(name: &str, header: &str, data_len: usize) -> std::path::PathBuf {
         let p = std::env::temp_dir().join(format!("lc-weights-{name}-{}.safetensors", std::process::id()));
         let mut b = (header.len() as u64).to_le_bytes().to_vec();
         b.extend_from_slice(header.as_bytes());
@@ -147,6 +161,22 @@ mod tests {
         let _ = std::fs::remove_file(p);
     }
 
+    #[test]
+    fn a_tensor_of_another_dtype_fails_when_read_not_when_opened() {
+        // Meta's original checkpoint stores RoPE tables as C64 next to its F32 weights (#3378)
+        let p = write(
+            "c64",
+            r#"{"freqs_cis":{"dtype":"C64","shape":[2],"data_offsets":[0,16]},"a":{"dtype":"F32","shape":[2],"data_offsets":[16,24]}}"#,
+            24,
+        );
+        let w = Weights::open(&p).unwrap();
+        assert!(w.contains("freqs_cis"));
+        let err = w.read("freqs_cis").unwrap_err().to_string();
+        assert!(err.contains("unsupported dtype") && err.contains("C64"), "{err}");
+        assert_eq!(w.read("a").unwrap().dims(), &[2]);
+        let _ = std::fs::remove_file(p);
+    }
+
     #[test]
     fn tensors_must_lie_inside_the_file() {
         let ok = write("ok", r#"{"a":{"dtype":"F32","shape":[2,2],"data_offsets":[0,16]}}"#, 16);

docs/ai-masks.md

@@ -153,7 +153,9 @@ the Mozilla root certificates; no OpenSSL, `ring` or `aws-lc`, nothing compiled
 ### By hand
 
 Put `model.safetensors`, `vocab.json` and `merges.txt` from `facebook/sam3` in the model folder:
-`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. Developers
+`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. It must be
+the Hugging Face format (3,439,938,512 bytes): `sam3.safetensors` from community mirrors, renamed, is
+Meta's original checkpoint, and LightCraft says so instead of loading it. Developers
 can use the installer scripts, which download from Hugging Face with your token and verify the
 weights:
 

docs/parity-checklist.md

@@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).
 | LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) |
 | LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) |
 | LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) |
-| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); the stored grid is enlarged along the photo's edges (a guided filter of its logits by log luminance, so no outline spills onto the background under a raised Exposure, #3272). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); the stored grid is enlarged along the photo's edges (a guided filter of its logits by log luminance, so no outline spills onto the background under a raised Exposure, #3272). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md; Meta's original checkpoint installed in place of the Hugging Face one is named as such, #3378). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
 | LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic |
 | LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty |
 | LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); with the mask overlay hidden (O / the eye, or after a mask slider hid it) or a colour overlay at opacity 0 it is not tinted while painting either, so dodging and burning is judged on the photo, and its effect shows once the stroke is committed (#2549); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); oversized path segments skip intermediate dabs instead of allocating a million samples; Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt |

docs/whats-new.md

@@ -9,6 +9,12 @@
   edit history now keep the first 64 too, in order, and the extra ones are skipped as they are read. Masks made in
   the app (up to 5 samples) are unchanged.
 
+### A SAM 3 model in the wrong format says so
+- Object and Describe need `model.safetensors` from `facebook/sam3` in the Hugging Face format. The
+  `sam3.safetensors` on community mirrors is Meta's original checkpoint, with other tensor names; installed in its
+  place, AI masks failed with "unsupported dtype C64". LightCraft now says that the file is Meta's original
+  checkpoint and which file it needs instead.
+
 ## 2026-10-29
 
 ### Object and Describe masks show their pins