Merge PR #2067: LR-MASK-OBJECTS: SAM 3 falls back to the CPU when the GPU can't run it (#1974)

crates/engine/src/segment/mod.rs

@@ -75,6 +75,7 @@ pub(crate) struct Input {
 }
 
 /// What a detail pass re-runs.
+#[derive(Clone)]
 #[cfg_attr(not(feature = "sam"), allow(dead_code))]
 pub(crate) enum Prompt {
     Clicks(Vec<Click>),
@@ -440,7 +441,11 @@ impl Session {
         let job = self.segment_job(id, kind, Tag::Wait, tx, region)?;
         self.segmenter.worker.submit(job)?;
         match rx.recv_timeout(WAIT) {
-            Ok(o) => o.result,
+            Ok(o) => {
+                // (shown by the next poll in an app; logged anyway)
+                self.segmenter.messages.extend(o.notice);
+                o.result
+            }
             Err(_) => Err("the AI model took too long; try again".into()),
         }
     }
@@ -579,7 +584,7 @@ impl Session {
                 self.segment_queue(id, kind, tag, Some(region))?;
             } else {
                 let patch = self.segment_wait(id, kind, Some(region))?;
-                let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false };
+                let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false, notice: None };
                 let mut polled = Polled::default();
                 self.apply_outcome(outcome, &mut polled);
             }
@@ -618,6 +623,7 @@ impl Session {
         if o.superseded {
             return;
         }
+        polled.messages.extend(o.notice);
         match o.tag {
             Tag::Wait => {}
             Tag::Prepare => {
@@ -711,6 +717,18 @@ impl Session {
 mod tests {
     use super::*;
 
+    #[cfg(feature = "sam")]
+    #[test]
+    fn a_worker_notice_reaches_the_user_with_the_result() {
+        // the job after which the model runs on the CPU (the GPU failed) says so
+        let mut s = Session::with_demo();
+        let notice = Some(worker::GPU_FAILED.to_string());
+        s.segmenter.results.0.send(worker::Outcome { tag: Tag::Prepare, result: Ok(None), superseded: false, notice }).unwrap();
+        let polled = s.segment_poll();
+        assert_eq!(polled.messages, vec![worker::GPU_FAILED.to_string()]);
+        assert!(s.segment_poll().messages.is_empty(), "once");
+    }
+
     fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask {
         let l: Vec<f32> =
             (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect();

crates/engine/src/segment/worker.rs

@@ -7,6 +7,13 @@
 //! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy
 //! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded
 //! after [`IDLE_UNLOAD`] without requests (it holds several GB).
+//!
+//! Where the model can run on the GPU (Metal on macOS), a job that fails there (a panic or an
+//! error from candle, e.g. a kernel the GPU's compiler can't build, issue #1974) frees the
+//! model and runs once more on the CPU. When that works, the model stays on the CPU for the rest
+//! of the session and the job's outcome says so; when it fails too, the GPU wasn't the cause and
+//! the next job uses it again. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep the model on
+//! the CPU from the start.
 
 use std::collections::VecDeque;
 use std::panic::AssertUnwindSafe;
@@ -29,6 +36,7 @@ use crate::media::RenderJob;
 pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60);
 
 /// What to compute.
+#[derive(Clone)]
 pub enum Kind {
     /// Load the model and encode the photo (so the first click is fast).
     Prepare,
@@ -42,6 +50,12 @@ pub enum Kind {
     /// Panics (tests of the worker's recovery).
     #[cfg(test)]
     Panic,
+    /// Fails like the model on a GPU that can't run it (a panic or a candle error), and
+    /// succeeds on the CPU (tests of the CPU fallback).
+    #[cfg(test)]
+    FailOnGpu {
+        panic: bool,
+    },
 }
 
 /// One request.
@@ -65,6 +79,70 @@ pub struct Outcome {
     pub result: Result<Option<SegMask>, String>,
     /// A newer request for the same selection came in; this one was skipped.
     pub superseded: bool,
+    /// Something the user should know besides the result ([`GPU_FAILED`]).
+    pub notice: Option<String>,
+}
+
+/// The notice of the job after which the model runs on the CPU.
+pub const GPU_FAILED: &str =
+    "The GPU couldn't run the AI mask model, so it runs on the CPU until LightCraft restarts (slower). Details are in the log.";
+
+/// Where the model is loaded.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum Placement {
+    /// On the GPU when there is one (Metal on macOS); a job that fails there is retried on the CPU.
+    Gpu,
+    /// On the CPU: this build has no GPU path, the environment switches the GPU off, or the model
+    /// failed on it.
+    Cpu,
+}
+
+impl Placement {
+    fn initial() -> Self {
+        if !lightcraft_segment::GPU_BUILD {
+            return Placement::Cpu;
+        }
+        if let Some(why) = lightcraft_gpu::env_off_reason() {
+            log::info!("SAM 3 runs on the CPU ({why})");
+            return Placement::Cpu;
+        }
+        Placement::Gpu
+    }
+}
+
+/// Why a job failed.
+#[derive(Debug)]
+enum Failure {
+    /// The model failed (a candle error, or a panic): on the GPU, the CPU may run it.
+    Model(String),
+    /// Anything else (no model files, the photo, a remote worker): another device won't help.
+    Other(String),
+}
+
+impl Failure {
+    fn message(self) -> String {
+        match self {
+            Failure::Model(m) | Failure::Other(m) => m,
+        }
+    }
+}
+
+impl From<String> for Failure {
+    fn from(m: String) -> Self {
+        Failure::Other(m)
+    }
+}
+
+impl From<&str> for Failure {
+    fn from(m: &str) -> Self {
+        Failure::Other(m.into())
+    }
+}
+
+impl From<lightcraft_segment::Error> for Failure {
+    fn from(e: lightcraft_segment::Error) -> Self {
+        if matches!(e, lightcraft_segment::Error::Candle(_)) { Failure::Model(e.to_string()) } else { Failure::Other(e.to_string()) }
+    }
 }
 
 /// Counters the session reads without locking.
@@ -159,7 +237,7 @@ impl Worker {
         // (re)start the thread
         let (tx, rx) = mpsc::channel();
         let shared = self.shared.clone();
-        match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) {
+        match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD, Placement::initial())) {
             Ok(t) => {
                 self.thread = Some(t);
                 self.tx = Some(tx.clone());
@@ -191,7 +269,7 @@ fn supersedes(later: &Job, job: &Job) -> bool {
     }
 }
 
-fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
+fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration, mut placement: Placement) {
     let mut state = State::default();
     let mut queue: VecDeque<Job> = VecDeque::new();
     loop {
@@ -219,41 +297,78 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
         let done = Done(if is_detail { &shared.detail } else { &shared.pending });
         if queue.iter().any(|later| supersedes(later, &job)) {
             drop(done);
-            let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true });
+            let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true, notice: None });
             continue;
         }
-        let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) {
-            Ok(r) => r,
-            Err(p) => {
-                let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
-                log::error!("SAM 3 panicked: {why}");
-                // don't trust what the model kept: load it again next time
-                state = State::default();
-                Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string())
-            }
+        // what a retry on the CPU needs (running the job takes them)
+        let retry = (placement == Placement::Gpu && remote::configured().is_none()).then(|| (job.kind.clone(), job.render.clone()));
+        let mut result = attempt(&mut state, shared, &mut job, placement);
+        let mut notice = None;
+        let gpu_failed = match &result {
+            Err(Failure::Model(why)) => Some(why.clone()),
+            _ => None,
         };
+        if let (Some(why), Some((kind, render))) = (gpu_failed, retry) {
+            // the GPU may be the cause (e.g. a kernel its compiler can't build, issue #1974): free
+            // the model there and try once on the CPU
+            log::warn!("SAM 3 failed on the GPU ({why}); trying the CPU");
+            state = State::default();
+            (job.kind, job.render) = (kind, render);
+            match attempt(&mut state, shared, &mut job, Placement::Cpu) {
+                Ok(r) => {
+                    log::warn!("SAM 3 runs on the CPU from now on: it failed on the GPU and works on the CPU");
+                    placement = Placement::Cpu;
+                    notice = Some(GPU_FAILED.to_string());
+                    result = Ok(r);
+                }
+                Err(e) => {
+                    // the CPU fails too, so the GPU isn't the cause: the next job uses it again
+                    log::warn!("SAM 3 failed on the CPU too: {}", e.message());
+                    state = State::default();
+                }
+            }
+        }
+        let result = result.map_err(Failure::message);
         if result.is_err() && !is_detail {
             state.cache = None;
         }
         shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst);
         drop(done);
-        let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
+        let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false, notice });
     }
 }
 
-/// The model for `dir`, loading it when needed.
-fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> {
+/// Run `job` with the model on `placement`. A panic is a [`Failure::Model`] and drops what the
+/// model kept.
+fn attempt(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {
+    match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(state, shared, job, placement))) {
+        Ok(r) => r,
+        Err(p) => {
+            let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_default();
+            log::error!("SAM 3 panicked: {why}");
+            // don't trust what the model kept: load it again next time
+            *state = State::default();
+            Err(Failure::Model("the AI model failed unexpectedly (it will be reloaded on the next try)".into()))
+        }
+    }
+}
+
+/// The model for `dir`, loading it on `placement` when needed.
+fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path, placement: Placement) -> Result<&'a mut Sam3, Failure> {
     if state.model.as_ref().is_none_or(|(d, _)| d != dir) {
         state.model = None;
         state.cache = None;
         let _busy = Flag::raise(&shared.encoding);
         let t = web_time::Instant::now();
-        let m = Sam3::load(dir).map_err(|e| e.to_string())?;
-        log::info!("SAM 3 loaded in {:?}", t.elapsed());
+        let m = match placement {
+            Placement::Gpu => Sam3::load(dir)?,
+            Placement::Cpu => Sam3::load_cpu(dir)?,
+        };
+        log::info!("SAM 3 loaded in {:?} ({})", t.elapsed(), if m.device().is_cpu() { "CPU" } else { "GPU" });
         state.model = Some((dir.to_path_buf(), m));
         shared.loaded.store(true, Ordering::SeqCst);
     }
-    state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string())
+    state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".into())
 }
 
 /// Render the model input.
@@ -275,16 +390,16 @@ fn render(job: &mut Job) -> Result<Input, String> {
 }
 
 /// The model and the encoded photo `job.key` (rendering and encoding it when needed).
-fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> {
-    model(state, shared, &job.dir)?;
+fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<(&'a mut Sam3, &'a mut Encoded), Failure> {
+    model(state, shared, &job.dir, placement)?;
     if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) {
         state.cache = None;
         let input = render(job)?;
         let _busy = Flag::raise(&shared.encoding);
         let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?;
         let t = web_time::Instant::now();
-        let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
-        m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?;
+        let mut enc = m.encode(&input.rgb, input.w, input.h)?;
+        m.prepare_clicks(&mut enc)?;
         log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
         state.cache = Some((job.key, enc));
     }
@@ -328,7 +443,7 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)
             }
         }
         #[cfg(test)]
-        Kind::Panic => return Err("test panic request".into()),
+        Kind::Panic | Kind::FailOnGpu { .. } => return Err("test panic request".into()),
     };
     let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } };
     let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation };
@@ -342,37 +457,44 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str)
     }))
 }
 
-fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
+fn run_job(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<Option<SegMask>, Failure> {
     if let Some(endpoint) = remote::configured() {
         let result = remote_job(state, shared, job, &endpoint);
         if result.is_err() {
             state.remote = None;
         }
-        return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."));
+        return result.map_err(|e| Failure::Other(format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")));
     }
     let side = lightcraft_segment::MASK_SIDE;
     match std::mem::replace(&mut job.kind, Kind::Prepare) {
         Kind::Prepare => {
-            encoded(state, shared, job)?;
+            encoded(state, shared, job, placement)?;
             Ok(None)
         }
         Kind::Clicks(clicks) => {
-            let (m, enc) = encoded(state, shared, job)?;
-            let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?;
+            let (m, enc) = encoded(state, shared, job, placement)?;
+            let p = m.segment_clicks(enc, &clicks)?;
             Ok(Some(SegMask::from_logits(side, &p.logits)))
         }
         Kind::Text(text) => {
-            let (m, enc) = encoded(state, shared, job)?;
+            let (m, enc) = encoded(state, shared, job, placement)?;
             Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l)))
         }
         #[cfg(test)]
         #[allow(clippy::panic)]
         Kind::Panic => panic!("a test panic"),
+        #[cfg(test)]
+        #[allow(clippy::panic)]
+        Kind::FailOnGpu { panic } => match placement {
+            Placement::Gpu if panic => panic!("a test panic on the GPU"),
+            Placement::Gpu => Err(Failure::Model("a test error on the GPU".into())),
+            Placement::Cpu => Ok(None),
+        },
         Kind::Detail { prompt, region } => {
             let input = render(job)?;
-            let m = model(state, shared, &job.dir)?;
+            let m = model(state, shared, &job.dir, placement)?;
             let t = web_time::Instant::now();
-            let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?;
+            let mut enc = m.encode(&input.rgb, input.w, input.h)?;
             let (rw, rh) = (region[2] - region[0], region[3] - region[1]);
             if !(rw > 0.0 && rh > 0.0) {
                 return Ok(None);
@@ -388,7 +510,7 @@ fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<S
                     if !inside.iter().any(|c| c.positive) {
                         return Ok(None);
                     }
-                    Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits)
+                    Some(m.segment_clicks(&mut enc, &inside)?.logits)
                 }
                 Prompt::Text(text) => text_logits(m, &mut enc, &text)?,
             };
@@ -438,7 +560,7 @@ mod tests {
         jtx.send(job(Tag::Wait, &tx)).unwrap();
         shared.pending.store(2, Ordering::SeqCst);
         drop(jtx);
-        run(&jrx, &shared, Duration::from_secs(60));
+        run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);
         let first = rx.recv().unwrap();
         assert!(first.result.unwrap_err().contains("failed unexpectedly"));
         let second = rx.recv().unwrap();
@@ -447,6 +569,72 @@ mod tests {
         assert!(!shared.encoding.load(Ordering::SeqCst));
     }
 
+    /// Run `kinds` in turn on a worker that starts on `placement`; their outcomes.
+    fn run_kinds(placement: Placement, kinds: Vec<Kind>) -> Vec<Outcome> {
+        let (tx, rx) = mpsc::channel();
+        let (jtx, jrx) = mpsc::channel();
+        let shared = Shared::default();
+        shared.pending.store(kinds.len(), Ordering::SeqCst);
+        for kind in kinds {
+            let mut j = job(Tag::Wait, &tx);
+            j.kind = kind;
+            jtx.send(j).unwrap();
+        }
+        drop(jtx);
+        run(&jrx, &shared, Duration::from_secs(60), placement);
+        assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset");
+        assert!(!shared.encoding.load(Ordering::SeqCst));
+        rx.try_iter().collect()
+    }
+
+    #[test]
+    fn a_job_that_panics_on_the_gpu_runs_on_the_cpu_and_so_do_the_next_ones() {
+        // issue #1974: candle panics compiling a Metal kernel the GPU's compiler can't build
+        let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: true }, Kind::FailOnGpu { panic: true }]);
+        assert_eq!(out.len(), 2);
+        assert!(matches!(out[0].result, Ok(None)), "retried on the CPU: {:?}", out[0].result.as_ref().err());
+        assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));
+        assert!(matches!(out[1].result, Ok(None)), "the next job starts on the CPU");
+        assert_eq!(out[1].notice, None, "said once");
+    }
+
+    #[test]
+    fn a_model_error_on_the_gpu_is_retried_on_the_cpu_too() {
+        let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: false }]);
+        assert!(matches!(out[0].result, Ok(None)));
+        assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED));
+    }
+
+    #[test]
+    fn a_job_that_fails_on_the_cpu_too_keeps_the_gpu() {
+        let out = run_kinds(Placement::Gpu, vec![Kind::Panic, Kind::FailOnGpu { panic: true }]);
+        assert!(out[0].result.as_ref().unwrap_err().contains("failed unexpectedly"));
+        assert_eq!(out[0].notice, None);
+        // not the GPU's fault: the next job tries the GPU first again
+        assert!(matches!(out[1].result, Ok(None)));
+        assert_eq!(out[1].notice.as_deref(), Some(GPU_FAILED));
+    }
+
+    #[test]
+    fn a_worker_on_the_cpu_never_tries_the_gpu() {
+        // no GPU in this build, or LIGHTCRAFT_GPU=0 / LIGHTCRAFT_GPU_BACKEND=off
+        let out = run_kinds(Placement::Cpu, vec![Kind::FailOnGpu { panic: true }]);
+        assert!(matches!(out[0].result, Ok(None)));
+        assert_eq!(out[0].notice, None);
+        if !lightcraft_segment::GPU_BUILD {
+            assert_eq!(Placement::initial(), Placement::Cpu);
+        }
+    }
+
+    #[test]
+    fn missing_model_files_are_not_retried_on_the_cpu() {
+        let out = run_kinds(Placement::Gpu, vec![Kind::Prepare]);
+        assert!(out[0].result.as_ref().unwrap_err().contains("not found"));
+        assert_eq!(out[0].notice, None);
+        let missing = lightcraft_segment::Error::Missing(PathBuf::from("models"));
+        assert!(matches!(Failure::from(missing), Failure::Other(_)));
+    }
+
     #[test]
     fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() {
         let (tx, rx) = mpsc::channel();
@@ -464,7 +652,7 @@ mod tests {
         drop(jtx);
         let shared = Shared::default();
         shared.pending.store(3, Ordering::SeqCst);
-        run(&jrx, &shared, Duration::from_secs(60));
+        run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu);
         let out: Vec<Outcome> = rx.try_iter().collect();
         assert_eq!(out.len(), 3);
         assert!(out[0].superseded && out[1].superseded && !out[2].superseded);

crates/gpu/src/lib.rs

@@ -75,12 +75,8 @@ fn record_fallback(reason: String) {
 /// E.g. "disabled by LIGHTCRAFT_GPU=0", "software adapter (llvmpipe …) skipped", "device lost …".
 /// Never blocks.
 pub fn unavailable_reason() -> Option<String> {
-    if env_disabled() {
-        #[cfg(not(target_arch = "wasm32"))]
-        if backend::env_off() {
-            return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off".into());
-        }
-        return Some("disabled by LIGHTCRAFT_GPU=0".into());
+    if let Some(why) = env_off_reason() {
+        return Some(why.into());
     }
     if !ENABLED.load(Ordering::Relaxed) {
         return Some("disabled by the GPU rendering preference (app.gpu)".into());
@@ -245,12 +241,26 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool {
 /// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
 #[cfg(not(target_arch = "wasm32"))]
 pub(crate) fn switched_off() -> Option<String> {
-    if env_disabled() {
-        return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
+    if let Some(why) = env_off_reason() {
+        return Some(why.into());
     }
     (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())
 }
 
+/// Why the environment switches the GPU off for the whole process (`LIGHTCRAFT_GPU=0` or
+/// `LIGHTCRAFT_GPU_BACKEND=off`), or `None`. Code with a GPU path of its own (the SAM 3 model on
+/// Metal) follows it too.
+pub fn env_off_reason() -> Option<&'static str> {
+    if !env_disabled() {
+        return None;
+    }
+    #[cfg(not(target_arch = "wasm32"))]
+    if backend::env_off() {
+        return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off");
+    }
+    Some("disabled by LIGHTCRAFT_GPU=0")
+}
+
 /// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.
 fn env_disabled() -> bool {
     static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();

crates/segment/src/lib.rs

@@ -87,6 +87,11 @@ pub fn is_model_dir(dir: &Path) -> bool {
     dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file()
 }
 
+#[cfg(not(target_arch = "wasm32"))]
+/// Whether [`best_device`] can be a GPU in this build (Metal on macOS). Where it can, a caller that
+/// sees the model fail there can load it again with [`Sam3::load_cpu`].
+pub const GPU_BUILD: bool = cfg!(target_os = "macos");
+
 #[cfg(not(target_arch = "wasm32"))]
 /// The best device here: Metal on macOS (when a GPU is available), else the CPU.
 pub fn best_device() -> Device {
@@ -134,6 +139,11 @@ impl Sam3 {
         Self::load_on(dir, best_device())
     }
 
+    /// [`Sam3::load`] on the CPU, whatever GPU there is.
+    pub fn load_cpu(dir: &Path) -> Result<Self> {
+        Self::load_on(dir, Device::Cpu)
+    }
+
     pub fn load_on(dir: &Path, device: Device) -> Result<Self> {
         if !is_model_dir(dir) {
             return Err(Error::Missing(dir.to_path_buf()));

crates/segment/src/remote.rs

@@ -108,10 +108,10 @@ impl Client {
     }
 }
 
-pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
+pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> crate::Result<Option<Vec<f32>>> {
     let mut merged: Option<Vec<f32>> = None;
     for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
-        if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
+        if let Some(p) = model.segment_text(enc, phrase, 0.5)? {
             if let Some(m) = &mut merged {
                 for (a, b) in m.iter_mut().zip(p.logits) {
                     *a = a.max(b);
@@ -130,7 +130,7 @@ fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result<
             model.prepare_clicks(enc).map_err(|e| e.to_string())?;
             Ok(None)
         }
-        Operation::Text(text) => text_logits(model, enc, &text),
+        Operation::Text(text) => text_logits(model, enc, &text).map_err(|e| e.to_string()),
         Operation::Clicks(points) => {
             let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect();
             Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits))

docs/ai-masks.md

@@ -2,7 +2,8 @@
 
 LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta 2025), run in
 pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU
-through Metal on macOS, on the CPU elsewhere (for now).
+through Metal on macOS, on the CPU elsewhere (for now). If the Mac's GPU can't run it, it runs on
+the CPU (see the worker below).
 
 **The model is optional.** It is not part of LightCraft, and nothing requires it: without it,
 Object and Describe offer to download it, every other feature (and every existing mask, see
@@ -228,6 +229,13 @@ wait and return the result (when built with the `sam` feature; the default CLI h
   encoded photo; the session sends it jobs and receives results over channels, never holding a
   lock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes an
   error and the model is reloaded on the next request; busy counters are reset by drop guards.
+- GPU fallback (issue #1974): a job that fails on the GPU (a panic or a candle error, e.g. an
+  AMD GPU in an Intel Mac whose Metal compiler crashes on a matmul kernel) frees the model and runs
+  once more on the CPU. If that works, the model stays on the CPU until LightCraft restarts and
+  the user is told once ("The GPU couldn't run the AI mask model…"); if the CPU fails too, the
+  error is reported and the next job tries the GPU again. Missing or damaged model files and
+  remote-worker errors are not retried. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep
+  the model on the CPU from the start.
 - Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered),
   each at most 1024² logits; more than 64 clicks are never sent to the model.
 - Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python

docs/gpu-pipeline.md

@@ -43,6 +43,8 @@ Overrides (read once at launch):
 - else `WGPU_BACKEND` (wgpu's own variable, same names) — before #136 only the window honoured it,
   the compute device always used Vulkan + DX12 (+ Metal).
 - `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected).
+- Both `LIGHTCRAFT_GPU=0` and `LIGHTCRAFT_GPU_BACKEND=off` also keep AI Denoise, the face models
+  and the AI mask model (SAM 3 on Metal) on the CPU.
 
 **DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders with
 FXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu's

docs/parity-checklist.md

@@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts).
 | LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) |
 | LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) |
 | LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) |
-| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
+| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) |
 | LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic |
 | LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty |
 | LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt |