diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs index c955dcc01..b69a6dca2 100644 --- a/crates/engine/src/segment/mod.rs +++ b/crates/engine/src/segment/mod.rs @@ -75,6 +75,7 @@ pub(crate) struct Input { } /// What a detail pass re-runs. +#[derive(Clone)] #[cfg_attr(not(feature = "sam"), allow(dead_code))] pub(crate) enum Prompt { Clicks(Vec), @@ -440,7 +441,11 @@ impl Session { let job = self.segment_job(id, kind, Tag::Wait, tx, region)?; self.segmenter.worker.submit(job)?; match rx.recv_timeout(WAIT) { - Ok(o) => o.result, + Ok(o) => { + // (shown by the next poll in an app; logged anyway) + self.segmenter.messages.extend(o.notice); + o.result + } Err(_) => Err("the AI model took too long; try again".into()), } } @@ -579,7 +584,7 @@ impl Session { self.segment_queue(id, kind, tag, Some(region))?; } else { let patch = self.segment_wait(id, kind, Some(region))?; - let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false }; + let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false, notice: None }; let mut polled = Polled::default(); self.apply_outcome(outcome, &mut polled); } @@ -618,6 +623,7 @@ impl Session { if o.superseded { return; } + polled.messages.extend(o.notice); match o.tag { Tag::Wait => {} Tag::Prepare => { @@ -711,6 +717,18 @@ impl Session { mod tests { use super::*; + #[cfg(feature = "sam")] + #[test] + fn a_worker_notice_reaches_the_user_with_the_result() { + // the job after which the model runs on the CPU (the GPU failed) says so + let mut s = Session::with_demo(); + let notice = Some(worker::GPU_FAILED.to_string()); + s.segmenter.results.0.send(worker::Outcome { tag: Tag::Prepare, result: Ok(None), superseded: false, notice }).unwrap(); + let polled = s.segment_poll(); + assert_eq!(polled.messages, vec![worker::GPU_FAILED.to_string()]); + assert!(s.segment_poll().messages.is_empty(), "once"); + } + fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask { let l: Vec = (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect(); diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs index 5769fd2ae..efbb1c21c 100644 --- a/crates/engine/src/segment/worker.rs +++ b/crates/engine/src/segment/worker.rs @@ -7,6 +7,13 @@ //! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy //! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded //! after [`IDLE_UNLOAD`] without requests (it holds several GB). +//! +//! Where the model can run on the GPU (Metal on macOS), a job that fails there (a panic or an +//! error from candle, e.g. a kernel the GPU's compiler can't build, issue #1974) frees the +//! model and runs once more on the CPU. When that works, the model stays on the CPU for the rest +//! of the session and the job's outcome says so; when it fails too, the GPU wasn't the cause and +//! the next job uses it again. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep the model on +//! the CPU from the start. use std::collections::VecDeque; use std::panic::AssertUnwindSafe; @@ -29,6 +36,7 @@ use crate::media::RenderJob; pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60); /// What to compute. +#[derive(Clone)] pub enum Kind { /// Load the model and encode the photo (so the first click is fast). Prepare, @@ -42,6 +50,12 @@ pub enum Kind { /// Panics (tests of the worker's recovery). #[cfg(test)] Panic, + /// Fails like the model on a GPU that can't run it (a panic or a candle error), and + /// succeeds on the CPU (tests of the CPU fallback). + #[cfg(test)] + FailOnGpu { + panic: bool, + }, } /// One request. @@ -65,6 +79,70 @@ pub struct Outcome { pub result: Result, String>, /// A newer request for the same selection came in; this one was skipped. pub superseded: bool, + /// Something the user should know besides the result ([`GPU_FAILED`]). + pub notice: Option, +} + +/// The notice of the job after which the model runs on the CPU. +pub const GPU_FAILED: &str = + "The GPU couldn't run the AI mask model, so it runs on the CPU until LightCraft restarts (slower). Details are in the log."; + +/// Where the model is loaded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Placement { + /// On the GPU when there is one (Metal on macOS); a job that fails there is retried on the CPU. + Gpu, + /// On the CPU: this build has no GPU path, the environment switches the GPU off, or the model + /// failed on it. + Cpu, +} + +impl Placement { + fn initial() -> Self { + if !lightcraft_segment::GPU_BUILD { + return Placement::Cpu; + } + if let Some(why) = lightcraft_gpu::env_off_reason() { + log::info!("SAM 3 runs on the CPU ({why})"); + return Placement::Cpu; + } + Placement::Gpu + } +} + +/// Why a job failed. +#[derive(Debug)] +enum Failure { + /// The model failed (a candle error, or a panic): on the GPU, the CPU may run it. + Model(String), + /// Anything else (no model files, the photo, a remote worker): another device won't help. + Other(String), +} + +impl Failure { + fn message(self) -> String { + match self { + Failure::Model(m) | Failure::Other(m) => m, + } + } +} + +impl From for Failure { + fn from(m: String) -> Self { + Failure::Other(m) + } +} + +impl From<&str> for Failure { + fn from(m: &str) -> Self { + Failure::Other(m.into()) + } +} + +impl From for Failure { + fn from(e: lightcraft_segment::Error) -> Self { + if matches!(e, lightcraft_segment::Error::Candle(_)) { Failure::Model(e.to_string()) } else { Failure::Other(e.to_string()) } + } } /// Counters the session reads without locking. @@ -159,7 +237,7 @@ impl Worker { // (re)start the thread let (tx, rx) = mpsc::channel(); let shared = self.shared.clone(); - match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) { + match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD, Placement::initial())) { Ok(t) => { self.thread = Some(t); self.tx = Some(tx.clone()); @@ -191,7 +269,7 @@ fn supersedes(later: &Job, job: &Job) -> bool { } } -fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { +fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration, mut placement: Placement) { let mut state = State::default(); let mut queue: VecDeque = VecDeque::new(); loop { @@ -219,41 +297,78 @@ fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { let done = Done(if is_detail { &shared.detail } else { &shared.pending }); if queue.iter().any(|later| supersedes(later, &job)) { drop(done); - let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true }); + let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true, notice: None }); continue; } - let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) { - Ok(r) => r, - Err(p) => { - let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::().cloned()).unwrap_or_default(); - log::error!("SAM 3 panicked: {why}"); - // don't trust what the model kept: load it again next time - state = State::default(); - Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string()) - } + // what a retry on the CPU needs (running the job takes them) + let retry = (placement == Placement::Gpu && remote::configured().is_none()).then(|| (job.kind.clone(), job.render.clone())); + let mut result = attempt(&mut state, shared, &mut job, placement); + let mut notice = None; + let gpu_failed = match &result { + Err(Failure::Model(why)) => Some(why.clone()), + _ => None, }; + if let (Some(why), Some((kind, render))) = (gpu_failed, retry) { + // the GPU may be the cause (e.g. a kernel its compiler can't build, issue #1974): free + // the model there and try once on the CPU + log::warn!("SAM 3 failed on the GPU ({why}); trying the CPU"); + state = State::default(); + (job.kind, job.render) = (kind, render); + match attempt(&mut state, shared, &mut job, Placement::Cpu) { + Ok(r) => { + log::warn!("SAM 3 runs on the CPU from now on: it failed on the GPU and works on the CPU"); + placement = Placement::Cpu; + notice = Some(GPU_FAILED.to_string()); + result = Ok(r); + } + Err(e) => { + // the CPU fails too, so the GPU isn't the cause: the next job uses it again + log::warn!("SAM 3 failed on the CPU too: {}", e.message()); + state = State::default(); + } + } + } + let result = result.map_err(Failure::message); if result.is_err() && !is_detail { state.cache = None; } shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst); drop(done); - let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false }); + let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false, notice }); } } -/// The model for `dir`, loading it when needed. -fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> { +/// Run `job` with the model on `placement`. A panic is a [`Failure::Model`] and drops what the +/// model kept. +fn attempt(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result, Failure> { + match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(state, shared, job, placement))) { + Ok(r) => r, + Err(p) => { + let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::().cloned()).unwrap_or_default(); + log::error!("SAM 3 panicked: {why}"); + // don't trust what the model kept: load it again next time + *state = State::default(); + Err(Failure::Model("the AI model failed unexpectedly (it will be reloaded on the next try)".into())) + } + } +} + +/// The model for `dir`, loading it on `placement` when needed. +fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path, placement: Placement) -> Result<&'a mut Sam3, Failure> { if state.model.as_ref().is_none_or(|(d, _)| d != dir) { state.model = None; state.cache = None; let _busy = Flag::raise(&shared.encoding); let t = web_time::Instant::now(); - let m = Sam3::load(dir).map_err(|e| e.to_string())?; - log::info!("SAM 3 loaded in {:?}", t.elapsed()); + let m = match placement { + Placement::Gpu => Sam3::load(dir)?, + Placement::Cpu => Sam3::load_cpu(dir)?, + }; + log::info!("SAM 3 loaded in {:?} ({})", t.elapsed(), if m.device().is_cpu() { "CPU" } else { "GPU" }); state.model = Some((dir.to_path_buf(), m)); shared.loaded.store(true, Ordering::SeqCst); } - state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string()) + state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".into()) } /// Render the model input. @@ -275,16 +390,16 @@ fn render(job: &mut Job) -> Result { } /// The model and the encoded photo `job.key` (rendering and encoding it when needed). -fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> { - model(state, shared, &job.dir)?; +fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result<(&'a mut Sam3, &'a mut Encoded), Failure> { + model(state, shared, &job.dir, placement)?; if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) { state.cache = None; let input = render(job)?; let _busy = Flag::raise(&shared.encoding); let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?; let t = web_time::Instant::now(); - let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; - m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?; + let mut enc = m.encode(&input.rgb, input.w, input.h)?; + m.prepare_clicks(&mut enc)?; log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed()); state.cache = Some((job.key, enc)); } @@ -328,7 +443,7 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) } } #[cfg(test)] - Kind::Panic => return Err("test panic request".into()), + Kind::Panic | Kind::FailOnGpu { .. } => return Err("test panic request".into()), }; let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } }; let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation }; @@ -342,37 +457,44 @@ fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) })) } -fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result, String> { +fn run_job(state: &mut State, shared: &Shared, job: &mut Job, placement: Placement) -> Result, Failure> { if let Some(endpoint) = remote::configured() { let result = remote_job(state, shared, job, &endpoint); if result.is_err() { state.remote = None; } - return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")); + return result.map_err(|e| Failure::Other(format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."))); } let side = lightcraft_segment::MASK_SIDE; match std::mem::replace(&mut job.kind, Kind::Prepare) { Kind::Prepare => { - encoded(state, shared, job)?; + encoded(state, shared, job, placement)?; Ok(None) } Kind::Clicks(clicks) => { - let (m, enc) = encoded(state, shared, job)?; - let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?; + let (m, enc) = encoded(state, shared, job, placement)?; + let p = m.segment_clicks(enc, &clicks)?; Ok(Some(SegMask::from_logits(side, &p.logits))) } Kind::Text(text) => { - let (m, enc) = encoded(state, shared, job)?; + let (m, enc) = encoded(state, shared, job, placement)?; Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l))) } #[cfg(test)] #[allow(clippy::panic)] Kind::Panic => panic!("a test panic"), + #[cfg(test)] + #[allow(clippy::panic)] + Kind::FailOnGpu { panic } => match placement { + Placement::Gpu if panic => panic!("a test panic on the GPU"), + Placement::Gpu => Err(Failure::Model("a test error on the GPU".into())), + Placement::Cpu => Ok(None), + }, Kind::Detail { prompt, region } => { let input = render(job)?; - let m = model(state, shared, &job.dir)?; + let m = model(state, shared, &job.dir, placement)?; let t = web_time::Instant::now(); - let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; + let mut enc = m.encode(&input.rgb, input.w, input.h)?; let (rw, rh) = (region[2] - region[0], region[3] - region[1]); if !(rw > 0.0 && rh > 0.0) { return Ok(None); @@ -388,7 +510,7 @@ fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result text_logits(m, &mut enc, &text)?, }; @@ -438,7 +560,7 @@ mod tests { jtx.send(job(Tag::Wait, &tx)).unwrap(); shared.pending.store(2, Ordering::SeqCst); drop(jtx); - run(&jrx, &shared, Duration::from_secs(60)); + run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu); let first = rx.recv().unwrap(); assert!(first.result.unwrap_err().contains("failed unexpectedly")); let second = rx.recv().unwrap(); @@ -447,6 +569,72 @@ mod tests { assert!(!shared.encoding.load(Ordering::SeqCst)); } + /// Run `kinds` in turn on a worker that starts on `placement`; their outcomes. + fn run_kinds(placement: Placement, kinds: Vec) -> Vec { + let (tx, rx) = mpsc::channel(); + let (jtx, jrx) = mpsc::channel(); + let shared = Shared::default(); + shared.pending.store(kinds.len(), Ordering::SeqCst); + for kind in kinds { + let mut j = job(Tag::Wait, &tx); + j.kind = kind; + jtx.send(j).unwrap(); + } + drop(jtx); + run(&jrx, &shared, Duration::from_secs(60), placement); + assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset"); + assert!(!shared.encoding.load(Ordering::SeqCst)); + rx.try_iter().collect() + } + + #[test] + fn a_job_that_panics_on_the_gpu_runs_on_the_cpu_and_so_do_the_next_ones() { + // issue #1974: candle panics compiling a Metal kernel the GPU's compiler can't build + let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: true }, Kind::FailOnGpu { panic: true }]); + assert_eq!(out.len(), 2); + assert!(matches!(out[0].result, Ok(None)), "retried on the CPU: {:?}", out[0].result.as_ref().err()); + assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED)); + assert!(matches!(out[1].result, Ok(None)), "the next job starts on the CPU"); + assert_eq!(out[1].notice, None, "said once"); + } + + #[test] + fn a_model_error_on_the_gpu_is_retried_on_the_cpu_too() { + let out = run_kinds(Placement::Gpu, vec![Kind::FailOnGpu { panic: false }]); + assert!(matches!(out[0].result, Ok(None))); + assert_eq!(out[0].notice.as_deref(), Some(GPU_FAILED)); + } + + #[test] + fn a_job_that_fails_on_the_cpu_too_keeps_the_gpu() { + let out = run_kinds(Placement::Gpu, vec![Kind::Panic, Kind::FailOnGpu { panic: true }]); + assert!(out[0].result.as_ref().unwrap_err().contains("failed unexpectedly")); + assert_eq!(out[0].notice, None); + // not the GPU's fault: the next job tries the GPU first again + assert!(matches!(out[1].result, Ok(None))); + assert_eq!(out[1].notice.as_deref(), Some(GPU_FAILED)); + } + + #[test] + fn a_worker_on_the_cpu_never_tries_the_gpu() { + // no GPU in this build, or LIGHTCRAFT_GPU=0 / LIGHTCRAFT_GPU_BACKEND=off + let out = run_kinds(Placement::Cpu, vec![Kind::FailOnGpu { panic: true }]); + assert!(matches!(out[0].result, Ok(None))); + assert_eq!(out[0].notice, None); + if !lightcraft_segment::GPU_BUILD { + assert_eq!(Placement::initial(), Placement::Cpu); + } + } + + #[test] + fn missing_model_files_are_not_retried_on_the_cpu() { + let out = run_kinds(Placement::Gpu, vec![Kind::Prepare]); + assert!(out[0].result.as_ref().unwrap_err().contains("not found")); + assert_eq!(out[0].notice, None); + let missing = lightcraft_segment::Error::Missing(PathBuf::from("models")); + assert!(matches!(Failure::from(missing), Failure::Other(_))); + } + #[test] fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() { let (tx, rx) = mpsc::channel(); @@ -464,7 +652,7 @@ mod tests { drop(jtx); let shared = Shared::default(); shared.pending.store(3, Ordering::SeqCst); - run(&jrx, &shared, Duration::from_secs(60)); + run(&jrx, &shared, Duration::from_secs(60), Placement::Cpu); let out: Vec = rx.try_iter().collect(); assert_eq!(out.len(), 3); assert!(out[0].superseded && out[1].superseded && !out[2].superseded); diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index b0409962d..7efae6a57 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -75,12 +75,8 @@ fn record_fallback(reason: String) { /// E.g. "disabled by LIGHTCRAFT_GPU=0", "software adapter (llvmpipe …) skipped", "device lost …". /// Never blocks. pub fn unavailable_reason() -> Option { - if env_disabled() { - #[cfg(not(target_arch = "wasm32"))] - if backend::env_off() { - return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()); - } - return Some("disabled by LIGHTCRAFT_GPU=0".into()); + if let Some(why) = env_off_reason() { + return Some(why.into()); } if !ENABLED.load(Ordering::Relaxed) { return Some("disabled by the GPU rendering preference (app.gpu)".into()); @@ -245,12 +241,26 @@ pub fn wait_idle(timeout: std::time::Duration) -> bool { /// environment switches and the rendering preference, but not a render-device failure (the devices are separate). #[cfg(not(target_arch = "wasm32"))] pub(crate) fn switched_off() -> Option { - if env_disabled() { - return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into()); + if let Some(why) = env_off_reason() { + return Some(why.into()); } (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into()) } +/// Why the environment switches the GPU off for the whole process (`LIGHTCRAFT_GPU=0` or +/// `LIGHTCRAFT_GPU_BACKEND=off`), or `None`. Code with a GPU path of its own (the SAM 3 model on +/// Metal) follows it too. +pub fn env_off_reason() -> Option<&'static str> { + if !env_disabled() { + return None; + } + #[cfg(not(target_arch = "wasm32"))] + if backend::env_off() { + return Some("disabled by LIGHTCRAFT_GPU_BACKEND=off"); + } + Some("disabled by LIGHTCRAFT_GPU=0") +} + /// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process. fn env_disabled() -> bool { static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 6d488a919..6c24d0836 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -87,6 +87,11 @@ pub fn is_model_dir(dir: &Path) -> bool { dir.join(WEIGHTS_FILE).is_file() && dir.join("vocab.json").is_file() && dir.join("merges.txt").is_file() } +#[cfg(not(target_arch = "wasm32"))] +/// Whether [`best_device`] can be a GPU in this build (Metal on macOS). Where it can, a caller that +/// sees the model fail there can load it again with [`Sam3::load_cpu`]. +pub const GPU_BUILD: bool = cfg!(target_os = "macos"); + #[cfg(not(target_arch = "wasm32"))] /// The best device here: Metal on macOS (when a GPU is available), else the CPU. pub fn best_device() -> Device { @@ -134,6 +139,11 @@ impl Sam3 { Self::load_on(dir, best_device()) } + /// [`Sam3::load`] on the CPU, whatever GPU there is. + pub fn load_cpu(dir: &Path) -> Result { + Self::load_on(dir, Device::Cpu) + } + pub fn load_on(dir: &Path, device: Device) -> Result { if !is_model_dir(dir) { return Err(Error::Missing(dir.to_path_buf())); diff --git a/crates/segment/src/remote.rs b/crates/segment/src/remote.rs index cdee096fe..10f7b2931 100644 --- a/crates/segment/src/remote.rs +++ b/crates/segment/src/remote.rs @@ -108,10 +108,10 @@ impl Client { } } -pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { +pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> crate::Result>> { let mut merged: Option> = None; for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { - if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { + if let Some(p) = model.segment_text(enc, phrase, 0.5)? { if let Some(m) = &mut merged { for (a, b) in m.iter_mut().zip(p.logits) { *a = a.max(b); @@ -130,7 +130,7 @@ fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result< model.prepare_clicks(enc).map_err(|e| e.to_string())?; Ok(None) } - Operation::Text(text) => text_logits(model, enc, &text), + Operation::Text(text) => text_logits(model, enc, &text).map_err(|e| e.to_string()), Operation::Clicks(points) => { let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect(); Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits)) diff --git a/docs/ai-masks.md b/docs/ai-masks.md index b02912abc..cd803b388 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -2,7 +2,8 @@ LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta 2025), run in pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU -through Metal on macOS, on the CPU elsewhere (for now). +through Metal on macOS, on the CPU elsewhere (for now). If the Mac's GPU can't run it, it runs on +the CPU (see the worker below). **The model is optional.** It is not part of LightCraft, and nothing requires it: without it, Object and Describe offer to download it, every other feature (and every existing mask, see @@ -225,6 +226,13 @@ wait and return the result (when built with the `sam` feature; the default CLI h encoded photo; the session sends it jobs and receives results over channels, never holding a lock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes an error and the model is reloaded on the next request; busy counters are reset by drop guards. +- GPU fallback (issue #1974): a job that fails on the GPU (a panic or a candle error, e.g. an + AMD GPU in an Intel Mac whose Metal compiler crashes on a matmul kernel) frees the model and runs + once more on the CPU. If that works, the model stays on the CPU until LightCraft restarts and + the user is told once ("The GPU couldn't run the AI mask model…"); if the CPU fails too, the + error is reported and the next job tries the GPU again. Missing or damaged model files and + remote-worker errors are not retried. `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off` keep + the model on the CPU from the start. - Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered), each at most 1024² logits; more than 64 clicks are never sent to the model. - Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python diff --git a/docs/gpu-pipeline.md b/docs/gpu-pipeline.md index b74af85a2..5bd0119f9 100644 --- a/docs/gpu-pipeline.md +++ b/docs/gpu-pipeline.md @@ -43,6 +43,8 @@ Overrides (read once at launch): - else `WGPU_BACKEND` (wgpu's own variable, same names) — before #136 only the window honoured it, the compute device always used Vulkan + DX12 (+ Metal). - `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected). +- Both `LIGHTCRAFT_GPU=0` and `LIGHTCRAFT_GPU_BACKEND=off` also keep AI Denoise, the face models + and the AI mask model (SAM 3 on Metal) on the CPU. **DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders with FXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu's diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 95dcd2aef..a78ff0f7c 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -293,7 +293,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts). | LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model; a photo that carries person mattes (DNG semantic masks: iPhone ProRAW portrait / skin / hair / teeth / glasses mattes) uses them instead (synthetic files only: no corpus file carries person mattes) | | LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`), `crates/pipeline/src/masks.rs` (`Mattes`), `crates/raw/src/semantic.rs` | heuristic; a photo that carries a sky matte (DNG semantic mask, e.g. iPhone ProRAW) uses it instead, through orientation / crop / lens corrections, preview = export (checked on one CC0 iPhone 12 Pro ProRAW and synthetic files) | | LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject mask (heuristic, or the photo's person mattes) | -| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) | +| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `cmd:app.openSamModelFolder`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors; a job that fails on the Mac's GPU runs again on the CPU, which then serves the rest of the session, #1974), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand: the dialog names the three files, opens the configured model folder with the host opener, preserves setup on an open error, and links docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) | | LR-MASK-PEOPLE | People parts | P2 | ⬜ | | shape exists (no UI): uses the photo's person / skin / hair / teeth / glasses mattes when it carries them (DNG semantic masks), else the subject heuristic | | LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty | | LR-MASK-BRUSH | Brush mask | P0 | 🟡 | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs`, `crates/ui-egui/src/brush_live.rs` | size/feather/flow/density/erase; Create New Mask starts a separate brush in paint mode; the stroke shows while the button is held with the committed stroke's dab spacing, feather and flow, so fast strokes have no gaps (#517); coverage builds up by passes, not hold time (holding the brush still adds nothing, live or committed, #518); Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); ⇧-click draws a straight stroke from where the last stroke ended (`connect`), tapering size, feather and flow from that stroke's values when they changed; no A/B brushes; pen pressure (Brush ▸ Pen Pressure, on by default; `pressure` per point in `mask.brushStroke`) scales each dab's size (a tenth to all of it) and flow, live and committed alike, where winit reports a pen force (Windows; not macOS / Linux tablets yet); no tilt |