Merge PR #550 from NikitkaRa: Pictures: Remove Background with a classical matte

crates/engine/src/cmd/background.rs

@@ -0,0 +1,242 @@
+//! Picture Format ▸ Remove Background: a classical matte, no model. The background is what the
+//! picture's edges reach through similar colours: a flood fill from every edge pixel, each step
+//! close to the pixel before it (gradients and soft shadows) and to the edge colour it started
+//! from (so it can't leak into the subject). It turns transparent, with a one-pixel soft edge.
+
+use std::collections::VecDeque;
+
+use deckcraft_model::{ShapeId, ShapeKind};
+use serde_json::{Value, json};
+
+use super::*;
+use crate::{Result, Session};
+
+/// Larger pictures are refused rather than matted slowly (16 megapixels).
+const MAX_PIXELS: u64 = 16_000_000;
+
+pub fn specs() -> Vec<CommandSpec> {
+    vec![cmd!(
+        "picture.removeBackground",
+        "Remove Background",
+        ["Picture Format", "Adjust"],
+        None,
+        "{tolerance?: 0..100 (how different a colour may be from the edge's, default 20), ids?}",
+        has_selection,
+        remove_background
+    )]
+}
+
+fn dist(a: [u8; 4], b: [u8; 4]) -> f64 {
+    let d = |i: usize| f64::from(a[i]) - f64::from(b[i]);
+    (d(0) * d(0) + d(1) * d(1) + d(2) * d(2)).sqrt()
+}
+
+/// Make the background of `img` transparent. `tolerance` is 0–100. Returns how many pixels
+/// became transparent.
+pub fn matte(img: &mut image::RgbaImage, tolerance: f64) -> usize {
+    let (w, h) = (img.width() as usize, img.height() as usize);
+    if w == 0 || h == 0 {
+        return 0;
+    }
+    // Distances run 0–441 (black to white); the step limit follows gradients, the seed limit
+    // keeps the fill on the edge's colour.
+    let seed_tol = tolerance.clamp(0.0, 100.0) * 4.41;
+    let step_tol = (seed_tol * 0.35).max(6.0);
+    let px = |img: &image::RgbaImage, i: usize| img.get_pixel((i % w) as u32, (i / w) as u32).0;
+    // 0: subject, 1: background; the seed colour each background pixel was reached from.
+    let mut bg = vec![false; w * h];
+    let mut seed = vec![[0u8; 4]; w * h];
+    let mut queue = VecDeque::new();
+    let push_edge = |i: usize, bg: &mut [bool], seed: &mut [[u8; 4]], q: &mut VecDeque<usize>| {
+        if let Some(b) = bg.get_mut(i)
+            && !*b
+        {
+            *b = true;
+            if let Some(s) = seed.get_mut(i) {
+                *s = px(img, i);
+            }
+            q.push_back(i);
+        }
+    };
+    for x in 0..w {
+        push_edge(x, &mut bg, &mut seed, &mut queue);
+        push_edge((h - 1) * w + x, &mut bg, &mut seed, &mut queue);
+    }
+    for y in 0..h {
+        push_edge(y * w, &mut bg, &mut seed, &mut queue);
+        push_edge(y * w + w - 1, &mut bg, &mut seed, &mut queue);
+    }
+    while let Some(i) = queue.pop_front() {
+        let (x, y) = (i % w, i / w);
+        let here = px(img, i);
+        let from = seed.get(i).copied().unwrap_or(here);
+        let mut near = [None; 4];
+        if x > 0 {
+            near[0] = Some(i - 1);
+        }
+        if x + 1 < w {
+            near[1] = Some(i + 1);
+        }
+        if y > 0 {
+            near[2] = Some(i - w);
+        }
+        if y + 1 < h {
+            near[3] = Some(i + w);
+        }
+        for j in near.into_iter().flatten() {
+            if bg.get(j).copied().unwrap_or(true) {
+                continue;
+            }
+            let c = px(img, j);
+            // Transparent pixels are background already.
+            if c[3] < 16 || (dist(c, here) <= step_tol && dist(c, from) <= seed_tol) {
+                if let Some(b) = bg.get_mut(j) {
+                    *b = true;
+                }
+                if let Some(s) = seed.get_mut(j) {
+                    *s = from;
+                }
+                queue.push_back(j);
+            }
+        }
+    }
+    // A background edge that is the subject's colour throughout (the subject touches the
+    // edge on every side) is no background: leave such a picture alone.
+    let n = bg.iter().filter(|b| **b).count();
+    if n == w * h {
+        return 0;
+    }
+    let is_bg = |x: usize, y: usize| bg.get(y * w + x).copied().unwrap_or(false);
+    let mut soft = vec![];
+    for y in 0..h {
+        for x in 0..w {
+            if is_bg(x, y) {
+                continue;
+            }
+            let touching =
+                (x > 0 && is_bg(x - 1, y)) || (x + 1 < w && is_bg(x + 1, y)) || (y > 0 && is_bg(x, y - 1)) || (y + 1 < h && is_bg(x, y + 1));
+            if touching {
+                soft.push((x, y));
+            }
+        }
+    }
+    for (i, b) in bg.iter().enumerate() {
+        if *b {
+            img.get_pixel_mut((i % w) as u32, (i / w) as u32).0[3] = 0;
+        }
+    }
+    for (x, y) in soft {
+        let p = img.get_pixel_mut(x as u32, y as u32);
+        p.0[3] = (u16::from(p.0[3]) * 2 / 3) as u8;
+    }
+    n
+}
+
+fn remove_background(s: &mut Session, p: &Value) -> Result<Value> {
+    let tolerance = f64_or(p, "tolerance", 20.0);
+    let ids = targets(s, p)?;
+    let mut done: Vec<(ShapeId, String, Vec<u8>)> = vec![];
+    {
+        let st = s.doc()?;
+        for id in &ids {
+            let Some(sh) = st.shape(*id) else { continue };
+            let ShapeKind::Picture { fill } = &sh.kind else { continue };
+            let Some(item) = st.doc.media(fill.media) else { continue };
+            let Ok(reader) = image::ImageReader::new(std::io::Cursor::new(item.data.as_slice())).with_guessed_format() else { continue };
+            let Ok((iw, ih)) = reader.into_dimensions() else { continue };
+            if u64::from(iw) * u64::from(ih) > MAX_PIXELS {
+                return Err(bad("picture.removeBackground", format!("{iw}×{ih} pixels is too large; compress the picture first")));
+            }
+            let Ok(dynimg) = image::load_from_memory(&item.data) else { continue };
+            let mut img = dynimg.to_rgba8();
+            if matte(&mut img, tolerance) == 0 {
+                continue;
+            }
+            let mut png = vec![];
+            if image::DynamicImage::ImageRgba8(img).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).is_err() {
+                continue;
+            }
+            let stem = item.name.rsplit_once('.').map_or(item.name.as_str(), |(a, _)| a);
+            done.push((*id, format!("{stem}-no-background.png"), png));
+        }
+    }
+    if done.is_empty() {
+        return Err(bad("picture.removeBackground", "no background found: select a picture whose edges show its background"));
+    }
+    let n = done.len();
+    s.edit(|doc, sel| {
+        let added: Vec<(ShapeId, deckcraft_model::MediaId)> =
+            done.into_iter().map(|(id, name, png)| (id, doc.add_media(&name, "image/png", png))).collect();
+        let shapes = crate::shapes_mut(doc, sel).ok_or_else(|| bad("picture.removeBackground", "no slide"))?;
+        for (id, media) in added {
+            if let Some(sh) = deckcraft_model::find_shape_mut(shapes, id)
+                && let ShapeKind::Picture { fill } = &mut sh.kind
+            {
+                fill.media = media;
+            }
+        }
+        Ok(())
+    })?;
+    Ok(json!({"pictures": n}))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A red disc on a white-to-grey gradient.
+    fn disc() -> image::RgbaImage {
+        image::RgbaImage::from_fn(80, 60, |x, y| {
+            let (dx, dy) = (f64::from(x) - 40.0, f64::from(y) - 30.0);
+            if dx * dx + dy * dy < 15.0 * 15.0 {
+                image::Rgba([220, 30, 30, 255])
+            } else {
+                let g = 255 - (x / 2) as u8;
+                image::Rgba([g, g, g, 255])
+            }
+        })
+    }
+
+    #[test]
+    fn matte_clears_the_gradient_and_keeps_the_subject() {
+        let mut img = disc();
+        let n = matte(&mut img, 20.0);
+        assert!(n > 80 * 60 / 2, "{n}");
+        assert_eq!(img.get_pixel(0, 0).0[3], 0);
+        assert_eq!(img.get_pixel(79, 59).0[3], 0, "the darker end of the gradient");
+        assert_eq!(img.get_pixel(40, 30).0, [220, 30, 30, 255]);
+        // The disc's rim is softened, not cut.
+        let rim = img.get_pixel(40, 16).0[3];
+        assert!(rim > 0 && rim < 255 || img.get_pixel(40, 15).0[3] < 255, "{rim}");
+        // A picture with nothing but subject at its edges is left alone.
+        let mut flat = image::RgbaImage::from_pixel(10, 10, image::Rgba([10, 200, 10, 255]));
+        assert_eq!(matte(&mut flat, 20.0), 0);
+        assert!(flat.pixels().all(|p| p.0[3] == 255));
+    }
+
+    #[test]
+    fn remove_background_replaces_the_picture_and_undoes() {
+        let mut png = vec![];
+        image::DynamicImage::ImageRgba8(disc()).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
+        let mut s = Session::with_new();
+        let r = s.execute("insert.picture", &json!({"data": base64_encode(&png), "name": "disc.png"})).unwrap();
+        let id = r["id"].as_u64().unwrap();
+        let media_of = |s: &Session| match &s.doc().unwrap().shape(ShapeId(id as u32)).unwrap().kind {
+            ShapeKind::Picture { fill } => fill.media,
+            _ => panic!("not a picture"),
+        };
+        let before = media_of(&s);
+        s.execute("picture.removeBackground", &json!({"ids": [id]})).unwrap();
+        let after = media_of(&s);
+        assert_ne!(before, after);
+        let item = s.doc().unwrap().doc.media(after).unwrap().clone();
+        assert_eq!(item.name, "disc-no-background.png");
+        let img = image::load_from_memory(&item.data).unwrap().to_rgba8();
+        assert_eq!(img.get_pixel(0, 0).0[3], 0);
+        s.execute("edit.undo", &json!({})).unwrap();
+        assert_eq!(media_of(&s), before);
+        // Not a picture: refused.
+        s.execute("shape.insert", &json!({"preset": "rect", "rect": [0, 0, 50, 50]})).unwrap();
+        assert!(s.execute("picture.removeBackground", &json!({})).is_err());
+    }
+}

crates/engine/src/cmd/mod.rs

@@ -2,6 +2,7 @@
 
 pub mod animation;
 pub mod arrange;
+pub mod background;
 pub mod design;
 pub mod edit;
 pub mod file;
@@ -132,6 +133,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
         v.extend(slide::specs());
         v.extend(insert::specs());
         v.extend(shape::specs());
+        v.extend(background::specs());
         v.extend(arrange::specs());
         v.extend(selection::specs());
         v.extend(merge::specs());

crates/ui-egui/src/ribbon.rs

@@ -2212,8 +2212,14 @@ fn size_group(app: &mut SlideApp, ui: &mut Ui) {
 
 fn picture_format(app: &mut SlideApp, ui: &mut Ui) {
     group(ui, |ui| {
-        let r = big_button(ui, Icon::RemoveBackground, "Remove\nBackground", false);
-        let _ = r.on_hover_text("Remove Background (coming soon)");
+        let r = big_button(ui, Icon::RemoveBackground, "Remove\nBackground", enabled(app, "picture.removeBackground"));
+        egui::Popup::menu(&r).show(|ui| {
+            for (label, tol) in [("Remove Background", 20.0), ("Remove More (looser)", 35.0), ("Remove Less (tighter)", 10.0)] {
+                if ui.button(label).clicked() {
+                    run(app, "picture.removeBackground", json!({"tolerance": tol}));
+                }
+            }
+        });
         let r = big_button(ui, Icon::Corrections, "Corrections", true);
         egui::Popup::menu(&r).show(|ui| {
             for b in [-0.4, -0.2, 0.0, 0.2, 0.4] {

docs/gaps.md

@@ -227,8 +227,7 @@ Detail in [animation-parity.md](animation-parity.md). 10–16 h; the pane is a *
 
 ### Smaller missing rows
 Zoom (summary/section/slide, 6–10 h), reuse slides (2–4 h), laser pointer (1–2 h), transition
-sounds (1–2 h), artistic effects (6–10 h), remove background (6–10 h; needs a model or a classical
-matte), compress pictures (2–3 h), insert screenshot (2–3 h), record audio/screen and record slide
+sounds (1–2 h), artistic effects (6–10 h), compress pictures (2–3 h), insert screenshot (2–3 h), record audio/screen and record slide
 show (8–12 h), compare presentations (6–10 h), password protection (4–6 h, ECMA-376 agile
 encryption), ink to shape/text (6–10 h), icons and 3-D models (8–14 h, needs an open icon set).