mirror of
https://github.com/storytold/deckcraft.git
synced 2026-10-24 10:37:03 -06:00
Pictures: Remove Background with a classical matte
The Picture Format > Remove Background button was a disabled placeholder. picture.removeBackground now flood-fills from every edge pixel through similar colours: each step must be close to the pixel before it (so gradients and soft shadows follow) and to the edge colour it started from (so it can't leak into the subject). That region turns transparent with a one-pixel soft edge, and the picture points at a new PNG; undo brings the original back. Tolerance 0-100 (default 20); the ribbon offers normal, looser and tighter. Pictures whose edges are all subject are left alone, and pictures over 16 megapixels are refused. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Nikita Rakitin
@@ -0,0 +1,242 @@
|
||||
//! Picture Format ▸ Remove Background: a classical matte, no model. The background is what the
|
||||
//! picture's edges reach through similar colours: a flood fill from every edge pixel, each step
|
||||
//! close to the pixel before it (gradients and soft shadows) and to the edge colour it started
|
||||
//! from (so it can't leak into the subject). It turns transparent, with a one-pixel soft edge.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use deckcraft_model::{ShapeId, ShapeKind};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
use crate::{Result, Session};
|
||||
|
||||
/// Larger pictures are refused rather than matted slowly (16 megapixels).
|
||||
const MAX_PIXELS: u64 = 16_000_000;
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![cmd!(
|
||||
"picture.removeBackground",
|
||||
"Remove Background",
|
||||
["Picture Format", "Adjust"],
|
||||
None,
|
||||
"{tolerance?: 0..100 (how different a colour may be from the edge's, default 20), ids?}",
|
||||
has_selection,
|
||||
remove_background
|
||||
)]
|
||||
}
|
||||
|
||||
fn dist(a: [u8; 4], b: [u8; 4]) -> f64 {
|
||||
let d = |i: usize| f64::from(a[i]) - f64::from(b[i]);
|
||||
(d(0) * d(0) + d(1) * d(1) + d(2) * d(2)).sqrt()
|
||||
}
|
||||
|
||||
/// Make the background of `img` transparent. `tolerance` is 0–100. Returns how many pixels
|
||||
/// became transparent.
|
||||
pub fn matte(img: &mut image::RgbaImage, tolerance: f64) -> usize {
|
||||
let (w, h) = (img.width() as usize, img.height() as usize);
|
||||
if w == 0 || h == 0 {
|
||||
return 0;
|
||||
}
|
||||
// Distances run 0–441 (black to white); the step limit follows gradients, the seed limit
|
||||
// keeps the fill on the edge's colour.
|
||||
let seed_tol = tolerance.clamp(0.0, 100.0) * 4.41;
|
||||
let step_tol = (seed_tol * 0.35).max(6.0);
|
||||
let px = |img: &image::RgbaImage, i: usize| img.get_pixel((i % w) as u32, (i / w) as u32).0;
|
||||
// 0: subject, 1: background; the seed colour each background pixel was reached from.
|
||||
let mut bg = vec![false; w * h];
|
||||
let mut seed = vec![[0u8; 4]; w * h];
|
||||
let mut queue = VecDeque::new();
|
||||
let push_edge = |i: usize, bg: &mut [bool], seed: &mut [[u8; 4]], q: &mut VecDeque<usize>| {
|
||||
if let Some(b) = bg.get_mut(i)
|
||||
&& !*b
|
||||
{
|
||||
*b = true;
|
||||
if let Some(s) = seed.get_mut(i) {
|
||||
*s = px(img, i);
|
||||
}
|
||||
q.push_back(i);
|
||||
}
|
||||
};
|
||||
for x in 0..w {
|
||||
push_edge(x, &mut bg, &mut seed, &mut queue);
|
||||
push_edge((h - 1) * w + x, &mut bg, &mut seed, &mut queue);
|
||||
}
|
||||
for y in 0..h {
|
||||
push_edge(y * w, &mut bg, &mut seed, &mut queue);
|
||||
push_edge(y * w + w - 1, &mut bg, &mut seed, &mut queue);
|
||||
}
|
||||
while let Some(i) = queue.pop_front() {
|
||||
let (x, y) = (i % w, i / w);
|
||||
let here = px(img, i);
|
||||
let from = seed.get(i).copied().unwrap_or(here);
|
||||
let mut near = [None; 4];
|
||||
if x > 0 {
|
||||
near[0] = Some(i - 1);
|
||||
}
|
||||
if x + 1 < w {
|
||||
near[1] = Some(i + 1);
|
||||
}
|
||||
if y > 0 {
|
||||
near[2] = Some(i - w);
|
||||
}
|
||||
if y + 1 < h {
|
||||
near[3] = Some(i + w);
|
||||
}
|
||||
for j in near.into_iter().flatten() {
|
||||
if bg.get(j).copied().unwrap_or(true) {
|
||||
continue;
|
||||
}
|
||||
let c = px(img, j);
|
||||
// Transparent pixels are background already.
|
||||
if c[3] < 16 || (dist(c, here) <= step_tol && dist(c, from) <= seed_tol) {
|
||||
if let Some(b) = bg.get_mut(j) {
|
||||
*b = true;
|
||||
}
|
||||
if let Some(s) = seed.get_mut(j) {
|
||||
*s = from;
|
||||
}
|
||||
queue.push_back(j);
|
||||
}
|
||||
}
|
||||
}
|
||||
// A background edge that is the subject's colour throughout (the subject touches the
|
||||
// edge on every side) is no background: leave such a picture alone.
|
||||
let n = bg.iter().filter(|b| **b).count();
|
||||
if n == w * h {
|
||||
return 0;
|
||||
}
|
||||
let is_bg = |x: usize, y: usize| bg.get(y * w + x).copied().unwrap_or(false);
|
||||
let mut soft = vec![];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
if is_bg(x, y) {
|
||||
continue;
|
||||
}
|
||||
let touching =
|
||||
(x > 0 && is_bg(x - 1, y)) || (x + 1 < w && is_bg(x + 1, y)) || (y > 0 && is_bg(x, y - 1)) || (y + 1 < h && is_bg(x, y + 1));
|
||||
if touching {
|
||||
soft.push((x, y));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (i, b) in bg.iter().enumerate() {
|
||||
if *b {
|
||||
img.get_pixel_mut((i % w) as u32, (i / w) as u32).0[3] = 0;
|
||||
}
|
||||
}
|
||||
for (x, y) in soft {
|
||||
let p = img.get_pixel_mut(x as u32, y as u32);
|
||||
p.0[3] = (u16::from(p.0[3]) * 2 / 3) as u8;
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
fn remove_background(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let tolerance = f64_or(p, "tolerance", 20.0);
|
||||
let ids = targets(s, p)?;
|
||||
let mut done: Vec<(ShapeId, String, Vec<u8>)> = vec![];
|
||||
{
|
||||
let st = s.doc()?;
|
||||
for id in &ids {
|
||||
let Some(sh) = st.shape(*id) else { continue };
|
||||
let ShapeKind::Picture { fill } = &sh.kind else { continue };
|
||||
let Some(item) = st.doc.media(fill.media) else { continue };
|
||||
let Ok(reader) = image::ImageReader::new(std::io::Cursor::new(item.data.as_slice())).with_guessed_format() else { continue };
|
||||
let Ok((iw, ih)) = reader.into_dimensions() else { continue };
|
||||
if u64::from(iw) * u64::from(ih) > MAX_PIXELS {
|
||||
return Err(bad("picture.removeBackground", format!("{iw}×{ih} pixels is too large; compress the picture first")));
|
||||
}
|
||||
let Ok(dynimg) = image::load_from_memory(&item.data) else { continue };
|
||||
let mut img = dynimg.to_rgba8();
|
||||
if matte(&mut img, tolerance) == 0 {
|
||||
continue;
|
||||
}
|
||||
let mut png = vec![];
|
||||
if image::DynamicImage::ImageRgba8(img).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).is_err() {
|
||||
continue;
|
||||
}
|
||||
let stem = item.name.rsplit_once('.').map_or(item.name.as_str(), |(a, _)| a);
|
||||
done.push((*id, format!("{stem}-no-background.png"), png));
|
||||
}
|
||||
}
|
||||
if done.is_empty() {
|
||||
return Err(bad("picture.removeBackground", "no background found: select a picture whose edges show its background"));
|
||||
}
|
||||
let n = done.len();
|
||||
s.edit(|doc, sel| {
|
||||
let added: Vec<(ShapeId, deckcraft_model::MediaId)> =
|
||||
done.into_iter().map(|(id, name, png)| (id, doc.add_media(&name, "image/png", png))).collect();
|
||||
let shapes = crate::shapes_mut(doc, sel).ok_or_else(|| bad("picture.removeBackground", "no slide"))?;
|
||||
for (id, media) in added {
|
||||
if let Some(sh) = deckcraft_model::find_shape_mut(shapes, id)
|
||||
&& let ShapeKind::Picture { fill } = &mut sh.kind
|
||||
{
|
||||
fill.media = media;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(json!({"pictures": n}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A red disc on a white-to-grey gradient.
|
||||
fn disc() -> image::RgbaImage {
|
||||
image::RgbaImage::from_fn(80, 60, |x, y| {
|
||||
let (dx, dy) = (f64::from(x) - 40.0, f64::from(y) - 30.0);
|
||||
if dx * dx + dy * dy < 15.0 * 15.0 {
|
||||
image::Rgba([220, 30, 30, 255])
|
||||
} else {
|
||||
let g = 255 - (x / 2) as u8;
|
||||
image::Rgba([g, g, g, 255])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matte_clears_the_gradient_and_keeps_the_subject() {
|
||||
let mut img = disc();
|
||||
let n = matte(&mut img, 20.0);
|
||||
assert!(n > 80 * 60 / 2, "{n}");
|
||||
assert_eq!(img.get_pixel(0, 0).0[3], 0);
|
||||
assert_eq!(img.get_pixel(79, 59).0[3], 0, "the darker end of the gradient");
|
||||
assert_eq!(img.get_pixel(40, 30).0, [220, 30, 30, 255]);
|
||||
// The disc's rim is softened, not cut.
|
||||
let rim = img.get_pixel(40, 16).0[3];
|
||||
assert!(rim > 0 && rim < 255 || img.get_pixel(40, 15).0[3] < 255, "{rim}");
|
||||
// A picture with nothing but subject at its edges is left alone.
|
||||
let mut flat = image::RgbaImage::from_pixel(10, 10, image::Rgba([10, 200, 10, 255]));
|
||||
assert_eq!(matte(&mut flat, 20.0), 0);
|
||||
assert!(flat.pixels().all(|p| p.0[3] == 255));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_background_replaces_the_picture_and_undoes() {
|
||||
let mut png = vec![];
|
||||
image::DynamicImage::ImageRgba8(disc()).write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
|
||||
let mut s = Session::with_new();
|
||||
let r = s.execute("insert.picture", &json!({"data": base64_encode(&png), "name": "disc.png"})).unwrap();
|
||||
let id = r["id"].as_u64().unwrap();
|
||||
let media_of = |s: &Session| match &s.doc().unwrap().shape(ShapeId(id as u32)).unwrap().kind {
|
||||
ShapeKind::Picture { fill } => fill.media,
|
||||
_ => panic!("not a picture"),
|
||||
};
|
||||
let before = media_of(&s);
|
||||
s.execute("picture.removeBackground", &json!({"ids": [id]})).unwrap();
|
||||
let after = media_of(&s);
|
||||
assert_ne!(before, after);
|
||||
let item = s.doc().unwrap().doc.media(after).unwrap().clone();
|
||||
assert_eq!(item.name, "disc-no-background.png");
|
||||
let img = image::load_from_memory(&item.data).unwrap().to_rgba8();
|
||||
assert_eq!(img.get_pixel(0, 0).0[3], 0);
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
assert_eq!(media_of(&s), before);
|
||||
// Not a picture: refused.
|
||||
s.execute("shape.insert", &json!({"preset": "rect", "rect": [0, 0, 50, 50]})).unwrap();
|
||||
assert!(s.execute("picture.removeBackground", &json!({})).is_err());
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
pub mod animation;
|
||||
pub mod arrange;
|
||||
pub mod background;
|
||||
pub mod design;
|
||||
pub mod edit;
|
||||
pub mod file;
|
||||
@@ -130,6 +131,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
|
||||
v.extend(slide::specs());
|
||||
v.extend(insert::specs());
|
||||
v.extend(shape::specs());
|
||||
v.extend(background::specs());
|
||||
v.extend(arrange::specs());
|
||||
v.extend(selection::specs());
|
||||
v.extend(merge::specs());
|
||||
|
||||
@@ -2049,8 +2049,14 @@ fn size_group(app: &mut SlideApp, ui: &mut Ui) {
|
||||
|
||||
fn picture_format(app: &mut SlideApp, ui: &mut Ui) {
|
||||
group(ui, |ui| {
|
||||
let r = big_button(ui, Icon::RemoveBackground, "Remove\nBackground", false);
|
||||
let _ = r.on_hover_text("Remove Background (coming soon)");
|
||||
let r = big_button(ui, Icon::RemoveBackground, "Remove\nBackground", enabled(app, "picture.removeBackground"));
|
||||
egui::Popup::menu(&r).show(|ui| {
|
||||
for (label, tol) in [("Remove Background", 20.0), ("Remove More (looser)", 35.0), ("Remove Less (tighter)", 10.0)] {
|
||||
if ui.button(label).clicked() {
|
||||
run(app, "picture.removeBackground", json!({"tolerance": tol}));
|
||||
}
|
||||
}
|
||||
});
|
||||
let r = big_button(ui, Icon::Corrections, "Corrections", true);
|
||||
egui::Popup::menu(&r).show(|ui| {
|
||||
for b in [-0.4, -0.2, 0.0, 0.2, 0.4] {
|
||||
|
||||
+1
-2
@@ -185,8 +185,7 @@ Detail in [animation-parity.md](animation-parity.md). 10–16 h; the pane is a *
|
||||
|
||||
### Smaller missing rows
|
||||
Zoom (summary/section/slide, 6–10 h), reuse slides (2–4 h), laser pointer (1–2 h), transition
|
||||
sounds (1–2 h), artistic effects (6–10 h), remove background (6–10 h; needs a model or a classical
|
||||
matte), compress pictures (2–3 h), insert screenshot (2–3 h), record audio/screen and record slide
|
||||
sounds (1–2 h), artistic effects (6–10 h), compress pictures (2–3 h), insert screenshot (2–3 h), record audio/screen and record slide
|
||||
show (8–12 h), compare presentations (6–10 h), password protection (4–6 h, ECMA-376 agile
|
||||
encryption), ink to shape/text (6–10 h), icons and 3-D models (8–14 h, needs an open icon set).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user