Image Trace: the semi-transparent edge of a cut-out image traces as the color beside it

A pixel on the edge of a cut-out image with an alpha of 128 or more was
traced with its own color. When that color was a darkened edge tone, it
took whichever palette color lay nearest, and the edge traced as 1 px
slivers of a color that appears nowhere near it.

In Color and Grayscale, a pixel with an alpha of 128 to 254 next to a
pixel under half opaque now takes a color of the fully opaque pixels
within two pixels of it when its own color is that color or a darker
shade of it, counting only pixels it reaches without crossing a
transparent gap; of several such colors, it takes the one the most of
those pixels have. It keeps its own color when one of those pixels has
it, or when its color is not a shade of any of theirs, so a thin
semi-transparent outline of another color stays as it is. Ties go to the color nearest its own, by its gray
in Grayscale. Opaque images trace as before.

crates/trace/src/lib.rs

@@ -563,6 +563,10 @@ fn trace_paths(img: &Raster, params: &TraceParams, max_anchors: usize) -> Result
         rest
     });
     let (mut q, rare) = quantize::quantize_keeping(without.as_ref().unwrap_or(img), params);
+    // The semi-transparent edge of a cut-out image traces as the color beside it.
+    if matches!(params.mode, Mode::Color | Mode::Grayscale) {
+        quantize::settle_cut_edges(&mut q, img, params.mode == Mode::Grayscale);
+    }
     drop(without);
     // Each gradient is a palette entry of its own (its colour: the gradient's middle).
     let mut gradients: Vec<Option<TracedGradient>> = vec![None; q.palette.len()];

crates/trace/src/quantize.rs

@@ -395,6 +395,112 @@ fn compact(mut labels: Vec<u16>, palette: Vec<[u8; 3]>) -> Quantized {
     Quantized { labels, palette: out }
 }
 
+/// `q` with the semi-transparent pixels along the cut edge of `img` given the color beside them,
+/// and the colors that leaves unused dropped.
+///
+/// A traced pixel with an alpha of 128 to 254 and a pixel above, below, left or right of it with an
+/// alpha under 128 is on the cut edge. When its color is the color of a fully opaque pixel within
+/// two pixels of it, or a darker shade of that color (see [`edge_blend_of`]), it is that color
+/// blended with the transparent background, and its own color can lie nearest to an unrelated
+/// palette color, which would trace as a sliver of that color along the edge. Such a pixel takes
+/// the color that most of those opaque pixels have, and of colors that tie, the one nearest to its
+/// own color. Only the pixels it reaches through pixels of an alpha of 128 or more, going up, down,
+/// left or right, count, so that art across a transparent gap is left out. It keeps its own color
+/// when one of the fully opaque pixels it reaches has it, or when its color is a blend of none of
+/// them, as with a thin semi-transparent outline of another color. Pixels outside the image do not
+/// count as transparent. With `gray` (Grayscale), a pixel's color is its gray (see [`luma`]).
+pub(crate) fn settle_cut_edges(q: &mut Quantized, img: &Raster, gray: bool) {
+    let (w, h) = (img.width as usize, img.height as usize);
+    let px = img.rgba.as_chunks::<4>().0;
+    if q.labels.len() != w * h || px.len() != w * h {
+        return;
+    }
+    let alpha = |j: usize| px.get(j).map_or(0, |p| p[3]);
+    let color = |p: &[u8; 4]| if gray { [luma(*p); 3] } else { [p[0], p[1], p[2]] };
+    let mut changed = false;
+    let mut stack: Vec<(usize, usize)> = Vec::new();
+    for i in 0..w * h {
+        let mine = q.labels[i];
+        if mine == TRANSPARENT || !(128..=254).contains(&alpha(i)) {
+            continue;
+        }
+        let (x, y) = (i % w, i / w);
+        let cut =
+            (x > 0 && alpha(i - 1) < 128) || (x + 1 < w && alpha(i + 1) < 128) || (y > 0 && alpha(i - w) < 128) || (y + 1 < h && alpha(i + w) < 128);
+        if !cut {
+            continue;
+        }
+        let Some(p) = px.get(i) else { continue };
+        let own = oklab(color(p).map(f64::from));
+        // The colors of the fully opaque pixels within two pixels that it reaches and is a blend
+        // of, and how many have each.
+        let (x0, y0, x1, y1) = (x.saturating_sub(2), y.saturating_sub(2), (x + 2).min(w - 1), (y + 2).min(h - 1));
+        let mut reached = [[false; 5]; 5];
+        let mut near: Vec<(u16, usize)> = Vec::new();
+        let mut keep = false;
+        stack.clear();
+        stack.push((x, y));
+        if let Some(r) = reached.get_mut(y - y0).and_then(|row| row.get_mut(x - x0)) {
+            *r = true;
+        }
+        while let Some((cx, cy)) = stack.pop() {
+            let j = cy * w + cx;
+            let l = q.labels[j];
+            if alpha(j) == 255 && l != TRANSPARENT {
+                keep |= l == mine;
+                if px.get(j).is_some_and(|b| edge_blend_of(own, color(b), p[3])) {
+                    match near.iter_mut().find(|n| n.0 == l) {
+                        Some(n) => n.1 += 1,
+                        None => near.push((l, 1)),
+                    }
+                }
+            }
+            for (nx, ny) in [(cx.wrapping_sub(1), cy), (cx + 1, cy), (cx, cy.wrapping_sub(1)), (cx, cy + 1)] {
+                if nx < x0 || nx > x1 || ny < y0 || ny > y1 || alpha(ny * w + nx) < 128 {
+                    continue;
+                }
+                let Some(r) = reached.get_mut(ny - y0).and_then(|row| row.get_mut(nx - x0)) else { continue };
+                if !*r {
+                    *r = true;
+                    stack.push((nx, ny));
+                }
+            }
+        }
+        if keep || near.is_empty() {
+            continue;
+        }
+        let most = near.iter().map(|n| n.1).max().unwrap_or(0);
+        let tied: Vec<(u16, [f64; 3])> =
+            near.iter().filter(|n| n.1 == most).filter_map(|n| Some((n.0, oklab(q.palette.get(n.0 as usize)?.map(f64::from))))).collect();
+        let tied_labs: Vec<[f64; 3]> = tied.iter().map(|t| t.1).collect();
+        let Some(&(to, _)) = tied.get(nearest_lab(&tied_labs, own)) else { continue };
+        q.labels[i] = to;
+        changed = true;
+    }
+    if changed {
+        *q = compact(std::mem::take(&mut q.labels), std::mem::take(&mut q.palette));
+    }
+}
+
+/// How far (Oklab × 100) the color of a pixel on a cut edge may lie from the color beside it, or
+/// from a darker shade of that color, and still count as a blend of it with the transparent
+/// background.
+const EDGE_BLEND: f64 = 8.0;
+
+/// Whether `own` (Oklab), the color of a pixel with an alpha of `alpha`, is `beside` (sRGB) blended
+/// with the transparent background: whether it lies within [`EDGE_BLEND`] of the line in Oklab from
+/// `beside` to `beside` with each channel scaled by `alpha` / 510.
+fn edge_blend_of(own: [f64; 3], beside: [u8; 3], alpha: u8) -> bool {
+    let to = oklab(beside.map(f64::from));
+    let k = f64::from(alpha) / 510.0;
+    let from = oklab(beside.map(|v| f64::from(v) * k));
+    let d = [to[0] - from[0], to[1] - from[1], to[2] - from[2]];
+    let len2 = d[0] * d[0] + d[1] * d[1] + d[2] * d[2];
+    let t = if len2 > 0.0 { ((0..3).map(|c| (own[c] - from[c]) * d[c]).sum::<f64>() / len2).clamp(0.0, 1.0) } else { 0.0 };
+    let off2: f64 = (0..3).map(|c| (own[c] - from[c] - t * d[c]).powi(2)).sum();
+    off2 <= EDGE_BLEND * EDGE_BLEND
+}
+
 #[derive(Clone, Copy, Default)]
 struct Bin {
     n: u64,

crates/trace/src/tests.rs

@@ -222,6 +222,128 @@ fn transparent_pixels_are_not_traced() {
     assert!((net_area(&res.paths[0].path) - 800.0).abs() < 1.0);
 }
 
+/// A light square cut out of a transparent background, with a blue and a black block on it and a
+/// semi-transparent tan pixel row and column along two of its cut edges. `alpha` is the tan pixels'
+/// alpha.
+fn cut_out_with_edge(alpha: u8) -> Raster {
+    Raster::from_fn(200, 200, |x, y| {
+        if (60..140).contains(&x) && (40..90).contains(&y) {
+            [40, 120, 240, 255]
+        } else if (60..140).contains(&x) && (110..150).contains(&y) {
+            BLACK
+        } else if (20..180).contains(&x) && (20..180).contains(&y) {
+            [240, 240, 220, 255]
+        } else if (y == 180 && (30..170).contains(&x)) || (x == 180 && (30..110).contains(&y)) {
+            [160, 120, 80, alpha]
+        } else {
+            [0, 0, 0, 0]
+        }
+    })
+}
+
+#[test]
+fn a_cut_out_edge_traces_as_the_color_beside_it() {
+    // The tan of the edge pixels is nearer to the blue than to the light color beside them, but
+    // it is a darker shade of the light color, blended with the transparent background.
+    for mode in [Mode::Color, Mode::Grayscale] {
+        let params = TraceParams {
+            mode,
+            palette: Palette::Limited,
+            colors: 3,
+            paths: 90.0,
+            corners: 60.0,
+            noise: 25,
+            method: Method::Abutting,
+            ..TraceParams::default()
+        };
+        let res = trace(&cut_out_with_edge(192), &params);
+        // The blue block's color: that of the smallest path around the block's center.
+        let around = |p: &&TracedPath| p.path.bounds().is_some_and(|b| b.x0 < 100.0 && b.x1 > 100.0 && b.y0 < 65.0 && b.y1 > 65.0);
+        let size = |p: &TracedPath| p.path.bounds().map_or(f64::INFINITY, |b| b.width() * b.height());
+        let block = res.paths.iter().filter(around).min_by(|a, b| size(a).total_cmp(&size(b))).unwrap().color;
+        for p in res.paths.iter().filter(|p| p.color == block) {
+            let b = p.path.bounds().unwrap();
+            assert!(b.x0 > 59.0 && b.y0 > 39.0 && b.x1 < 141.0 && b.y1 < 91.0, "{mode:?}: a path of the blue block's color reaches {b:?}");
+        }
+    }
+}
+
+#[test]
+fn an_opaque_edge_keeps_its_own_color() {
+    // The same art with the edge pixels opaque: they are a tan line of their own, and stay so.
+    let img = cut_out_with_edge(255);
+    for mode in [Mode::Color, Mode::Grayscale] {
+        let q = quantize(&img, &TraceParams { mode, colors: 3, ..TraceParams::default() });
+        let mut settled = q.clone();
+        crate::quantize::settle_cut_edges(&mut settled, &img, mode == Mode::Grayscale);
+        assert_eq!(settled, q, "{mode:?}");
+    }
+}
+
+#[test]
+fn a_cut_out_edge_in_grayscale_ties_by_its_gray() {
+    // A red edge pixel between two opaque red pixels, one in a dark and one in a light gray of the
+    // palette: its gray (76) is nearer to the dark one, though its color looks nearer to the light
+    // one.
+    let img = Raster::from_fn(3, 2, |x, y| match (x, y) {
+        (0, 0) | (2, 0) => [255, 0, 0, 255],
+        (1, 0) => [255, 0, 0, 192],
+        _ => [0, 0, 0, 0],
+    });
+    let t = TRANSPARENT;
+    let q = Quantized { labels: vec![0, 2, 1, t, t, t], palette: vec![[50, 50, 50], [150, 150, 150], [90, 90, 90]] };
+    let mut gray = q.clone();
+    crate::quantize::settle_cut_edges(&mut gray, &img, true);
+    assert_eq!(gray.palette.get(gray.labels[1] as usize), Some(&[50, 50, 50]));
+    let mut color = q;
+    crate::quantize::settle_cut_edges(&mut color, &img, false);
+    assert_eq!(color.palette.get(color.labels[1] as usize), Some(&[150, 150, 150]));
+}
+
+#[test]
+fn a_semi_transparent_line_across_a_gap_keeps_its_own_color() {
+    // A light block, a transparent column, then a semi-transparent tan line: the block lies within
+    // two pixels of the line, but across the gap, so the line is no edge of it.
+    let img = Raster::from_fn(20, 10, |x, _| match x {
+        0..=9 => [240, 240, 220, 255],
+        11 => [160, 120, 80, 192],
+        _ => [0, 0, 0, 0],
+    });
+    let labels = (0..200).map(|i| match i % 20 {
+        0..=9 => 0,
+        11 => 1,
+        _ => TRANSPARENT,
+    });
+    let q = Quantized { labels: labels.collect(), palette: vec![[240, 240, 220], [160, 120, 80]] };
+    let mut settled = q.clone();
+    crate::quantize::settle_cut_edges(&mut settled, &img, false);
+    assert_eq!(settled, q);
+}
+
+#[test]
+fn a_thin_semi_transparent_outline_keeps_its_own_color() {
+    // A blue square on a transparent background with a one-pixel, partly transparent yellow
+    // outline around it: the yellow is no shade of the blue, so the outline stays yellow.
+    for alpha in [128, 192, 254] {
+        let img = Raster::from_fn(60, 60, |x, y| {
+            if (10..50).contains(&x) && (10..50).contains(&y) {
+                [40, 120, 240, 255]
+            } else if (9..51).contains(&x) && (9..51).contains(&y) {
+                [250, 200, 40, alpha]
+            } else {
+                [0, 0, 0, 0]
+            }
+        });
+        for mode in [Mode::Color, Mode::Grayscale] {
+            let q = quantize(&img, &TraceParams { mode, colors: 3, ..TraceParams::default() });
+            assert_ne!(q.labels[9 * 60 + 30], q.labels[30 * 60 + 30], "{mode:?}, alpha {alpha}: the outline has a color of its own");
+            let mut settled = q.clone();
+            crate::quantize::settle_cut_edges(&mut settled, &img, mode == Mode::Grayscale);
+            assert_eq!(settled, q, "{mode:?}, alpha {alpha}");
+        }
+    }
+}
+
 #[test]
 fn threshold_controls_black_and_white() {
     let img = Raster::from_fn(40, 40, |x, _| if x < 20 { [100, 100, 100, 255] } else { WHITE });