Gradient mesh: Object > Create Gradient Mesh on an embedded image (#1676)

The mesh covers the image's rectangle with its transform kept; each point takes the average colour and opacity of the picture around it. A linked image is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/doc/src/pixels.rs

@@ -79,6 +79,59 @@ impl ImageBlob {
         sample_pixels(&img, x * k(img.width(), object.0), y * k(img.height(), object.1), size)
     }
 
+    /// The colours of a `rows` × `cols` mesh laid over the image: `(rows + 1) * (cols + 1)` straight
+    /// RGBA samples, row-major, one per grid line crossing (row 0 is the top). Each is the
+    /// alpha-weighted average of the area around the crossing, one grid cell wide and high (a
+    /// quarter of each of the neighbouring cells), not a single pixel. A crossing over transparent
+    /// pixels gets their average colour at alpha 0. `None` when `rows` or `cols` is 0 or the
+    /// image can't be decoded.
+    pub fn grid_samples(&self, rows: u32, cols: u32) -> Option<Vec<[u8; 4]>> {
+        if rows == 0 || cols == 0 {
+            return None;
+        }
+        let img = self.decoded()?;
+        let (w, h) = (img.width() as usize, img.height() as usize);
+        if w == 0 || h == 0 {
+            return None;
+        }
+        // The pixel columns [from, to) around grid line `i` of `n`, at least one pixel wide.
+        let span = |i: u32, n: u32, len: usize| {
+            let at = |k: f64| (k / n as f64 * len as f64).round().clamp(0.0, len as f64) as usize;
+            let (from, to) = (at(i as f64 - 0.5), at(i as f64 + 0.5));
+            let from = from.min(len - 1);
+            (from, to.max(from + 1))
+        };
+        let mut out = Vec::with_capacity((rows as usize + 1) * (cols as usize + 1));
+        for r in 0..=rows {
+            let (y0, y1) = span(r, rows, h);
+            for c in 0..=cols {
+                let (x0, x1) = span(c, cols, w);
+                let (mut sum, mut plain, mut alpha, mut n) = ([0u64; 3], [0u64; 3], 0u64, 0u64);
+                for y in y0..y1 {
+                    for x in x0..x1 {
+                        let [pr, pg, pb, pa] = img.get_pixel(x as u32, y as u32).0;
+                        let a = pa as u64;
+                        for (k, v) in [pr, pg, pb].into_iter().enumerate() {
+                            sum[k] += v as u64 * a;
+                            plain[k] += v as u64;
+                        }
+                        alpha += a;
+                        n += 1;
+                    }
+                }
+                let n = n.max(1);
+                out.push(if alpha == 0 {
+                    let m = |k: usize| ((plain[k] + n / 2) / n) as u8;
+                    [m(0), m(1), m(2), 0]
+                } else {
+                    let m = |k: usize| ((sum[k] + alpha / 2) / alpha) as u8;
+                    [m(0), m(1), m(2), ((alpha + n / 2) / n) as u8]
+                });
+            }
+        }
+        Some(out)
+    }
+
     /// The decoded pixels (the last image decoded is kept for the next sample).
     fn decoded(&self) -> Option<Arc<image::RgbaImage>> {
         thread_local! {

crates/engine/src/cmd/live.rs

@@ -238,7 +238,7 @@ pub fn specs() -> Vec<CommandSpec> {
             "Create Gradient Mesh…",
             ["Object"],
             None,
-            "{ids?, rows?: n (4), cols?: n (4), appearance?: flat|center|edge, highlight?: % (100), at?: [x,y], columns?: as cols} convert filled paths into gradient meshes in their fill colour (a gradient fill: the colour it paints at each point), lightened per `appearance` (with `at`: 1×1 mesh plus lines through that point) → {ids, index?}",
+            "{ids?, rows?: n (4), cols?: n (4), appearance?: flat|center|edge, highlight?: % (100), at?: [x,y], columns?: as cols} convert filled paths into gradient meshes in their fill colour (a gradient fill: the colour it paints at each point), lightened per `appearance` (with `at`: 1×1 mesh plus lines through that point); an embedded image becomes a mesh over its rectangle (its transform kept) whose points take the average colour and opacity of the picture around them (a linked image is refused: embed it first) → {ids, index?}",
             has_selection,
             mesh_create
         ),
@@ -1348,18 +1348,33 @@ fn mesh_create(s: &mut Session, p: &Value) -> Result<Value> {
     let (ids, index) = s.edit("Create Gradient Mesh", |d, sel| {
         let mut out = vec![];
         let mut index = None;
+        let mut refused = None;
         for id in &roots {
             let Some(n) = d.node(*id).cloned() else { continue };
-            let Some(path) = outline_of(&n) else { continue };
-            let m = match n.appearance.fill_paint() {
-                // The points take the colours the gradient paints there.
-                Paint::Gradient(g) => {
-                    path.bounds().and_then(|b| GradientMesh::for_path_with(&path, rows, cols, &*gradient_sampler(&g, b))).map(|mut m| {
+            let m = if let NodeKind::Image(im) = &n.kind {
+                // An embedded image: the points take the picture's colours there.
+                match image_mesh(d, im, rows, cols) {
+                    Ok(mut m) => {
                         m.highlight(app, highlight);
-                        m
-                    })
+                        Some(m)
+                    }
+                    Err(why) => {
+                        refused.get_or_insert(why);
+                        continue;
+                    }
+                }
+            } else {
+                let Some(path) = outline_of(&n) else { continue };
+                match n.appearance.fill_paint() {
+                    // The points take the colours the gradient paints there.
+                    Paint::Gradient(g) => {
+                        path.bounds().and_then(|b| GradientMesh::for_path_with(&path, rows, cols, &*gradient_sampler(&g, b))).map(|mut m| {
+                            m.highlight(app, highlight);
+                            m
+                        })
+                    }
+                    p => GradientMesh::for_path(&path, rows, cols, p.color().unwrap_or(Color::WHITE), app, highlight),
                 }
-                p => GradientMesh::for_path(&path, rows, cols, p.color().unwrap_or(Color::WHITE), app, highlight),
             };
             let Some(mut m) = m else { continue };
             if let Some(q) = at {
@@ -1375,7 +1390,7 @@ fn mesh_create(s: &mut Session, p: &Value) -> Result<Value> {
             out.push(*id);
         }
         if out.is_empty() {
-            return Err(bad(C, "select a path or compound path"));
+            return Err(bad(C, refused.unwrap_or("select a path, a compound path or an embedded image")));
         }
         sel.set(out.iter().copied());
         Ok((out, index))
@@ -1387,6 +1402,31 @@ fn mesh_create(s: &mut Session, p: &Value) -> Result<Value> {
     Ok(r)
 }
 
+/// Create Gradient Mesh on an embedded image: a `rows` × `cols` mesh over the picture's rectangle
+/// (placed with the image's transform), each point taking the average colour and opacity of the
+/// area around it. Linked images are refused: the documentation allows embedded ones only.
+fn image_mesh(d: &Document, im: &vectorcraft_doc::ImageObject, rows: u32, cols: u32) -> std::result::Result<GradientMesh, &'static str> {
+    if im.link.is_some() {
+        return Err("a linked image can't become a gradient mesh: embed it first");
+    }
+    let xf = im.xf.as_coeffs();
+    if im.width == 0 || im.height == 0 || !xf.iter().all(|k| k.is_finite()) {
+        return Err("the image has no usable size or position");
+    }
+    let samples = d.images.get(&im.key).and_then(|b| b.grid_samples(rows, cols)).ok_or("the image's pixels can't be read")?;
+    let (w, h) = (im.width as f64, im.height as f64);
+    let at = |u: f64, v: f64| im.xf * Point::new(u * w, v * h);
+    if ![at(0.0, 0.0), at(1.0, 0.0), at(0.0, 1.0), at(1.0, 1.0)].iter().all(|p| p.x.abs() < 1e15 && p.y.abs() < 1e15) {
+        return Err("the image has no usable size or position");
+    }
+    let cols_n = cols as usize + 1;
+    let steps = |n: u32| (0..=n).map(|i| i as f64 / n as f64).collect::<Vec<_>>();
+    Ok(GradientMesh::from_grid(&steps(cols), &steps(rows), &at, &|r, c, _| {
+        let [pr, pg, pb, pa] = samples.get(r * cols_n + c).copied().unwrap_or([0; 4]);
+        (Color::rgb8(pr, pg, pb), pa as f32 / 255.0)
+    }))
+}
+
 /// Run `f` on the gradient mesh `p.id` (one undo step `label`); with `envelopes`, a mesh
 /// envelope's grid too (its points and handles are written back: editing fills in the handles).
 fn with_mesh<T>(s: &mut Session, p: &Value, cmd: &str, label: &str, envelopes: bool, f: impl FnOnce(&mut GradientMesh) -> Result<T>) -> Result<T> {

crates/engine/src/tests_live.rs

@@ -732,3 +732,66 @@ fn disabled_without_live_selection() {
     assert!(s.execute("object.blend.make", &json!({})).is_err(), "needs two objects");
     let _ = Paint::solid(Color::BLACK);
 }
+
+/// A selected embedded 8×4 image at (100, 50), 2 pt per pixel: red on the left half, blue on the
+/// right half (which is transparent when `clear_right`). `link` makes it a linked image.
+fn two_colour_image(s: &mut Session, clear_right: bool, linked: bool) -> NodeId {
+    let right = image::Rgba([0, 0, 255, if clear_right { 0 } else { 255 }]);
+    let img = image::RgbaImage::from_fn(8, 4, |x, _| if x < 4 { image::Rgba([255, 0, 0, 255]) } else { right });
+    let mut png = vec![];
+    img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
+    s.edit("Place", |d, sel| {
+        d.images.insert("pic".into(), vectorcraft_doc::ImageBlob::png(png));
+        let id = d.alloc_id();
+        let xf = vectorcraft_geom::Affine::translate((100.0, 50.0)) * vectorcraft_geom::Affine::scale(2.0);
+        let link = linked.then(|| vectorcraft_doc::LinkInfo { path: "/tmp/pic.png".into(), ..Default::default() });
+        let im = vectorcraft_doc::ImageObject { key: "pic".into(), width: 8, height: 4, xf, link, placement: Default::default() };
+        d.insert(d.default_layer(), 0, Node::new(id, NodeKind::Image(Box::new(im))))?;
+        sel.set([id]);
+        Ok(id)
+    })
+    .unwrap()
+}
+
+#[test]
+fn mesh_create_on_an_embedded_image_takes_its_colours_and_placement() {
+    let mut s = session();
+    let id = two_colour_image(&mut s, false, false);
+    let undo = s.doc().unwrap().history.undo.len();
+    s.execute("object.mesh.create", &json!({"rows": 1, "cols": 2})).unwrap();
+    assert_eq!(s.doc().unwrap().history.undo.len(), undo + 1, "one undo step");
+    let m = mesh_of(&s, id);
+    assert_eq!((m.rows, m.cols, m.points.len()), (1, 2, 6));
+    // Row 0 is the image's top: red at the left edge, the average of both halves in the middle
+    // (an area, not one pixel), blue at the right edge.
+    let hex = |i: usize| m.points[i].color.to_hex();
+    assert_eq!((hex(0).as_str(), hex(1).as_str(), hex(2).as_str()), ("#ff0000", "#800080", "#0000ff"));
+    assert_eq!((hex(3).as_str(), hex(5).as_str()), ("#ff0000", "#0000ff"));
+    assert!(m.points.iter().all(|p| p.opacity == 1.0));
+    // The mesh keeps the image's transform: (100, 50) to (116, 58).
+    let near = |p: Point, x: f64, y: f64| close(p.x, x) && close(p.y, y);
+    assert!(near(m.points[0].p, 100.0, 50.0) && near(m.points[2].p, 116.0, 50.0) && near(m.points[5].p, 116.0, 58.0));
+    assert_eq!(node(&s, id).kind_label(), "Mesh");
+    s.execute("edit.undo", &json!({})).unwrap();
+    assert_eq!(node(&s, id).kind_label(), "Image");
+}
+
+#[test]
+fn mesh_create_on_an_image_keeps_alpha_and_refuses_linked_and_big_grids() {
+    let mut s = session();
+    let id = two_colour_image(&mut s, true, false);
+    s.execute("object.mesh.create", &json!({"rows": 1, "cols": 1})).unwrap();
+    let m = mesh_of(&s, id);
+    assert_eq!(m.points.iter().map(|p| p.opacity).collect::<Vec<_>>(), [1.0, 0.0, 1.0, 0.0]);
+    assert_eq!(m.points[0].color.to_hex(), "#ff0000");
+
+    let mut s = session();
+    two_colour_image(&mut s, false, false);
+    assert!(s.execute("object.mesh.create", &json!({"rows": 51, "cols": 4})).is_err(), "grid capped");
+
+    let mut s = session();
+    let id = two_colour_image(&mut s, false, true);
+    let e = s.execute("object.mesh.create", &json!({})).unwrap_err().to_string();
+    assert!(e.contains("linked"), "{e}");
+    assert_eq!(node(&s, id).kind_label(), "Image", "left as it was");
+}

crates/engine/tests/command_sweep.rs

@@ -735,3 +735,63 @@ fn text_commands_with_junk_params_on_inline_graphics() {
     }
     assert!(failures.is_empty(), "{} failures:\n{}", failures.len(), failures.join("\n"));
 }
+
+/// Create Gradient Mesh on embedded, linked and broken images (bad pixels, zero size, a NaN
+/// transform) with junk rows and columns: never a panic or a broken tree.
+#[test]
+fn gradient_mesh_from_images_with_junk_params() {
+    use vectorcraft_doc::{ImageBlob, ImageObject, LinkInfo, Node, NodeKind};
+    use vectorcraft_geom::Affine;
+    let img = image::RgbaImage::from_fn(5, 3, |x, y| image::Rgba([x as u8 * 50, y as u8 * 90, 7, if x == 2 { 0 } else { 255 }]));
+    let mut png = vec![];
+    img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap();
+    let params = [
+        json!({}),
+        json!({"rows": 1, "cols": 1}),
+        json!({"rows": 50, "cols": 50}),
+        json!({"rows": 0, "cols": 3}),
+        json!({"rows": 51, "cols": 3}),
+        json!({"rows": -1, "cols": 1e308}),
+        json!({"rows": "x", "columns": null, "appearance": 5}),
+        json!({"rows": 7, "cols": 9, "appearance": "edge", "highlight": f64::MAX}),
+        json!({"at": [2.5, 1.0]}),
+        json!({"at": [1e308, -1e308]}),
+        json!({"ids": [u64::MAX]}),
+    ];
+    // (stored bytes, width, height, transform, linked)
+    let variants: [(Vec<u8>, u32, u32, Affine, bool); 6] = [
+        (png.clone(), 5, 3, Affine::scale(4.0), false),
+        (png.clone(), 5, 3, Affine::scale(4.0), true),
+        (png.clone(), 0, 0, Affine::scale(4.0), false),
+        (png.clone(), 5, 3, Affine::new([f64::NAN, 0.0, 0.0, 1.0, 0.0, 0.0]), false),
+        (png[..png.len() / 2].to_vec(), 5, 3, Affine::IDENTITY, false),
+        (png.clone(), u32::MAX, u32::MAX, Affine::scale(1e300), false),
+    ];
+    let mut failures = vec![];
+    for (i, (bytes, w, h, xf, linked)) in variants.iter().enumerate() {
+        for p in &params {
+            let mut s = Fixture::Empty.session();
+            let placed = s.edit("Place", |d, sel| {
+                d.images.insert("pic".into(), ImageBlob::png(bytes.clone()));
+                let id = d.alloc_id();
+                let link = linked.then(|| LinkInfo { path: "/nonexistent/pic.png".into(), ..Default::default() });
+                let im = ImageObject { key: "pic".into(), width: *w, height: *h, xf: *xf, link, placement: Default::default() };
+                d.insert(d.default_layer(), 0, Node::new(id, NodeKind::Image(Box::new(im))))?;
+                sel.set([id]);
+                Ok(())
+            });
+            if placed.is_err() {
+                // Insert already refuses a NaN or out-of-range transform.
+                assert!(i >= 3, "variant {i}: {placed:?}");
+                continue;
+            }
+            let r = catch_quiet(|| no_panic(s.execute("object.mesh.create", p)));
+            if r.is_err() {
+                failures.push(format!("PANIC variant {i} {p}"));
+            } else if let Err(e) = check_session(&s) {
+                failures.push(format!("variant {i} {p}: {e}"));
+            }
+        }
+    }
+    assert!(failures.is_empty(), "{} failures:\n{}", failures.len(), failures.join("\n"));
+}

docs/gaps.md

@@ -227,8 +227,10 @@ from G14 and the budget run of G15: **~170–270 h** one agent, ~50–80 h wall
   where three or more overlap. The group renders, saves and exports through its live expansion. Still missing: the
   right-click Bring Forward / Send Backward menu where three or more objects overlap; adding objects to the group by
   dragging them in the Layers panel is untested.
-- **Missing (asked for by users, from earlier versions):** Object › Create Gradient
-  Mesh on an embedded image, which turns it into a mesh of its colours (#1676); radial gradients in Image Trace's
+- **Done (from users' reports):** Object › Create Gradient Mesh on an embedded image turns it into a mesh over its
+  rectangle, with the image's transform kept; each point takes the average colour and opacity of the picture around it
+  (#1676). A linked image is refused, as the public documentation has it.
+- **Missing (asked for by users, from earlier versions):** radial gradients in Image Trace's
   Gradients option, which traces linear ones only (draft #1849). Inside the estimate below.
 - **Missing:** gradient dither and perceptual blending, gradient presets (30.0); tangent and perpendicular
   snapping and the consolidated snapping options (30.0); artboard locking, artboard colours and managing
@@ -355,6 +357,7 @@ from G14 and the budget run of G15: **~170–270 h** one agent, ~50–80 h wall
 | 2026-10-14 | minor | G6: image variables (`VariableKind::Image`, `variable.makeImageDynamic`, path values on `dataset.select` keeping bounds); remaining: scripting surface, Batch, graph variables, dataset import/export; estimate 8–14 → 6–12 h |
 | 2026-10-14 | minor | Effect > Warp: style, orientation and percent controls; focused checks passed and original captures reviewed; scores and hours unchanged |
 | 2026-10-14 | minor | G11: open-path anchor Delete removes adjoining segments without bridging the gap; explicit Remove Anchors keeps its reconnect/refit behavior. Scores and hours unchanged. |
+| 2026-10-16 | minor | G13: Object › Create Gradient Mesh on an embedded image (#1676); a linked image is refused. No entry closed, estimates unchanged |
 | 2026-10-15 | minor | G13: Object › Intertwine (#1763): Make, Edit, Release, the Intertwine tool (hover, click, drawn path) and `object.intertwine.arrange` (toggle, front, back). Bring Forward / Send Backward stay missing; adding objects through the Layers panel is untested. No entry closed, estimates unchanged |
 | 2026-10-14 | minor | MCP `add_text` exposes the existing `vertical` option for point, area and path type; horizontal remains the default; scores and hours unchanged. |
 | 2026-10-14 | minor | Vertical fullwidth question and exclamation marks keep their nominal glyph upright when the font has no vertical alternate; scores and hours unchanged. |