Images: an image zoomed in on has its colours made only for the tiles in view (#4155)

From about 50% zoom on the file of #4155, the pixels drawn are the full 141-megapixel photo,
and in a CMYK document they were separated into ink planes (or colorized, or proofed) whole
for every view: 8 to 9 s of white canvas a zoom step.

When the pixels drawn are larger than a 1,024-pixel tile, their colours are made again for the
frame and part of them is out of view, only the tiles in view are made, each kept under its own
key (`ImageKey::tile`), and drawn together as one pixmap so their edges don't show. The colours
are made pixel by pixel, so the result is the same as for the whole image. CMYK images painting
their own inks are still made whole.

Zooming in to 50%, 100% and 200% on that file went from 9.0, 8.6 and 8.7 s to 0.78, 0.05 and
0.05 s (M3 Pro, one run each), with the same pixels on screen.

crates/render/src/blobs.rs

@@ -21,6 +21,7 @@ pub(crate) fn blob_key(blob: &vectorcraft_doc::ImageBlob) -> (images::ImageKey,
         luts: (lut.as_ref().map_or(0, |l| l.1), 0),
         plates: 0,
         mip: 0,
+        tile: None,
     };
     (key, lut.map(|l| l.0), id.size)
 }

crates/render/src/images.rs

@@ -108,6 +108,8 @@ pub(crate) struct ImageKey {
     pub plates: u64,
     /// Halved this many times ([`half`]).
     pub mip: u8,
+    /// A [`TILE`]-pixel square of the pixels (column, row), when only part of them is made.
+    pub tile: Option<(u16, u16)>,
 }
 
 /// A blob's content key and whether it holds CMYK inks, worked out once per blob allocation, and
@@ -413,6 +415,36 @@ fn mean(v: [u8; 4]) -> u8 {
     ((v.iter().map(|&c| u32::from(c)).sum::<u32>() + 2) / 4) as u8
 }
 
+/// The side of the squares whose colours are made one by one when only part of an image's
+/// pixels show ([`ImageKey::tile`]): a large photo zoomed in on is separated, colorized or proofed
+/// only where it shows.
+pub(crate) const TILE: u32 = 1024;
+
+/// The `w`×`h` pixels of `pm` from (`x`, `y`). `None` unless they lie inside `pm`.
+pub(crate) fn crop(pm: &Pixmap, x: u32, y: u32, w: u32, h: u32) -> Option<Pixmap> {
+    let (pw, ph) = (u32::from(pm.width()), u32::from(pm.height()));
+    if w == 0 || h == 0 || x.checked_add(w)? > pw || y.checked_add(h)? > ph {
+        return None;
+    }
+    let (stride, x, w) = (pw as usize, x as usize, w as usize);
+    let rows = pm.data().chunks_exact(stride).skip(y as usize).take(h as usize);
+    let data: Vec<PremulRgba8> = rows.flat_map(|row| row.get(x..x + w).unwrap_or_default().iter().copied()).collect();
+    (data.len() == w * h as usize).then(|| Pixmap::from_parts(data, w as u16, h as u16))
+}
+
+/// `src` copied into `dst` with its top left at (`x`, `y`), as far as `dst` reaches.
+pub(crate) fn paste(dst: &mut Pixmap, src: &Pixmap, x: usize, y: usize) {
+    let (dw, sw) = (usize::from(dst.width()), usize::from(src.width()));
+    let dst_rows = dst.data_mut().chunks_exact_mut(dw).skip(y);
+    for (to, from) in dst_rows.zip(src.data().chunks_exact(sw)) {
+        let row = to.get_mut(x..).unwrap_or_default();
+        let n = row.len().min(from.len());
+        if let (Some(to), Some(from)) = (row.get_mut(..n), from.get(..n)) {
+            to.copy_from_slice(from);
+        }
+    }
+}
+
 /// Pixels `pm` colorized with `rgb`: each pixel takes the color, as opaque as the pixel is dark
 /// (255 less its Rec. 709 luma) times its alpha, so white pixels are transparent.
 pub(crate) fn colorized(pm: &Pixmap, [r, g, b]: [u8; 3]) -> Pixmap {
@@ -448,7 +480,7 @@ mod tests {
     use super::*;
 
     fn key(content: &str) -> ImageKey {
-        ImageKey { content: content.into(), adjust: None, grey: false, tint: None, ink: Ink::Display, luts: (0, 0), plates: 0, mip: 0 }
+        ImageKey { content: content.into(), adjust: None, grey: false, tint: None, ink: Ink::Display, luts: (0, 0), plates: 0, mip: 0, tile: None }
     }
 
     #[test]

crates/render/src/ink.rs

@@ -395,11 +395,17 @@ pub(crate) fn cmyk_pixmap(blob: &ImageBlob, lut: &InkLut) -> Option<Pixmap> {
     inks_pixmap(&blob.cmyk()?, lut)
 }
 
+/// Whether image pixels `key` are a CMYK image's inks as shown (not adjusted or greyscale), so
+/// they paint those inks ([`key_inks`]).
+pub(crate) fn shows_own_inks(key: &ImageKey) -> bool {
+    key.luts.0 != 0 && key.adjust.is_none() && !key.grey
+}
+
 /// The ink amounts behind image pixels `key` when they are CMYK image `blob`'s inks as shown (not
 /// adjusted or greyscale): its own inks, halved as its pixels are to `key`'s mip
 /// ([`images::half_inks`]). `None` for other pixels.
 pub(crate) fn key_inks(key: &ImageKey, blob: Option<&ImageBlob>) -> Option<Inks> {
-    if key.luts.0 == 0 || key.adjust.is_some() || key.grey {
+    if !shows_own_inks(key) {
         return None;
     }
     (0..key.mip).try_fold(blob?.cmyk()?, |inks, _| images::half_inks(&inks))
@@ -538,6 +544,7 @@ mod tests {
             luts: (0, 0),
             plates: 0,
             mip: 0,
+            tile: None,
         };
         let pm =
             Arc::new(Pixmap::from_parts(vec![PremulRgba8 { r: 204, g: 102, b: 51, a: 255 }, PremulRgba8 { r: 102, g: 51, b: 26, a: 128 }], 2, 1));

crates/render/src/renderer/images.rs

@@ -71,45 +71,48 @@ impl Renderer {
                     }
                 }
             }
-            kept
+            kept.map(Shown::whole)
         } else if let Some(pm) = mips.then(|| self.kept_mip(f, &im.key, im.pixels.as_deref(), outline, tint, unit)).flatten() {
-            Some(pm)
+            Some(Shown::whole(pm))
         } else {
             self.image_pixmap(f.doc, &im.key, im.pixels.as_deref(), outline, !mips).map(|(key, pm)| {
                 // Shrunk first, so what is made of the pixels below is made of the ones drawn: a
                 // 141-megapixel photo is separated at the size it shows, not at its own (#4155).
                 let (key, pm) = if mips { image_mip(key, pm, unit) } else { (key, pm) };
                 let blob = f.doc.image_blob(im);
-                let (key, pm) = match tint {
-                    Some(rgb) => colorized_pixmap(key, &pm, rgb),
-                    None => (key, pm),
-                };
-                // A colorized image is already in the frame's ink ([`image_tint`]), and with
-                // Separations Preview its color is already the visible plates' (`proof::prepare`).
-                let (key, pm) = if outline || tint.is_some() { (key, pm) } else { ink::ink_image(&key, &pm, f.ink, blob, true) };
-                match f.plates.filter(|_| !outline && tint.is_none()) {
-                    Some(lut) => proof::plate_image(lut, &key, &pm, blob, true).1,
-                    None => pm,
+                // Zoomed in on it, only the tiles in view.
+                if let Some(window) = in_view(f, im, &key, &pm, outline, tint) {
+                    return window_pixmap(f, &key, &pm, window, outline, tint, blob);
                 }
+                Shown::whole(shown_pixmap(f, key, pm, outline, tint, blob))
             })
         };
-        if let Some(pm) = shown {
-            let sx = im.width as f64 / pm.width().max(1) as f64;
-            let sy = im.height as f64 / pm.height().max(1) as f64;
+        if let Some(Shown { pm, level: (lw, lh), at: (ax, ay) }) = shown {
+            // The pixels drawn are `pm`, from (`ax`, `ay`) of the `lw`×`lh` pixels of the level.
+            let sx = im.width as f64 / f64::from(lw.max(1));
+            let sy = im.height as f64 / f64::from(lh.max(1));
+            let drawn = Affine::scale_non_uniform(sx, sy) * Affine::translate((f64::from(ax), f64::from(ay)));
+            let part = Rect::new(
+                f64::from(ax) * sx,
+                f64::from(ay) * sy,
+                f64::from(ax + u32::from(pm.width())) * sx,
+                f64::from(ay + u32::from(pm.height())) * sy,
+            )
+            .intersect(rect);
             let quality = if f.opts.smooth_images && !pixelated { peniko::ImageQuality::Medium } else { peniko::ImageQuality::Low };
             let sampler = peniko::ImageSampler { quality, ..Default::default() };
             ctx.set_paint(vello_cpu::Image { image: vello_cpu::ImageSource::Pixmap(pm), sampler });
             match area {
                 Some((bp, rule)) => {
                     ctx.set_transform(f.view);
-                    ctx.set_paint_transform(im.xf * Affine::scale_non_uniform(sx, sy));
+                    ctx.set_paint_transform(im.xf * drawn);
                     ctx.set_fill_rule(fill_rule(*rule));
                     ctx.fill_path(bp);
                 }
                 None => {
                     ctx.set_transform(f.view * im.xf);
-                    ctx.set_paint_transform(Affine::scale_non_uniform(sx, sy));
-                    ctx.fill_rect(&rect);
+                    ctx.set_paint_transform(drawn);
+                    ctx.fill_rect(&part);
                 }
             }
             ctx.reset_paint_transform();
@@ -307,6 +310,117 @@ impl Renderer {
     }
 }
 
+/// Image pixels [`Renderer::draw_image_in`] draws: `pm`, from (`at`) of the `level` pixels of the
+/// mip drawn (all of them, or the tiles in view).
+struct Shown {
+    pm: Arc<Pixmap>,
+    level: (u32, u32),
+    at: (u32, u32),
+}
+
+impl Shown {
+    /// All of the pixels of a level.
+    fn whole(pm: Arc<Pixmap>) -> Self {
+        let level = (u32::from(pm.width()), u32::from(pm.height()));
+        Self { pm, level, at: (0, 0) }
+    }
+}
+
+/// Image pixels `pm` (kept as `key`) as [`Renderer::draw_image_in`] draws them: in greyscale with
+/// `grey` (the outline view's) or colorized with `tint` as they are, else painted with the frame's
+/// ink and shown through its Separations Preview plates. Kept with the images under
+/// [`shown_key`], and marked used there.
+fn shown_pixmap(
+    f: &Frame,
+    key: images::ImageKey,
+    pm: Arc<Pixmap>,
+    grey: bool,
+    tint: Option<[u8; 3]>,
+    blob: Option<&vectorcraft_doc::ImageBlob>,
+) -> Arc<Pixmap> {
+    let (key, pm) = match tint {
+        Some(rgb) => colorized_pixmap(key, &pm, rgb),
+        None => (key, pm),
+    };
+    // A colorized image is already in the frame's ink ([`image_tint`]), and with Separations
+    // Preview its color is already the visible plates' (`proof::prepare`).
+    if grey || tint.is_some() {
+        return pm;
+    }
+    let (key, pm) = ink::ink_image(&key, &pm, f.ink, blob, true);
+    match f.plates {
+        Some(lut) => proof::plate_image(lut, &key, &pm, blob, true).1,
+        None => pm,
+    }
+}
+
+/// The tiles of image pixels `pm` (kept as `key`, the level of image `im` drawn) in the frame's
+/// view, as a range of columns and rows, when the pixels' colours are made again
+/// ([`shown_pixmap`]: colorized, separated or proofed), more than one tile of them is made, and
+/// some are out of view. `None` otherwise: all the pixels are made, as they are for a CMYK image's
+/// own inks (made from the whole image's).
+fn in_view(
+    f: &Frame,
+    im: &vectorcraft_doc::ImageObject,
+    key: &images::ImageKey,
+    pm: &Pixmap,
+    grey: bool,
+    tint: Option<[u8; 3]>,
+) -> Option<([u32; 2], [u32; 2])> {
+    let made = !grey && (tint.is_some() || ink::ink_key(key, f.ink) != *key || f.plates.is_some());
+    let (w, h) = (u32::from(pm.width()), u32::from(pm.height()));
+    if !made || (w <= images::TILE && h <= images::TILE) || ink::shows_own_inks(key) {
+        return None;
+    }
+    // The view in the level's pixels, a couple wider for smoothing at its edge.
+    let to_level = (im.xf * Affine::scale_non_uniform(im.width as f64 / f64::from(w), im.height as f64 / f64::from(h))).inverse();
+    let v = f.visible;
+    let corners = [(v.x0, v.y0), (v.x1, v.y0), (v.x1, v.y1), (v.x0, v.y1)].map(|c| to_level * vectorcraft_geom::Point::from(c));
+    let (x0, x1) = corners.iter().fold((f64::INFINITY, f64::NEG_INFINITY), |(a, b), p| (a.min(p.x), b.max(p.x)));
+    let (y0, y1) = corners.iter().fold((f64::INFINITY, f64::NEG_INFINITY), |(a, b), p| (a.min(p.y), b.max(p.y)));
+    if ![x0, x1, y0, y1].iter().all(|v| v.is_finite()) {
+        return None;
+    }
+    let tiles = |lo: f64, hi: f64, n: u32| {
+        let last = n.saturating_sub(1) / images::TILE;
+        let at = |v: f64| ((v.clamp(0.0, f64::from(n.saturating_sub(1))) as u32) / images::TILE).min(last);
+        ([at(lo - 2.0), at(hi + 2.0)], last)
+    };
+    let ((cols, last_col), (rows, last_row)) = (tiles(x0, x1, w), tiles(y0, y1, h));
+    // All of them in view: made whole, as without tiles.
+    (cols != [0, last_col] || rows != [0, last_row]).then_some((cols, rows))
+}
+
+/// The tiles `cols` × `rows` of image pixels `pm` (kept as `key`) as [`shown_pixmap`] makes them,
+/// each made and kept on its own ([`images::ImageKey::tile`]), in one pixmap.
+fn window_pixmap(
+    f: &Frame,
+    key: &images::ImageKey,
+    pm: &Pixmap,
+    (cols, rows): ([u32; 2], [u32; 2]),
+    grey: bool,
+    tint: Option<[u8; 3]>,
+    blob: Option<&vectorcraft_doc::ImageBlob>,
+) -> Shown {
+    let (w, h) = (u32::from(pm.width()), u32::from(pm.height()));
+    let t = images::TILE;
+    let at = (cols[0] * t, rows[0] * t);
+    let size = (((cols[1] + 1) * t).min(w) - at.0, ((rows[1] + 1) * t).min(h) - at.1);
+    let mut window = Pixmap::new(size.0 as u16, size.1 as u16);
+    for (col, row) in (rows[0]..=rows[1]).flat_map(|r| (cols[0]..=cols[1]).map(move |c| (c, r))) {
+        let tile_key = images::ImageKey { tile: Some((col as u16, row as u16)), ..key.clone() };
+        let (x, y) = (col * t, row * t);
+        let made = lookup(&shown_key(f, tile_key.clone(), grey, tint), true).or_else(|| {
+            let part = images::crop(pm, x, y, t.min(w - x), t.min(h - y))?;
+            Some(shown_pixmap(f, tile_key, Arc::new(part), grey, tint, blob))
+        });
+        if let Some(made) = made {
+            images::paste(&mut window, &made, (x - at.0) as usize, (y - at.1) as usize);
+        }
+    }
+    Shown { pm: Arc::new(window), level: (w, h), at }
+}
+
 /// The key [`Renderer::draw_image_in`] gives the pixels it draws of image pixels `key`: in
 /// greyscale with `grey` (the outline view's) or colorized with `tint` as they are, else painted
 /// with the frame's ink and shown through its Separations Preview plates.

crates/render/src/tests/memory.rs

@@ -2494,3 +2494,53 @@ fn a_reduced_image_on_the_ink_planes_shows_its_average() {
         assert!(px.iter().zip(want).all(|(a, b)| a.abs_diff(b) <= 2), "pixel {i}: {px:?}, the full frame averages {want:?}");
     }
 }
+
+/// A `w`×`h` PNG with a different colour in every pixel, so a pixel drawn from the wrong place
+/// shows; its pixels depend on `seed` (each test its own image: the store is shared).
+fn varied_png(w: u32, h: u32, seed: u8) -> Vec<u8> {
+    let img = image::RgbaImage::from_fn(w, h, |x, y| image::Rgba([(x * 5 + y) as u8, (x / 3 + y * 7) as u8, (x ^ y) as u8 ^ seed, 255]));
+    let mut out = vec![];
+    img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png).unwrap();
+    out
+}
+
+/// #4155: an image drawn larger than a tile with its colours made again (here separated into ink
+/// planes) has them made only for the tiles in view. Zoomed in on a 141-megapixel photo, the whole
+/// photo was separated for every view, taking seconds each time.
+#[test]
+fn a_zoomed_in_image_is_separated_only_where_it_shows() {
+    let _fonts = fonts_steady();
+    let (w, h) = (2 * images::TILE + 100, 40);
+    let d = cmyk_doc_with("wide", ImageBlob::new("image/png", varied_png(w, h, 1)), w, h);
+    let on = RenderOptions { image_mips: true, ..Default::default() };
+    let _ = Renderer::new().render_region_inks(&d, Rect::new(0.0, 0.0, 300.0, 40.0), 1.0, &on);
+    let full = blob_key(&d.images["wide"]).0;
+    let s = images::store();
+    for plane in [Ink::Cmy, Ink::K] {
+        assert!(s.peek(&ink::ink_key(&full, plane)).is_none(), "the whole {plane:?} plane was made");
+        let tile = |col: u16| ink::ink_key(&images::ImageKey { tile: Some((col, 0)), ..full.clone() }, plane);
+        assert!(s.peek(&tile(0)).is_some(), "the {plane:?} plane of the tile in view was made");
+        assert!(s.peek(&tile(1)).is_none() && s.peek(&tile(2)).is_none(), "tiles out of view were made");
+    }
+}
+
+/// An image drawn from tiles draws the pixels it draws whole, across the tiles' edges too.
+#[test]
+fn a_tiled_image_draws_what_the_whole_image_draws() {
+    let _fonts = fonts_steady();
+    let (w, h) = (2 * images::TILE + 100, 40);
+    let d = cmyk_doc_with("seams", ImageBlob::new("image/png", varied_png(w, h, 2)), w, h);
+    let on = RenderOptions { image_mips: true, ..Default::default() };
+    // Across the edge between the first two tiles.
+    let (x0, x1) = (f64::from(images::TILE) - 150.0, f64::from(images::TILE) + 150.0);
+    let drawn = Renderer::new().render_region_inks(&d, Rect::new(x0, 0.0, x1, 40.0), 1.0, &on);
+    let (key, full) = crate::blobs::blob_pixmap(&d.images["seams"], false).unwrap();
+    let plane = |ink| ink::ink_image(&key, &full, ink, d.images.get("seams"), false).1;
+    let bytes = |pm: &Pixmap| pm.data().iter().flat_map(|p| [p.r, p.g, p.b, p.a]).collect::<Vec<u8>>();
+    let whole = ink::amounts(&bytes(&plane(Ink::Cmy)), &bytes(&plane(Ink::K)));
+    for (i, got) in drawn.as_chunks::<4>().0.iter().enumerate() {
+        let (x, y) = (x0 as usize + i % 300, i / 300);
+        let want = &whole[(y * w as usize + x) * 4..][..4];
+        assert!(got.iter().zip(want).all(|(a, b)| a.abs_diff(*b) <= 1), "pixel ({x}, {y}): {got:?}, whole {want:?}");
+    }
+}