Fix #2820: images drawn small redraw from their kept mips while being moved

With image mips on (the canvas always sets them), drawing an image
fetched its full decoded pixels first, decoding them again when the
shared image store no longer held them, and only then looked up the mip
it draws. The full pixels are only peeked, so they are the least
recently used entries and the first the store drops over its budget
(1 GiB on macOS and Windows). Once a document's full-size images exceed
it (20 photos of 6000 x 4000 px are 1.9 GB as RGBA), every frame decoded
every image again, though only their small mips were drawn: moving one
image redrew seconds later.

The store now remembers the size each blob's pixels decode to, so the
mip level is found without the pixels, and a mip the store holds is
drawn (and marked used) without fetching or decoding them. Without the
mip, at full size, or with mips off (exports) drawing is unchanged.

Measured on an Apple M4 (load average 15-24 from other work), 20 noise
PNGs of 6000 x 4000 px fitted in a 2880 x 1800 view, one moved 5 pt per
frame: before, 20 decodes and 1.1-3.0 s every frame; after, the first
frame decodes 20 and the later ones none, in 3-5 ms.

crates/render/src/images.rs

@@ -66,11 +66,13 @@ pub(crate) struct ImageKey {
     pub mip: u8,
 }
 
-/// A blob's content key and whether it holds CMYK inks, worked out once per blob allocation.
+/// A blob's content key and whether it holds CMYK inks, worked out once per blob allocation, and
+/// the size of its pixels once they have been decoded ([`remember_size`]).
 #[derive(Clone)]
 pub(crate) struct BlobId {
     pub content: Arc<str>,
     pub cmyk: bool,
+    pub size: Option<(u16, u16)>,
 }
 
 struct Entry {
@@ -114,6 +116,14 @@ impl ImageStore {
         self.entries.get(key).map(|e| e.pm.clone())
     }
 
+    /// Drop `key`'s pixels, as the budget drops the least recently used.
+    #[cfg(test)]
+    pub(crate) fn remove(&mut self, key: &ImageKey) {
+        if let Some(e) = self.entries.remove(key) {
+            self.bytes = self.bytes.saturating_sub(pixmap_bytes(&e.pm));
+        }
+    }
+
     /// Keep `pm` as `key` (or, when another renderer made it meanwhile, return theirs), then drop
     /// entries over budget.
     pub(crate) fn insert(&mut self, key: ImageKey, pm: Arc<Pixmap>) -> Arc<Pixmap> {
@@ -167,6 +177,15 @@ impl ImageStore {
         }
         self.ids.insert(Arc::as_ptr(bytes) as usize, (Arc::downgrade(bytes), id));
     }
+
+    fn remember_size(&mut self, bytes: &Arc<Vec<u8>>, size: (u16, u16)) {
+        let at = Arc::as_ptr(bytes);
+        if let Some((w, id)) = self.ids.get_mut(&(at as usize))
+            && std::ptr::eq(w.as_ptr(), at)
+        {
+            id.size = Some(size);
+        }
+    }
 }
 
 static STORE: LazyLock<Mutex<ImageStore>> = LazyLock::new(|| Mutex::new(ImageStore::new(default_budget())));
@@ -193,11 +212,17 @@ pub(crate) fn blob_id(blob: &ImageBlob) -> BlobId {
         return id;
     }
     // Outside the lock: hashing a large image takes a while.
-    let id = BlobId { content: format!("{}:{}", blob.content_key(), blob.bytes.len()).into(), cmyk: blob.is_cmyk() };
+    let id = BlobId { content: format!("{}:{}", blob.content_key(), blob.bytes.len()).into(), cmyk: blob.is_cmyk(), size: None };
     store().remember_id(&blob.bytes, id.clone());
     id
 }
 
+/// Note that `blob`'s pixels are `size`: [`blob_id`] tells it from then on, so that the mips made
+/// of them are found while the store holds the mips but not the pixels.
+pub(crate) fn remember_size(blob: &ImageBlob, size: (u16, u16)) {
+    store().remember_size(&blob.bytes, size);
+}
+
 /// `pm` at half the size (rounded up), each pixel the mean of the 2×2 it covers (the last row and
 /// column repeat on odd sizes).
 pub(crate) fn half(pm: &Pixmap) -> Pixmap {

crates/render/src/ink.rs

@@ -311,18 +311,33 @@ pub(crate) fn cmyk_pixmap(blob: &ImageBlob, lut: &InkLut) -> Option<Pixmap> {
     inks_pixmap(&blob.cmyk()?, lut)
 }
 
+/// The separation tables of placed images ([`Ink::Cmy`], [`Ink::K`]) and their generation.
+fn separation() -> (Arc<[ProofLut; 2]>, u64) {
+    cached(&SEPARATE, |cms| {
+        let intent = cms.settings().intent;
+        [Ink::Cmy, Ink::K].map(|ink| ProofLut::build(|rgb| ink.plane(cms.srgb_to_cmyk(rgb, intent))))
+    })
+}
+
+/// The key of image pixels `key` painted as `plane` with the separation tables of `generation`.
+fn plane_key(key: &ImageKey, plane: Ink, generation: u64) -> ImageKey {
+    ImageKey { ink: plane, luts: (key.luts.0, generation), ..key.clone() }
+}
+
+/// The key [`ink_image`] gives image pixels `key` painted with `ink`.
+pub(crate) fn ink_key(key: &ImageKey, ink: Ink) -> ImageKey {
+    if matches!(ink, Ink::Display | Ink::RichBlack) { key.clone() } else { plane_key(key, ink, separation().1) }
+}
+
 /// Image pixels `pm` (cached as `key`, from `blob` when it is a document's image as it is) as
 /// painted with `ink`, cached with the images, and their key.
 pub(crate) fn ink_image(key: &ImageKey, pm: &Arc<Pixmap>, ink: Ink, blob: Option<&ImageBlob>, touch: bool) -> (ImageKey, Arc<Pixmap>) {
     if matches!(ink, Ink::Display | Ink::RichBlack) {
         return (key.clone(), pm.clone());
     }
-    let (luts, generation) = cached(&SEPARATE, |cms| {
-        let intent = cms.settings().intent;
-        [Ink::Cmy, Ink::K].map(|ink| ProofLut::build(|rgb| ink.plane(cms.srgb_to_cmyk(rgb, intent))))
-    });
-    let plane_key = |plane: Ink| ImageKey { ink: plane, luts: (key.luts.0, generation), ..key.clone() };
-    let want = plane_key(ink);
+    let (luts, generation) = separation();
+    let keyed = |plane: Ink| plane_key(key, plane, generation);
+    let want = keyed(ink);
     let hit = {
         let mut s = images::store();
         if touch { s.get(&want) } else { s.peek(&want) }
@@ -357,8 +372,8 @@ pub(crate) fn ink_image(key: &ImageKey, pm: &Arc<Pixmap>, ink: Ink, blob: Option
     });
     let [cmy, k] = planes;
     let mut s = images::store();
-    let cmy = s.insert(plane_key(Ink::Cmy), cmy);
-    let k = s.insert(plane_key(Ink::K), k);
+    let cmy = s.insert(keyed(Ink::Cmy), cmy);
+    let k = s.insert(keyed(Ink::K), k);
     (want, if ink == Ink::K { k } else { cmy })
 }
 

crates/render/src/lib.rs

@@ -1912,12 +1912,21 @@ impl Renderer {
         // Outline mode draws the image's frame, and with Document Setup → Show Images in Outline
         // Mode its pixels in greyscale under the frame.
         let outline = f.opts.outline;
-        // With mips the full pixels are only looked at (not marked used) until they're drawn.
+        // With mips the full pixels are only looked at (not marked used) until they're drawn, and
+        // a mip the store holds is drawn without them: they may have been dropped (#2820).
         let mips = f.opts.image_mips;
-        let pixels = if outline && !f.doc.setup.outline_images { None } else { self.image_pixmap(f.doc, &im.key, outline, !mips) };
-        if let Some((key, pm)) = pixels {
-            let (key, pm) = if outline { (key, pm) } else { ink::ink_image(&key, &pm, f.ink, f.doc.images.get(&im.key), !mips) };
-            let pm = if mips { image_mip(&key, pm, f.view * im.xf * Affine::scale_non_uniform(im.width as f64, im.height as f64)) } else { pm };
+        let unit = f.view * im.xf * Affine::scale_non_uniform(im.width as f64, im.height as f64);
+        let shown = if outline && !f.doc.setup.outline_images {
+            None
+        } else if let Some(pm) = mips.then(|| self.kept_mip(f, &im.key, outline, unit)).flatten() {
+            Some(pm)
+        } else {
+            self.image_pixmap(f.doc, &im.key, outline, !mips).map(|(key, pm)| {
+                let (key, pm) = if outline { (key, pm) } else { ink::ink_image(&key, &pm, f.ink, f.doc.images.get(&im.key), !mips) };
+                if mips { image_mip(&key, pm, unit) } else { pm }
+            })
+        };
+        if let Some(pm) = shown {
             let sx = im.width as f64 / pm.width().max(1) as f64;
             let sy = im.height as f64 / pm.height().max(1) as f64;
             let quality = if f.opts.smooth_images { peniko::ImageQuality::Medium } else { peniko::ImageQuality::Low };
@@ -2021,6 +2030,36 @@ impl Renderer {
         let pm = images::store().insert(key.clone(), Arc::new(pm));
         Some((key, pm))
     }
+
+    /// The key [`Self::image_pixmap`] gives the pixels of image `key` and their size, without
+    /// making or looking at them: `None` until the image has been decoded.
+    fn image_key(&self, doc: &Document, key: &str, grey: bool) -> Option<(images::ImageKey, (u16, u16))> {
+        let (key, size) = match doc.images.get(key) {
+            Some(blob) => {
+                let (key, _, size) = blob_key(blob);
+                (key, size?)
+            }
+            None => {
+                let (src, hash, _) = self.adjusts.get(key)?;
+                let (src_key, size) = self.image_key(doc, src, false)?;
+                (images::ImageKey { adjust: Some(*hash), ..src_key }, size)
+            }
+        };
+        Some((if grey { images::ImageKey { grey: true, ..key } } else { key }, size))
+    }
+
+    /// The mip [`image_mip`] draws image `key` (in greyscale with `grey`, the outline view's) with
+    /// through `m` when the store holds it, marked used there; `None` when it draws the full pixels
+    /// or the mip is yet to be made.
+    fn kept_mip(&self, f: &Frame, key: &str, grey: bool, m: Affine) -> Option<Arc<Pixmap>> {
+        let (key, (w, h)) = self.image_key(f.doc, key, grey)?;
+        let key = if grey { key } else { ink::ink_key(&key, f.ink) };
+        let level = mip_level(m, w, h);
+        if level == 0 {
+            return None;
+        }
+        images::store().get(&images::ImageKey { mip: level, ..key })
+    }
 }
 
 /// `key`'s pixels in the shared image store; `touch` marks them used.
@@ -2029,31 +2068,49 @@ fn lookup(key: &images::ImageKey, touch: bool) -> Option<Arc<Pixmap>> {
     if touch { s.get(key) } else { s.peek(key) }
 }
 
-/// The decoded pixels of `blob` (a CMYK image's inks shown through the working CMYK profile, again
-/// when the colour settings change), from the shared store, and their key there.
-fn blob_pixmap(blob: &vectorcraft_doc::ImageBlob, touch: bool) -> Option<(images::ImageKey, Arc<Pixmap>)> {
+/// The key of `blob`'s decoded pixels in the shared store, the display table a CMYK image's inks
+/// show through (its generation is in the key), and the pixels' size once decoded.
+fn blob_key(blob: &vectorcraft_doc::ImageBlob) -> (images::ImageKey, Option<Arc<ink::InkLut>>, Option<(u16, u16)>) {
     let id = images::blob_id(blob);
     let lut = id.cmyk.then(ink::display_lut);
     let key =
         images::ImageKey { content: id.content, adjust: None, grey: false, ink: Ink::Display, luts: (lut.as_ref().map_or(0, |l| l.1), 0), mip: 0 };
-    if let Some(pm) = lookup(&key, touch) {
-        return Some((key, pm));
-    }
-    // Decoded outside the lock: other renderers carry on meanwhile.
-    let pm = match lut.and_then(|(lut, _)| ink::cmyk_pixmap(blob, &lut)) {
+    (key, lut.map(|l| l.0), id.size)
+}
+
+/// The decoded pixels of `blob` (a CMYK image's inks shown through the working CMYK profile, again
+/// when the colour settings change), from the shared store, and their key there.
+fn blob_pixmap(blob: &vectorcraft_doc::ImageBlob, touch: bool) -> Option<(images::ImageKey, Arc<Pixmap>)> {
+    let (key, lut, size) = blob_key(blob);
+    let pm = match lookup(&key, touch) {
         Some(pm) => pm,
-        None => paint::decode_pixmap(&blob.bytes)?,
+        None => {
+            // Decoded outside the lock: other renderers carry on meanwhile.
+            let pm = match lut.and_then(|lut| ink::cmyk_pixmap(blob, &lut)) {
+                Some(pm) => pm,
+                None => paint::decode_pixmap(&blob.bytes)?,
+            };
+            images::store().insert(key.clone(), Arc::new(pm))
+        }
     };
-    let pm = images::store().insert(key.clone(), Arc::new(pm));
+    let decoded = (pm.width(), pm.height());
+    if size != Some(decoded) {
+        images::remember_size(blob, decoded);
+    }
     Some((key, pm))
 }
 
+/// How many times to halve image pixels of `w`×`h` drawn through `m` (unit square → device
+/// pixels): [`images::mip_level`].
+fn mip_level(m: Affine, w: u16, h: u16) -> u8 {
+    images::mip_level(m * Affine::scale_non_uniform(1.0 / w.max(1) as f64, 1.0 / h.max(1) as f64), w, h)
+}
+
 /// Image pixels `pm` (stored as `key`) to draw through `m` (unit square → device pixels), shrunk
 /// by halves while still at least as fine as the device pixels ([`images::mip_level`]); the
 /// pixels drawn are marked used in the store.
 fn image_mip(key: &images::ImageKey, pm: Arc<Pixmap>, m: Affine) -> Arc<Pixmap> {
-    let (w, h) = (pm.width(), pm.height());
-    let level = images::mip_level(m * Affine::scale_non_uniform(1.0 / w.max(1) as f64, 1.0 / h.max(1) as f64), w, h);
+    let level = mip_level(m, pm.width(), pm.height());
     let mut s = images::store();
     if level == 0 {
         let _ = s.get(key); // Marks the full pixels used (they were only looked at).

crates/render/src/tests_memory.rs

@@ -110,6 +110,29 @@ fn image_mips_are_opt_in_and_change_nothing_off() {
     assert!(Renderer::new().render(&d, 48, 48, near, &on).pixels == Renderer::new().render(&d, 48, 48, near, &off).pixels);
 }
 
+/// #2820: with mips on, an image drawn at a quarter of its size draws its kept mip in later frames
+/// without its full pixels. Those go first when the store is over budget (only the mip is marked
+/// used), and were decoded again in every frame: dragging one of 20 large photos took seconds a
+/// frame.
+#[test]
+fn kept_mips_draw_without_decoding_the_full_pixels_again() {
+    let _fonts = fonts_steady();
+    let d = doc_with_image("img", png(64, 64, 104), 64, 64);
+    let on = RenderOptions { image_mips: true, ..Default::default() };
+    let quarter = Affine::scale(0.25);
+    let mut r = Renderer::new();
+    let _ = r.render(&d, 32, 16, quarter, &on);
+    let full = r.image_pixmap(&d, "img", false, false).unwrap().0;
+    images::store().remove(&full);
+    // The image dragged 4 pt (a device pixel) right.
+    let mut moved = d.clone();
+    let id = moved.layers[0].children().unwrap()[0].id;
+    moved.node_mut(id).unwrap().transform(Affine::translate((4.0, 0.0)), false);
+    let frame = r.render(&moved, 32, 16, quarter, &on);
+    assert!(images::store().peek(&full).is_none(), "the full pixels were decoded again");
+    assert!(frame.pixels == Renderer::new().render(&moved, 32, 16, quarter, &on).pixels);
+}
+
 /// A document of `n` point type objects reading `word`, in rows.
 fn text_doc(n: usize, word: &str) -> Document {
     use vectorcraft_doc::text::{CharStyle, TextObject};