diff --git a/crates/render/src/images.rs b/crates/render/src/images.rs index 4c41f71ca..602711329 100644 --- a/crates/render/src/images.rs +++ b/crates/render/src/images.rs @@ -66,11 +66,13 @@ pub(crate) struct ImageKey { pub mip: u8, } -/// A blob's content key and whether it holds CMYK inks, worked out once per blob allocation. +/// A blob's content key and whether it holds CMYK inks, worked out once per blob allocation, and +/// the size of its pixels once they have been decoded ([`remember_size`]). #[derive(Clone)] pub(crate) struct BlobId { pub content: Arc, pub cmyk: bool, + pub size: Option<(u16, u16)>, } struct Entry { @@ -114,6 +116,14 @@ impl ImageStore { self.entries.get(key).map(|e| e.pm.clone()) } + /// Drop `key`'s pixels, as the budget drops the least recently used. + #[cfg(test)] + pub(crate) fn remove(&mut self, key: &ImageKey) { + if let Some(e) = self.entries.remove(key) { + self.bytes = self.bytes.saturating_sub(pixmap_bytes(&e.pm)); + } + } + /// Keep `pm` as `key` (or, when another renderer made it meanwhile, return theirs), then drop /// entries over budget. pub(crate) fn insert(&mut self, key: ImageKey, pm: Arc) -> Arc { @@ -167,6 +177,15 @@ impl ImageStore { } self.ids.insert(Arc::as_ptr(bytes) as usize, (Arc::downgrade(bytes), id)); } + + fn remember_size(&mut self, bytes: &Arc>, size: (u16, u16)) { + let at = Arc::as_ptr(bytes); + if let Some((w, id)) = self.ids.get_mut(&(at as usize)) + && std::ptr::eq(w.as_ptr(), at) + { + id.size = Some(size); + } + } } static STORE: LazyLock> = LazyLock::new(|| Mutex::new(ImageStore::new(default_budget()))); @@ -193,11 +212,17 @@ pub(crate) fn blob_id(blob: &ImageBlob) -> BlobId { return id; } // Outside the lock: hashing a large image takes a while. - let id = BlobId { content: format!("{}:{}", blob.content_key(), blob.bytes.len()).into(), cmyk: blob.is_cmyk() }; + let id = BlobId { content: format!("{}:{}", blob.content_key(), blob.bytes.len()).into(), cmyk: blob.is_cmyk(), size: None }; store().remember_id(&blob.bytes, id.clone()); id } +/// Note that `blob`'s pixels are `size`: [`blob_id`] tells it from then on, so that the mips made +/// of them are found while the store holds the mips but not the pixels. +pub(crate) fn remember_size(blob: &ImageBlob, size: (u16, u16)) { + store().remember_size(&blob.bytes, size); +} + /// `pm` at half the size (rounded up), each pixel the mean of the 2×2 it covers (the last row and /// column repeat on odd sizes). pub(crate) fn half(pm: &Pixmap) -> Pixmap { diff --git a/crates/render/src/ink.rs b/crates/render/src/ink.rs index 64e6f45db..ad4e8cb23 100644 --- a/crates/render/src/ink.rs +++ b/crates/render/src/ink.rs @@ -311,18 +311,33 @@ pub(crate) fn cmyk_pixmap(blob: &ImageBlob, lut: &InkLut) -> Option { inks_pixmap(&blob.cmyk()?, lut) } +/// The separation tables of placed images ([`Ink::Cmy`], [`Ink::K`]) and their generation. +fn separation() -> (Arc<[ProofLut; 2]>, u64) { + cached(&SEPARATE, |cms| { + let intent = cms.settings().intent; + [Ink::Cmy, Ink::K].map(|ink| ProofLut::build(|rgb| ink.plane(cms.srgb_to_cmyk(rgb, intent)))) + }) +} + +/// The key of image pixels `key` painted as `plane` with the separation tables of `generation`. +fn plane_key(key: &ImageKey, plane: Ink, generation: u64) -> ImageKey { + ImageKey { ink: plane, luts: (key.luts.0, generation), ..key.clone() } +} + +/// The key [`ink_image`] gives image pixels `key` painted with `ink`. +pub(crate) fn ink_key(key: &ImageKey, ink: Ink) -> ImageKey { + if matches!(ink, Ink::Display | Ink::RichBlack) { key.clone() } else { plane_key(key, ink, separation().1) } +} + /// Image pixels `pm` (cached as `key`, from `blob` when it is a document's image as it is) as /// painted with `ink`, cached with the images, and their key. pub(crate) fn ink_image(key: &ImageKey, pm: &Arc, ink: Ink, blob: Option<&ImageBlob>, touch: bool) -> (ImageKey, Arc) { if matches!(ink, Ink::Display | Ink::RichBlack) { return (key.clone(), pm.clone()); } - let (luts, generation) = cached(&SEPARATE, |cms| { - let intent = cms.settings().intent; - [Ink::Cmy, Ink::K].map(|ink| ProofLut::build(|rgb| ink.plane(cms.srgb_to_cmyk(rgb, intent)))) - }); - let plane_key = |plane: Ink| ImageKey { ink: plane, luts: (key.luts.0, generation), ..key.clone() }; - let want = plane_key(ink); + let (luts, generation) = separation(); + let keyed = |plane: Ink| plane_key(key, plane, generation); + let want = keyed(ink); let hit = { let mut s = images::store(); if touch { s.get(&want) } else { s.peek(&want) } @@ -357,8 +372,8 @@ pub(crate) fn ink_image(key: &ImageKey, pm: &Arc, ink: Ink, blob: Option }); let [cmy, k] = planes; let mut s = images::store(); - let cmy = s.insert(plane_key(Ink::Cmy), cmy); - let k = s.insert(plane_key(Ink::K), k); + let cmy = s.insert(keyed(Ink::Cmy), cmy); + let k = s.insert(keyed(Ink::K), k); (want, if ink == Ink::K { k } else { cmy }) } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 6855cc872..b9bd3bcc1 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -1912,12 +1912,21 @@ impl Renderer { // Outline mode draws the image's frame, and with Document Setup → Show Images in Outline // Mode its pixels in greyscale under the frame. let outline = f.opts.outline; - // With mips the full pixels are only looked at (not marked used) until they're drawn. + // With mips the full pixels are only looked at (not marked used) until they're drawn, and + // a mip the store holds is drawn without them: they may have been dropped (#2820). let mips = f.opts.image_mips; - let pixels = if outline && !f.doc.setup.outline_images { None } else { self.image_pixmap(f.doc, &im.key, outline, !mips) }; - if let Some((key, pm)) = pixels { - let (key, pm) = if outline { (key, pm) } else { ink::ink_image(&key, &pm, f.ink, f.doc.images.get(&im.key), !mips) }; - let pm = if mips { image_mip(&key, pm, f.view * im.xf * Affine::scale_non_uniform(im.width as f64, im.height as f64)) } else { pm }; + let unit = f.view * im.xf * Affine::scale_non_uniform(im.width as f64, im.height as f64); + let shown = if outline && !f.doc.setup.outline_images { + None + } else if let Some(pm) = mips.then(|| self.kept_mip(f, &im.key, outline, unit)).flatten() { + Some(pm) + } else { + self.image_pixmap(f.doc, &im.key, outline, !mips).map(|(key, pm)| { + let (key, pm) = if outline { (key, pm) } else { ink::ink_image(&key, &pm, f.ink, f.doc.images.get(&im.key), !mips) }; + if mips { image_mip(&key, pm, unit) } else { pm } + }) + }; + if let Some(pm) = shown { let sx = im.width as f64 / pm.width().max(1) as f64; let sy = im.height as f64 / pm.height().max(1) as f64; let quality = if f.opts.smooth_images { peniko::ImageQuality::Medium } else { peniko::ImageQuality::Low }; @@ -2021,6 +2030,36 @@ impl Renderer { let pm = images::store().insert(key.clone(), Arc::new(pm)); Some((key, pm)) } + + /// The key [`Self::image_pixmap`] gives the pixels of image `key` and their size, without + /// making or looking at them: `None` until the image has been decoded. + fn image_key(&self, doc: &Document, key: &str, grey: bool) -> Option<(images::ImageKey, (u16, u16))> { + let (key, size) = match doc.images.get(key) { + Some(blob) => { + let (key, _, size) = blob_key(blob); + (key, size?) + } + None => { + let (src, hash, _) = self.adjusts.get(key)?; + let (src_key, size) = self.image_key(doc, src, false)?; + (images::ImageKey { adjust: Some(*hash), ..src_key }, size) + } + }; + Some((if grey { images::ImageKey { grey: true, ..key } } else { key }, size)) + } + + /// The mip [`image_mip`] draws image `key` (in greyscale with `grey`, the outline view's) with + /// through `m` when the store holds it, marked used there; `None` when it draws the full pixels + /// or the mip is yet to be made. + fn kept_mip(&self, f: &Frame, key: &str, grey: bool, m: Affine) -> Option> { + let (key, (w, h)) = self.image_key(f.doc, key, grey)?; + let key = if grey { key } else { ink::ink_key(&key, f.ink) }; + let level = mip_level(m, w, h); + if level == 0 { + return None; + } + images::store().get(&images::ImageKey { mip: level, ..key }) + } } /// `key`'s pixels in the shared image store; `touch` marks them used. @@ -2029,31 +2068,49 @@ fn lookup(key: &images::ImageKey, touch: bool) -> Option> { if touch { s.get(key) } else { s.peek(key) } } -/// The decoded pixels of `blob` (a CMYK image's inks shown through the working CMYK profile, again -/// when the colour settings change), from the shared store, and their key there. -fn blob_pixmap(blob: &vectorcraft_doc::ImageBlob, touch: bool) -> Option<(images::ImageKey, Arc)> { +/// The key of `blob`'s decoded pixels in the shared store, the display table a CMYK image's inks +/// show through (its generation is in the key), and the pixels' size once decoded. +fn blob_key(blob: &vectorcraft_doc::ImageBlob) -> (images::ImageKey, Option>, Option<(u16, u16)>) { let id = images::blob_id(blob); let lut = id.cmyk.then(ink::display_lut); let key = images::ImageKey { content: id.content, adjust: None, grey: false, ink: Ink::Display, luts: (lut.as_ref().map_or(0, |l| l.1), 0), mip: 0 }; - if let Some(pm) = lookup(&key, touch) { - return Some((key, pm)); - } - // Decoded outside the lock: other renderers carry on meanwhile. - let pm = match lut.and_then(|(lut, _)| ink::cmyk_pixmap(blob, &lut)) { + (key, lut.map(|l| l.0), id.size) +} + +/// The decoded pixels of `blob` (a CMYK image's inks shown through the working CMYK profile, again +/// when the colour settings change), from the shared store, and their key there. +fn blob_pixmap(blob: &vectorcraft_doc::ImageBlob, touch: bool) -> Option<(images::ImageKey, Arc)> { + let (key, lut, size) = blob_key(blob); + let pm = match lookup(&key, touch) { Some(pm) => pm, - None => paint::decode_pixmap(&blob.bytes)?, + None => { + // Decoded outside the lock: other renderers carry on meanwhile. + let pm = match lut.and_then(|lut| ink::cmyk_pixmap(blob, &lut)) { + Some(pm) => pm, + None => paint::decode_pixmap(&blob.bytes)?, + }; + images::store().insert(key.clone(), Arc::new(pm)) + } }; - let pm = images::store().insert(key.clone(), Arc::new(pm)); + let decoded = (pm.width(), pm.height()); + if size != Some(decoded) { + images::remember_size(blob, decoded); + } Some((key, pm)) } +/// How many times to halve image pixels of `w`×`h` drawn through `m` (unit square → device +/// pixels): [`images::mip_level`]. +fn mip_level(m: Affine, w: u16, h: u16) -> u8 { + images::mip_level(m * Affine::scale_non_uniform(1.0 / w.max(1) as f64, 1.0 / h.max(1) as f64), w, h) +} + /// Image pixels `pm` (stored as `key`) to draw through `m` (unit square → device pixels), shrunk /// by halves while still at least as fine as the device pixels ([`images::mip_level`]); the /// pixels drawn are marked used in the store. fn image_mip(key: &images::ImageKey, pm: Arc, m: Affine) -> Arc { - let (w, h) = (pm.width(), pm.height()); - let level = images::mip_level(m * Affine::scale_non_uniform(1.0 / w.max(1) as f64, 1.0 / h.max(1) as f64), w, h); + let level = mip_level(m, pm.width(), pm.height()); let mut s = images::store(); if level == 0 { let _ = s.get(key); // Marks the full pixels used (they were only looked at). diff --git a/crates/render/src/tests_memory.rs b/crates/render/src/tests_memory.rs index 46c649f2e..edbc5e028 100644 --- a/crates/render/src/tests_memory.rs +++ b/crates/render/src/tests_memory.rs @@ -110,6 +110,29 @@ fn image_mips_are_opt_in_and_change_nothing_off() { assert!(Renderer::new().render(&d, 48, 48, near, &on).pixels == Renderer::new().render(&d, 48, 48, near, &off).pixels); } +/// #2820: with mips on, an image drawn at a quarter of its size draws its kept mip in later frames +/// without its full pixels. Those go first when the store is over budget (only the mip is marked +/// used), and were decoded again in every frame: dragging one of 20 large photos took seconds a +/// frame. +#[test] +fn kept_mips_draw_without_decoding_the_full_pixels_again() { + let _fonts = fonts_steady(); + let d = doc_with_image("img", png(64, 64, 104), 64, 64); + let on = RenderOptions { image_mips: true, ..Default::default() }; + let quarter = Affine::scale(0.25); + let mut r = Renderer::new(); + let _ = r.render(&d, 32, 16, quarter, &on); + let full = r.image_pixmap(&d, "img", false, false).unwrap().0; + images::store().remove(&full); + // The image dragged 4 pt (a device pixel) right. + let mut moved = d.clone(); + let id = moved.layers[0].children().unwrap()[0].id; + moved.node_mut(id).unwrap().transform(Affine::translate((4.0, 0.0)), false); + let frame = r.render(&moved, 32, 16, quarter, &on); + assert!(images::store().peek(&full).is_none(), "the full pixels were decoded again"); + assert!(frame.pixels == Renderer::new().render(&moved, 32, 16, quarter, &on).pixels); +} + /// A document of `n` point type objects reading `word`, in rows. fn text_doc(n: usize, word: &str) -> Document { use vectorcraft_doc::text::{CharStyle, TextObject};