perf: skip redundant smart object resamples and per-pixel reads (#6406)

* skip redundant smart object resamples and per-pixel reads

* resample smart objects only when their re-render fails

---------

Co-authored-by: Brandon Thomas <echelon@gmail.com>

crates/engine/src/image_cmds.rs

@@ -1,6 +1,8 @@
 //! Image menu: Image Size, Canvas Size, Crop, Trim, Mode (colour model and
 //! bit depth) conversions, Duplicate.
 
+use std::convert::Infallible;
+
 use photocraft_algo::resample::{Resample, crop_surface, resize_surface_in_canvas, translate_surface};
 use photocraft_color::{ColorMode, SampleType};
 use photocraft_doc::{Document, Effect, Effects, FxPaint, Layer, LayerContent, LayerId, Size};
@@ -297,6 +299,7 @@ pub(crate) fn resample_doc(doc: &mut Document, nw: u32, nh: u32, filter: Resampl
     }
     // Content that reaches the canvas edge keeps it: a Background stays opaque to the border.
     let canvas = Rect::from_xywh(0, 0, ow, oh);
+    let taken = crate::smart_cmds::take_caches(&mut doc.layers);
     for_each_surface(&mut doc.layers, true, &mut |surf, is_mask| {
         *surf = resize_surface_in_canvas(surf, sx, sy, if is_mask { Resample::Bilinear } else { filter }, canvas);
     });
@@ -321,6 +324,7 @@ pub(crate) fn resample_doc(doc: &mut Document, nw: u32, nh: u32, filter: Resampl
     crate::canvas_geom::transform_geometry(doc, &photocraft_geom::Affine { m: [sx, 0.0, 0.0, sy, 0.0, 0.0] });
     doc.size = Size::new(nw, nh);
     crate::canvas_geom::refresh(doc, crate::canvas_geom::Refresh::All);
+    let Ok(()) = crate::smart_cmds::restore_unrendered(doc, taken, |c| Ok::<_, Infallible>(resize_surface_in_canvas(c, sx, sy, filter, canvas)));
 }
 
 fn bad(cmd: &str, msg: impl Into<String>) -> EngineError {

crates/engine/src/smart_cmds.rs

@@ -186,6 +186,33 @@ pub fn source_decode_count(bytes: &[u8], fmt: PixelFormat) -> u32 {
     SOURCE_CACHE.lock().ok().and_then(|c| c.iter().find(|e| e.key == key).map(|e| e.decodes)).unwrap_or(0)
 }
 
+pub(crate) fn take_caches(layers: &mut [Layer]) -> Vec<(LayerId, Surface)> {
+    let mut out = Vec::new();
+    for l in layers {
+        match &mut l.content {
+            LayerContent::Smart(sm) => out.extend(sm.cache.take().map(|c| (l.id, c))),
+            LayerContent::Group(g) => out.extend(take_caches(&mut g.children)),
+            _ => {}
+        }
+    }
+    out
+}
+
+pub(crate) fn restore_unrendered<E>(
+    doc: &mut Document,
+    taken: Vec<(LayerId, Surface)>,
+    mut fallback: impl FnMut(&Surface) -> std::result::Result<Surface, E>,
+) -> std::result::Result<(), E> {
+    for (id, cache) in taken {
+        if let Some(LayerContent::Smart(sm)) = doc.layer_mut(id).map(|l| &mut l.content)
+            && sm.cache.is_none()
+        {
+            sm.cache = Some(fallback(&cache)?);
+        }
+    }
+    Ok(())
+}
+
 fn buffer_to_surface(buf: &photocraft_compose::Buffer, fmt: PixelFormat) -> Surface {
     let n = fmt.channels();
     let mut data = vec![0.0f32; buf.px.len() * n];

crates/engine/src/transform_cmds.rs

@@ -838,13 +838,17 @@ fn transform_doc(
     // Photoshop turns the Background into a normal layer before transforming it.
     crate::extra_cmds::background_to_layer(doc, id);
     let l = doc.layer_mut(id).ok_or(EngineError::NoLayer(id))?;
-    transform_layer_with(if is_group { None } else { sel.as_ref() }, group, l, h, affine, interp, ctl)?;
+    let layer_sel = if is_group { None } else { sel.as_ref() };
+    check_layer_budget(layer_sel, l, h)?;
+    let taken = if h.is_identity() { Vec::new() } else { crate::smart_cmds::take_caches(std::slice::from_mut(l)) };
+    apply_transform(layer_sel, group, l, h, affine, interp, ctl)?;
     if !h.is_identity() {
         // Type layers re-render from their new transform.
         let snapshot = doc.clone();
         if let Some(l) = doc.layer_mut(id) {
             refresh_text(&snapshot, l);
         }
+        crate::smart_cmds::restore_unrendered(doc, taken, |c| warp_surface(c, c.content_bounds(), h, interp))?;
         // The selection outline moves with the pixels.
         if let Some(sel) = &doc.selection {
             doc.selection = Some(warp_gray(sel, h, Interp::Bilinear)?).filter(|s| !s.content_bounds().is_empty());

crates/engine/tests/smart_rerender_fallback.rs

@@ -0,0 +1,77 @@
+use std::sync::Arc;
+
+use photocraft_doc::{LayerContent, SmartObject, SmartSource};
+use photocraft_engine::{EngineError, Session};
+use serde_json::{Value, json};
+
+fn deny_blur(id: &str, _: &Value) -> photocraft_engine::Result<()> {
+    if id == "filter.blur.gaussianBlur" { Err(EngineError::Other("denied".into())) } else { Ok(()) }
+}
+
+fn edits() -> [(&'static str, Value); 2] {
+    [("edit.transform", json!({"scaleX": 50, "scaleY": 100})), ("image.imageSize", json!({"width": 100, "height": 50}))]
+}
+
+fn blurred_smart_object() -> Session {
+    let mut s = Session::new();
+    s.execute("file.new", json!({"width": 200, "height": 100})).unwrap();
+    s.execute("layer.new.layer", json!({})).unwrap();
+    s.execute("select.rect", json!({"x": 20, "y": 20, "width": 100, "height": 50})).unwrap();
+    s.execute("edit.fill", json!({"contents": "color", "color": "#ff0000"})).unwrap();
+    s.execute("select.deselect", json!({})).unwrap();
+    s.execute("layer.smartObjects.convertToSmartObject", json!({})).unwrap();
+    s.execute("filter.blur.gaussianBlur", json!({"radius": 2})).unwrap();
+    s
+}
+
+fn cache_width(s: &Session) -> i32 {
+    let st = s.active().unwrap();
+    let LayerContent::Smart(sm) = &st.doc.layer(st.active_layer.unwrap()).unwrap().content else { panic!("not a smart object") };
+    sm.cache.as_ref().unwrap().content_bounds().width() as i32
+}
+
+fn edit_smart_object(s: &mut Session, edit: impl FnOnce(&mut SmartObject)) {
+    let st = s.active_mut().unwrap();
+    let id = st.active_layer.unwrap();
+    let LayerContent::Smart(sm) = &mut Arc::make_mut(&mut st.doc).layer_mut(id).unwrap().content else { panic!("not a smart object") };
+    edit(sm);
+}
+
+#[test]
+fn a_smart_object_whose_filters_are_denied_keeps_the_resampled_fallback() {
+    for (cmd, params) in edits() {
+        let mut s = blurred_smart_object();
+        let before = cache_width(&s);
+        s.authorize = Some(deny_blur);
+        s.execute(cmd, params).unwrap();
+        let after = cache_width(&s);
+        assert!(after < before * 3 / 4, "{cmd}: the fallback stayed at {after} px wide (was {before})");
+    }
+}
+
+#[test]
+fn a_smart_object_that_fails_to_re_render_keeps_the_resampled_fallback() {
+    let unrenderable =
+        [SmartSource::Linked { path: "missing.psd".into() }, SmartSource::Embedded { file_name: "broken.png".into(), bytes: Arc::new(vec![0; 16]) }];
+    for (cmd, params) in edits() {
+        for source in &unrenderable {
+            let mut s = blurred_smart_object();
+            let before = cache_width(&s);
+            edit_smart_object(&mut s, |sm| sm.source = source.clone());
+            s.execute(cmd, params.clone()).unwrap();
+            let after = cache_width(&s);
+            assert!(after < before * 3 / 4, "{cmd} with {source:?}: the fallback stayed at {after} px wide (was {before})");
+        }
+    }
+}
+
+#[test]
+fn an_allowed_smart_object_still_re_renders_at_the_new_size() {
+    for (cmd, params) in edits() {
+        let mut s = blurred_smart_object();
+        let before = cache_width(&s);
+        s.execute(cmd, params).unwrap();
+        let after = cache_width(&s);
+        assert!(after < before * 3 / 4, "{cmd}: {after} px wide (was {before})");
+    }
+}

crates/paint/src/replace.rs

@@ -127,6 +127,8 @@ pub fn color_replacement_effect(pre: Surface, rs: ReplaceSettings) -> impl FnMut
             return;
         }
         let (w, h) = (r.width() as usize, r.height() as usize);
+        let mut pre_px = vec![[0.0f32; 4]; w * h];
+        pre.read_rgba_into(r, &mut pre_px);
         // Per-pixel match weight.
         let mut weight = vec![0.0f32; w * h];
         for y in r.y0..r.y1 {
@@ -136,7 +138,7 @@ pub fn color_replacement_effect(pre: Surface, rs: ReplaceSettings) -> impl FnMut
                 }
                 // Match against the pre-stroke colours, so pixels recoloured by earlier dabs of
                 // this stroke still count as the sampled colour.
-                let c = pre.rgba(x, y);
+                let c = pre_px[(y - r.y0) as usize * w + (x - r.x0) as usize];
                 if c[3] <= 0.0 {
                     continue;
                 }

crates/raster/src/lib.rs

@@ -1116,7 +1116,8 @@ impl Surface {
 
     /// Composite-friendly RGBA read (converts from the surface's model).
     pub fn rgba(&self, x: i32, y: i32) -> [f32; 4] {
-        let p = self.pixel(x, y);
+        let mut p = [0.0f32; 5];
+        self.read_pixel(x, y, &mut p);
         to_rgba(&self.format, &p)
     }
 }