Free Transform: a result may take what Memory Usage allows, not 2^30 px (#7807)

The fixed 2^30 px cap refused any transform of a stitched layer past about
1 GP, on any machine. A result is now priced in bytes at its pixel format
(with the alpha the warp adds) against Preferences > Performance > Memory
Usage (default 8192 MB), which Session::apply_prefs sets. The report's
37708 x 35259 px 8-bit result (5072 MB) now transforms: 10.7 s, 6.1 GB on
top of the source. Past the allowance the message names it:
"... (10144 MB), more than the 8192 MB Memory Usage allows (Preferences >
Performance)".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/prefs.rs

@@ -507,11 +507,14 @@ impl Default for Export {
     }
 }
 
+/// [`Performance::memory_usage_mb`]'s default.
+pub const DEFAULT_MEMORY_USAGE_MB: u32 = 8192;
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(default, rename_all = "camelCase")]
 pub struct Performance {
     /// Memory PhotoCraft may use, in MB: bounds each document's pixels plus its History (the
-    /// oldest states are dropped beyond it).
+    /// oldest states are dropped beyond it), and the result of one transform.
     pub memory_usage_mb: u32,
     /// Undo steps kept per document (History panel states).
     pub history_states: u32,
@@ -563,7 +566,7 @@ impl Performance {
 impl Default for Performance {
     fn default() -> Self {
         Self {
-            memory_usage_mb: 8192,
+            memory_usage_mb: DEFAULT_MEMORY_USAGE_MB,
             history_states: 50,
             cache_levels: 4,
             cache_tile_size: 8192,
@@ -1618,7 +1621,7 @@ impl Session {
     }
 
     /// Push engine-relevant preferences into live state: every document's history limit, the
-    /// layer-effect cache budget and the text engine's missing-glyph protection.
+    /// transform and layer-effect cache budgets and the text engine's missing-glyph protection.
     pub fn apply_prefs(&mut self) {
         let n = self.prefs().performance.history_states.max(1) as usize;
         let bytes = self.prefs().performance.history_budget_bytes();
@@ -1629,6 +1632,7 @@ impl Session {
             st.trim_history();
         }
         photocraft_compose::set_effect_cache_budget(budget << 20);
+        crate::transform_cmds::set_budget(u64::from(self.prefs().performance.memory_usage_mb) << 20);
         // Without waiting for the system font scan: this runs on the UI thread at launch.
         photocraft_text::set_shared_missing_glyph_protection(self.prefs().type_.missing_glyph_protection());
         crate::plugin_cmds::sync_prefs(self);

crates/engine/src/select_extra_cmds.rs

@@ -195,7 +195,7 @@ fn transform_selection(s: &mut Session, p: &Value) -> Result<Value> {
     }
     let selected = s.edit("Transform Selection", |doc, _| {
         if let Some(sel) = &doc.selection {
-            crate::transform_cmds::check_budget(sel.content_bounds(), &h, "selection")?;
+            crate::transform_cmds::check_budget(sel.content_bounds(), &h, "selection", sel.format())?;
             doc.selection = Some(crate::transform_cmds::warp_gray(sel, &h, interp)?).filter(|s| !s.content_bounds().is_empty());
         }
         Ok(doc.selection.is_some())

crates/engine/src/transform_cmds.rs

@@ -6,6 +6,7 @@ use photocraft_doc::{Document, Layer, LayerContent, LayerId, Locks};
 use photocraft_geom::{Affine, Rect};
 use photocraft_raster::{Interrupt, Surface};
 use serde_json::{Value, json};
+use std::sync::atomic::{AtomicU64, Ordering};
 
 use crate::commands::CommandSpec;
 use crate::{EngineError, Result, Session};
@@ -257,32 +258,57 @@ pub fn locked_for_transform(doc: &Document, id: LayerId) -> bool {
     false
 }
 
-/// The largest result (px) a single Free Transform warp may build: 2³⁰, the bound Puppet Warp uses
-/// too ([`photocraft_algo::puppet::MAX_WARP_PIXELS`]). Only the size of the result counts, not
-/// where it lands, so moving a layer far off the canvas stays allowed. A scale past it is refused
-/// rather than clipped (#3773): the warp would otherwise fill every tile of a result up to 2²¹ px
-/// on a side before anything could fail.
-pub const MAX_TRANSFORM_PIXELS: u64 = 1 << 30;
+/// The memory (bytes) a single Free Transform warp's result may take: Preferences › Performance ›
+/// Memory Usage, which [`Session::apply_prefs`] sets, rather than a fixed 2³⁰ px, so a stitched
+/// layer of a few gigapixels transforms on a machine that has the memory for it (#7807). Only the
+/// size of the result counts, not where it lands, so moving a layer far off the canvas stays
+/// allowed. A scale past it is refused rather than clipped (#3773): the warp would otherwise fill
+/// every tile of a result up to 2²¹ px on a side before anything could fail.
+static BUDGET: AtomicU64 = AtomicU64::new((crate::prefs::DEFAULT_MEMORY_USAGE_MB as u64) << 20);
 
-/// Tests use a far lower budget, so proving a refusal never comes near allocating the real one.
-#[cfg(not(test))]
-const BUDGET: u64 = MAX_TRANSFORM_PIXELS;
-#[cfg(test)]
-const BUDGET: u64 = 1 << 20;
-
-/// Refuse a warp of the content inside `src` by `h` whose result would exceed the pixel budget,
-/// before anything is built. `what` names the surface in the message, which names the limit
-/// rather than memory: nothing has been allocated, so no memory ran short (#7807).
-pub(crate) fn check_budget(src: Rect, h: &Homography, what: &str) -> Result<()> {
-    check_budget_with_limit(src, h, what, BUDGET)
+/// Set the memory (bytes) one transform's result may take (see [`budget`]).
+pub fn set_budget(bytes: u64) {
+    BUDGET.store(bytes, Ordering::Relaxed);
 }
 
-/// [`check_budget`] against `budget` px (the tests price the production limit with it).
-fn check_budget_with_limit(src: Rect, h: &Homography, what: &str, budget: u64) -> Result<()> {
+/// The memory (bytes) one transform's result may take: Memory Usage.
+pub fn budget() -> u64 {
+    BUDGET.load(Ordering::Relaxed)
+}
+
+/// What [`check_budget`] holds a result to: [`budget`].
+#[cfg(not(test))]
+fn active_budget() -> u64 {
+    budget()
+}
+
+/// Tests use a far lower budget (2²⁰ px of 8-bit RGBA), so proving a refusal never comes near
+/// allocating the real one.
+#[cfg(test)]
+fn active_budget() -> u64 {
+    4 << 20
+}
+
+/// Refuse a warp of the content inside `src` (pixels of `format`) by `h` whose result would take
+/// more memory than the budget, before anything is built. `what` names the surface in the message,
+/// which names Memory Usage rather than a shortage: nothing has been allocated yet.
+pub(crate) fn check_budget(src: Rect, h: &Homography, what: &str, format: PixelFormat) -> Result<()> {
+    check_budget_with_limit(src, h, what, format, active_budget())
+}
+
+/// [`check_budget`] against `budget` bytes (the tests price the production budget with it).
+fn check_budget_with_limit(src: Rect, h: &Homography, what: &str, format: PixelFormat, budget: u64) -> Result<()> {
     let Some((w, ht)) = warp_output_size(src, h) else { return Ok(()) };
-    if w.saturating_mul(ht) > budget {
+    // The warp writes each tile into the result as it goes, so the result is what it holds; it
+    // adds alpha to a format without it.
+    let bpp = PixelFormat::new(format.mode, format.sample, true).bytes_per_pixel() as u64;
+    let bytes = w.saturating_mul(ht).saturating_mul(bpp);
+    if bytes > budget {
+        let mb = |b: u64| b.div_ceil(1 << 20);
         return Err(EngineError::Other(format!(
-            "too large to transform: the transformed {what} would be {w} x {ht} px, more than the {budget} px limit; the document was not changed"
+            "too large to transform: the transformed {what} would be {w} x {ht} px ({} MB), more than the {} MB Memory Usage allows (Preferences › Performance); the document was not changed",
+            mb(bytes),
+            budget >> 20
         )));
     }
     Ok(())
@@ -303,10 +329,10 @@ fn check_layer_budget(doc_sel: Option<&Surface>, l: &Layer, h: &Homography) -> R
             LayerContent::Text(_) | LayerContent::Shape(_) => {}
             LayerContent::Smart(sm) => {
                 if let Some(c) = &sm.cache {
-                    check_budget(c.content_bounds(), h, "smart object")?;
+                    check_budget(c.content_bounds(), h, "smart object", c.format())?;
                 }
                 if let Some(m) = &sm.filter_mask {
-                    check_budget(m.surface.content_bounds(), h, "smart filter mask")?;
+                    check_budget(m.surface.content_bounds(), h, "smart filter mask", m.surface.format())?;
                 }
             }
             _ => {
@@ -316,7 +342,7 @@ fn check_layer_budget(doc_sel: Option<&Surface>, l: &Layer, h: &Homography) -> R
                         Some(sel) => surf.content_bounds().intersect(&sel.content_bounds()),
                         None => surf.content_bounds(),
                     };
-                    check_budget(src, h, &format!("layer \"{}\"", l.name))?;
+                    check_budget(src, h, &format!("layer \"{}\"", l.name), surf.format())?;
                 }
             }
         }
@@ -328,7 +354,7 @@ fn check_layer_budget(doc_sel: Option<&Surface>, l: &Layer, h: &Homography) -> R
                 Some(sel) => sel.content_bounds(),
                 None => m.surface.content_bounds(),
             };
-            check_budget(src, h, "layer mask")?;
+            check_budget(src, h, "layer mask", m.surface.format())?;
         }
     }
     Ok(())
@@ -336,7 +362,7 @@ fn check_layer_budget(doc_sel: Option<&Surface>, l: &Layer, h: &Homography) -> R
 
 /// `group` holds the locks `l` inherits from the groups around it ([`group_locks`]). A Background
 /// is turned into a normal layer by the caller first ([`crate::extra_cmds::background_to_layer`]).
-/// Refused, with `l` untouched, when a result would exceed [`MAX_TRANSFORM_PIXELS`].
+/// Refused, with `l` untouched, when a result would take more than [`budget`].
 pub(crate) fn transform_layer(doc_sel: Option<&Surface>, group: Locks, l: &mut Layer, h: &Homography, affine: Option<Affine>, interp: Interp) -> Result<()> {
     transform_layer_with(doc_sel, group, l, h, affine, interp, &Interrupt::NONE)
 }
@@ -664,14 +690,14 @@ fn transform_doc(
     let sel = doc.selection.clone();
     // The selection outline moves too: price it before anything is built.
     if let Some(sel) = sel.as_ref().filter(|_| !h.is_identity()) {
-        check_budget(sel.content_bounds(), h, "selection")?;
+        check_budget(sel.content_bounds(), h, "selection", sel.format())?;
     }
     if lone {
         // A targeted unlinked mask, alpha channel or Quick Mask transforms by itself.
         if !h.is_identity() {
             let (surf, _) = crate::channel_cmds::target_surface(doc, id, p)?;
             let src = sel.as_ref().map_or_else(|| surf.content_bounds(), Surface::content_bounds);
-            check_budget(src, h, "mask or channel")?;
+            check_budget(src, h, "mask or channel", surf.format())?;
             *surf = match &sel {
                 Some(sel) => warp_gray_selected(surf, sel, h, interp)?,
                 None => warp_gray(surf, h, interp)?,
@@ -742,7 +768,11 @@ mod tests {
         st.doc.layer(st.active_layer.unwrap()).unwrap().surface().unwrap().content_bounds()
     }
 
-    // ---- Pixel budget (#3773). Tests run with a 2²⁰ px budget, so nothing here comes near the real one. ----
+    // ---- Memory budget (#3773, #7807). Tests run with a 4 MB budget (2²⁰ px of 8-bit RGBA), so
+    // nothing here comes near the real one. ----
+
+    /// Memory Usage's default, in bytes.
+    const DEFAULT_BUDGET: u64 = (crate::prefs::DEFAULT_MEMORY_USAGE_MB as u64) << 20;
 
     #[test]
     fn huge_transform_scales_exceed_the_production_budget_before_clipping() {
@@ -750,7 +780,7 @@ mod tests {
         for scale in [1e7, 1e28, 1e306] {
             for (sx, sy) in [(scale, 1.0), (1.0, scale), (-scale, 1.0)] {
                 let h = Homography([sx, 0.0, 0.0, 0.0, sy, 0.0, 0.0, 0.0, 1.0]);
-                assert!(check_budget_with_limit(src, &h, "layer", MAX_TRANSFORM_PIXELS).is_err(), "{sx} x {sy} must be refused");
+                assert!(check_budget_with_limit(src, &h, "layer", PixelFormat::RGBA8, DEFAULT_BUDGET).is_err(), "{sx} x {sy} must be refused");
             }
         }
     }
@@ -759,7 +789,25 @@ mod tests {
     fn an_ordinary_transform_remains_within_the_production_budget() {
         let src = Rect::new(0, 0, 24, 16);
         let h = Homography([10_000.0, 0.0, 900_000.0, 0.0, 1.0, -900_000.0, 0.0, 0.0, 1.0]);
-        assert!(check_budget_with_limit(src, &h, "layer", MAX_TRANSFORM_PIXELS).is_ok());
+        assert!(check_budget_with_limit(src, &h, "layer", PixelFormat::RGBA8, DEFAULT_BUDGET).is_ok());
+    }
+
+    /// #7807: the stitched layer's 90 % and 15°, a 37708 x 35259 px result that the old 2³⁰ px limit
+    /// refused, fits the default Memory Usage at 8 bits (5072 MB); at 16 bits it needs Memory Usage
+    /// raised, and the message says so.
+    #[test]
+    fn a_gigapixel_layer_transforms_within_memory_usage() {
+        let src = Rect::new(0, 0, 35_020, 31_171);
+        let r = [0.0, 0.0, 35_020.0, 31_171.0];
+        let h = relative_matrix(&json!({"scaleX": 90, "scaleY": 90, "rotate": 15}), r).unwrap().unwrap();
+        assert_eq!(warp_output_size(src, &h), Some((37_708, 35_259)), "the report's result, past the old limit");
+        assert!(check_budget_with_limit(src, &h, "layer \"Layer 1\"", PixelFormat::RGBA8, DEFAULT_BUDGET).is_ok());
+        let err = check_budget_with_limit(src, &h, "layer \"Layer 1\"", PixelFormat::RGBA16, DEFAULT_BUDGET).unwrap_err().to_string();
+        assert!(err.contains("(10144 MB), more than the 8192 MB Memory Usage allows (Preferences › Performance)"), "{err}");
+        assert!(check_budget_with_limit(src, &h, "layer", PixelFormat::RGBA16, 2 * DEFAULT_BUDGET).is_ok());
+        // A format without alpha is priced as the warp makes it, with alpha.
+        let rgb8 = PixelFormat::new(PixelFormat::RGBA8.mode, PixelFormat::RGBA8.sample, false);
+        assert!(check_budget_with_limit(src, &h, "layer", rgb8, 5071 << 20).is_err());
     }
 
     /// A 16 x 16 document whose active layer is a 1 x 8 column at x = 10.
@@ -787,12 +835,15 @@ mod tests {
         let mut s = column();
         let st = s.active().unwrap();
         let (before, revision, steps) = (st.doc.clone(), st.revision, st.history.past_len());
-        // 200000 x 10 px of output: twice the test budget.
+        // 200000 x 10 px of 8-bit RGBA output: twice the test budget.
         let err = s.execute("edit.transform", stretch(2e5)).unwrap_err().to_string();
         assert!(err.starts_with("too large to transform: the transformed layer \"Layer 1\""), "{err}");
-        assert!(err.contains("would be 200002 x 10 px, more than the 1048576 px limit") && err.ends_with("the document was not changed"), "{err}");
-        // Nothing was allocated: the limit is named, not memory (#7807).
-        assert!(!err.contains("memory"), "{err}");
+        assert!(
+            err.contains("would be 200002 x 10 px (8 MB), more than the 4 MB Memory Usage allows") && err.ends_with("the document was not changed"),
+            "{err}"
+        );
+        // Nothing was allocated, so nothing ran short (#7807).
+        assert!(!err.contains("not enough memory"), "{err}");
         let st = s.active().unwrap();
         assert!(st.doc == before, "the document is untouched");
         assert_eq!((st.revision, st.history.past_len()), (revision, steps));
@@ -842,16 +893,18 @@ mod tests {
         assert_eq!(active_bounds(&s), Rect::new(10, 10, 30, 20), "and back, every pixel kept");
     }
 
+    /// Each priced at its own format, as the warp makes it: an 8-bit outline, mask or channel with
+    /// alpha is 2 bytes a pixel, so 400000 x 10 px of it is twice the test budget.
     #[test]
     fn selections_masks_and_channels_have_the_same_budget() {
         // The selection outline of a layer transform.
         let mut s = column();
         s.execute("select.rect", json!({"x": 10, "y": 2, "width": 1, "height": 8})).unwrap();
         let before = s.active().unwrap().doc.clone();
-        let err = s.execute("edit.transform", stretch(2e5)).unwrap_err().to_string();
+        let err = s.execute("edit.transform", stretch(4e5)).unwrap_err().to_string();
         assert!(err.contains("transformed selection would be"), "{err}");
         // Transform Selection.
-        let err = s.execute("select.transformSelection", stretch(2e5)).unwrap_err().to_string();
+        let err = s.execute("select.transformSelection", stretch(4e5)).unwrap_err().to_string();
         assert!(err.contains("transformed selection would be"), "{err}");
         assert!(s.active().unwrap().doc == before);
         // A targeted alpha channel.
@@ -863,7 +916,7 @@ mod tests {
         })
         .unwrap();
         let before = s.active().unwrap().doc.clone();
-        let mut p = stretch(2e5);
+        let mut p = stretch(4e5);
         p["target"] = json!({"channel": 0});
         let err = s.execute("edit.transform", p).unwrap_err().to_string();
         assert!(err.contains("transformed mask or channel would be"), "{err}");

crates/engine/tests/transform_budget_unclamped.rs

@@ -1,4 +1,5 @@
-//! #6430: integration tests use the production pixel budget, not the unit tests' reduced one.
+//! #6430: integration tests use the production budget (Memory Usage), not the unit tests' reduced
+//! one.
 use photocraft_engine::Session;
 use photocraft_geom::Rect;
 use serde_json::{Value, json};
@@ -20,7 +21,7 @@ fn assert_refused_without_changes(session: &mut Session, command: &str, params:
     let state = session.active().unwrap();
     let (document, revision, history) = (state.doc.clone(), state.revision, state.history.past_len());
     let error = session.execute(command, params).unwrap_err().to_string();
-    assert!(error.contains("limit"), "{error}");
+    assert!(error.starts_with("too large to transform"), "{error}");
     let state = session.active().unwrap();
     assert!(state.doc == document, "refusal changed the document");
     assert_eq!((state.revision, state.history.past_len()), (revision, history));
@@ -47,3 +48,24 @@ fn extreme_selection_scales_are_refused_without_losing_the_selection() {
         }
     }
 }
+
+/// #7807: what one transform's result may take is Memory Usage (Preferences › Performance), so
+/// raising it lets a refused transform through.
+#[test]
+fn memory_usage_is_what_a_transform_may_take() {
+    let mut session = painted();
+    // The 1 x 8 px column stretched 40000 times about its centre: 40003 x 10 px of 8-bit RGBA, 1.6 MB.
+    let stretch = json!({"scaleX": 4e6, "interpolation": "nearest"});
+    session.edit_prefs(|p| p.performance.memory_usage_mb = 1);
+    let state = session.active().unwrap();
+    let (document, history) = (state.doc.clone(), state.history.past_len());
+    let error = session.execute("edit.transform", stretch.clone()).unwrap_err().to_string();
+    assert!(error.contains("would be 40003 x 10 px (2 MB), more than the 1 MB Memory Usage allows (Preferences › Performance)"), "{error}");
+    let state = session.active().unwrap();
+    assert!(state.doc == document && state.history.past_len() == history, "refusal changed the document");
+    session.edit_prefs(|p| p.performance.memory_usage_mb = 2);
+    session.execute("edit.transform", stretch).unwrap();
+    let state = session.active().unwrap();
+    let bounds = state.doc.layer(state.active_layer.unwrap()).unwrap().surface().unwrap().content_bounds();
+    assert_eq!((bounds.width(), bounds.height()), (40_000, 8), "stretched, nothing clipped");
+}