Merge pull request #4640 from mockydevs/fix/ai-editing-data-by-memory-3510

Fix #3510: editing data past 512 MiB opens with its layers where memory allows

crates/eps/src/import/ai.rs

@@ -13,7 +13,7 @@
 //! groups, paths, colours, gradients, images) with newer ones (transparency, hidden objects) and
 //! dictionaries (names, artboards, type). `lex` scans it, `read` builds the document.
 //!
-//! Reading is bounded: the data decompresses to at most [`MAX_DATA`], and the reader caps nesting,
+//! Reading is bounded: the data decompresses to at most [`max_data`], and the reader caps nesting,
 //! objects and points. Anything it doesn't read is an error: the caller opens the file the way it
 //! did before (the printed page, or the PDF), with a warning saying why.
 
@@ -39,10 +39,27 @@ pub(crate) use read::{slot_frame, slot_of};
 #[cfg(test)]
 mod tests;
 
-/// Largest editing data read, decompressed (bytes). Files with embedded rasters reach 340 MB (a
-/// 30 MB poker box template unpacks to 322 MiB), so 64-bit builds read twice the old 256 MiB; a
-/// 32-bit address space (wasm) keeps it.
-pub const MAX_DATA: u64 = if cfg!(target_pointer_width = "64") { 512 << 20 } else { 256 << 20 };
+/// Least editing data read, decompressed (bytes), whatever the machine: 512 MiB on 64-bit builds (a
+/// 30 MB poker box template with embedded rasters unpacks to 322 MiB), 256 MiB in a 32-bit address
+/// space (wasm).
+pub const MIN_DATA: u64 = if cfg!(target_pointer_width = "64") { 512 << 20 } else { 256 << 20 };
+/// Most editing data read on any machine (bytes). Reading takes about as much memory again as the
+/// data (1.1 GB of data with 24 embedded 4096 x 4096 images peaked 1.1 GB above the start).
+pub const MAX_DATA: u64 = if cfg!(target_pointer_width = "64") { 2 << 30 } else { 256 << 20 };
+
+/// Largest editing data read, decompressed (bytes): an eighth of installed memory, from
+/// [`MIN_DATA`] to [`MAX_DATA`], so a file whose embedded rasters unpack past 512 MiB keeps its
+/// layers on a machine with room for it (#3510), while a small file that unpacks without end
+/// still stops early.
+pub fn max_data() -> u64 {
+    static LIMIT: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
+    *LIMIT.get_or_init(|| data_limit(vectorcraft_render::installed_memory()))
+}
+
+/// [`max_data`] on a machine with `memory` bytes installed (`None`: not known).
+fn data_limit(memory: Option<u64>) -> u64 {
+    memory.map_or(MIN_DATA, |m| (m / 8).clamp(MIN_DATA, MAX_DATA))
+}
 /// Largest Zstandard window accepted (a frame asking for more is damaged data).
 const MAX_ZSTD_WINDOW: u64 = 64 << 20;
 /// Most Zstandard frames read (the app writes one).
@@ -81,13 +98,13 @@ fn eps_section(section: &[u8]) -> Result<Vec<u8>, String> {
         }
         header.extend_from_slice(line);
         header.extend_from_slice(b"\n");
-        if header.len() as u64 > MAX_DATA {
+        if header.len() as u64 > max_data() {
             return Err(too_large());
         }
     }
     let Some(codec) = codec else {
         // Not compressed: the section is the data, as it is (images' samples are binary).
-        if section.len() as u64 > MAX_DATA {
+        if section.len() as u64 > max_data() {
             return Err(too_large());
         }
         return Ok(section.to_vec());
@@ -96,7 +113,7 @@ fn eps_section(section: &[u8]) -> Result<Vec<u8>, String> {
     for line in lines {
         let body = line.strip_prefix(b"%").unwrap_or(line);
         text.push_str(std::str::from_utf8(body).map_err(|_| damaged("its ASCII85 encoding"))?);
-        if text.len() as u64 > MAX_DATA {
+        if text.len() as u64 > max_data() {
             return Err(too_large());
         }
     }
@@ -105,21 +122,21 @@ fn eps_section(section: &[u8]) -> Result<Vec<u8>, String> {
     }
     let packed = ps::ascii85_decode(&text).ok_or_else(|| damaged("its ASCII85 encoding"))?;
     let mut data = header;
-    data.extend(codec.decode(&packed, MAX_DATA.saturating_sub(data.len() as u64))?);
+    data.extend(codec.decode(&packed, max_data().saturating_sub(data.len() as u64))?);
     Ok(data)
 }
 
 /// The editing data of a `.ai` file: its `AIPrivateData` streams joined (`raw`), decompressed. The
 /// compressed data may follow header comments that aren't.
 pub fn decode_private(raw: &[u8]) -> Result<Vec<u8>, String> {
-    if raw.len() as u64 > MAX_DATA {
+    if raw.len() as u64 > max_data() {
         return Err(too_large());
     }
     for (marker, codec) in [(&b"%AI24_ZStandard_Data"[..], Codec::Zstd), (b"%AI12_CompressedData", Codec::Zlib)] {
         if let Some(at) = find(raw, marker) {
             let (header, body) = raw.split_at(at);
             let mut data = header.to_vec();
-            data.extend(codec.decode(body.get(marker.len()..).unwrap_or_default(), MAX_DATA.saturating_sub(header.len() as u64))?);
+            data.extend(codec.decode(body.get(marker.len()..).unwrap_or_default(), max_data().saturating_sub(header.len() as u64))?);
             return Ok(data);
         }
     }
@@ -172,5 +189,5 @@ fn damaged(what: &str) -> String {
 }
 
 fn too_large() -> String {
-    format!("it decompresses to more than {} MB", MAX_DATA >> 20)
+    format!("it decompresses to more than {} MB", max_data() >> 20)
 }

crates/eps/src/import/ai/tests.rs

@@ -1266,7 +1266,51 @@ fn editing_data_past_256_mib_is_read() {
     assert!(raw.len() < 1 << 20, "zeros compress");
     let data = decode_private(&raw).unwrap();
     assert_eq!(data.len(), body.len());
-    assert!(too_large().contains("512 MB"));
+    assert_eq!(too_large(), format!("it decompresses to more than {} MB", max_data() >> 20));
+}
+
+/// The limit is an eighth of installed memory, never under the 512 MiB every 64-bit build read
+/// before and never over 2 GiB (#3510).
+#[cfg(target_pointer_width = "64")]
+#[test]
+fn the_editing_data_limit_follows_installed_memory() {
+    const GIB: u64 = 1 << 30;
+    assert_eq!(data_limit(None), 512 << 20, "unknown memory keeps the old limit");
+    assert_eq!(data_limit(Some(2 * GIB)), 512 << 20);
+    assert_eq!(data_limit(Some(4 * GIB)), 512 << 20);
+    assert_eq!(data_limit(Some(8 * GIB)), GIB);
+    assert_eq!(data_limit(Some(12 * GIB)), 3 * GIB / 2);
+    assert_eq!(data_limit(Some(16 * GIB)), 2 * GIB);
+    assert_eq!(data_limit(Some(256 * GIB)), 2 * GIB, "capped, so a tiny file can't take all of a big machine");
+    assert!((MIN_DATA..=MAX_DATA).contains(&max_data()));
+}
+
+/// On a machine with 8 GiB or more, editing data past the old 512 MiB limit opens with its layers
+/// (#3510, #4628): a layered file whose rasters unpack to 600 MiB.
+#[cfg(target_pointer_width = "64")]
+#[test]
+fn editing_data_past_512_mib_is_read_where_memory_allows() {
+    if max_data() < 601 << 20 {
+        return;
+    }
+    let mut body = stream(&format!("{}{}", layer(&rect_path(0.0, 0.0, 10.0, 10.0)), layer(&rect_path(20.0, 20.0, 30.0, 30.0)))).into_bytes();
+    let filler = vec![b'%'; (600 << 20) - body.len()];
+    let at = body.windows(14).position(|w| w == b"%%PageTrailer\n").unwrap();
+    body.splice(at..at, filler);
+    let raw = [&b"%AI12_CompressedData"[..], &crate::ps::deflate(&body)].concat();
+    let data = decode_private(&raw).unwrap();
+    assert_eq!(data.len(), body.len());
+    drop(body);
+    let r = read(&data).unwrap();
+    assert_eq!(r.doc.layers.len(), 2, "both layers are read");
+}
+
+/// Past the limit the data is still refused, so the file opens as its PDF part with a note.
+#[test]
+fn editing_data_past_the_limit_is_refused() {
+    let raw = [&b"%AI12_CompressedData"[..], &crate::ps::deflate(b"%%")].concat();
+    assert!(Codec::Zlib.decode(raw.get(20..).unwrap(), 1).is_err());
+    assert!(Codec::Zlib.decode(raw.get(20..).unwrap(), 2).is_ok());
 }
 
 #[test]

crates/render/src/images.rs

@@ -42,6 +42,15 @@ fn budget_for(memory: u64) -> usize {
     usize::try_from(memory / 8).unwrap_or(usize::MAX).min(IMAGE_CACHE_MAX)
 }
 
+/// Installed memory in bytes, `None` where it isn't known (the web, other systems). Other crates
+/// size their own budgets from it.
+pub fn installed_memory() -> Option<u64> {
+    #[cfg(target_arch = "wasm32")]
+    return None;
+    #[cfg(not(target_arch = "wasm32"))]
+    physical_memory()
+}
+
 /// Installed memory in bytes (`MemTotal` in `/proc/meminfo`).
 #[cfg(target_os = "linux")]
 fn physical_memory() -> Option<u64> {

crates/render/src/lib.rs

@@ -40,7 +40,7 @@ use lru::FrameCache;
 pub use brush_fx::instance_art;
 pub use changed::{Changed, changed, placed_area, view_pixels};
 pub use effects::stroke::width_outline;
-pub use images::{IMAGE_CACHE_MAX, image_cache_bytes, newly_undecodable, set_image_cache_budget};
+pub use images::{IMAGE_CACHE_MAX, image_cache_bytes, installed_memory, newly_undecodable, set_image_cache_budget};
 pub use live::expand_live;
 pub use pattern::render_pattern_swatch;
 pub use vectorcraft_effects as effects;

docs/file-format-parity.md

@@ -1,6 +1,6 @@
 # File format parity
 
-> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (Affinity import: attached Recolor Adjustment (RcRA) layers are imported as color overlays, #4420) · **Target:** Adobe Illustrator 2026 (30.x)
+> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-29 · **Change:** minor (`.ai` and Illustrator EPS import: the editing data is read up to an eighth of installed memory, 512 MiB to 2 GiB, #3510, #4628) · **Target:** Adobe Illustrator 2026 (30.x)
 
 Every format Illustrator opens, places, saves or exports, against VectorCraft's format table (`FORMATS`,
 `OPEN_EXTS`, `PLACE_EXTS` in `crates/engine/src/cmd/fileio/formats.rs`). Illustrator's list comes from its public
@@ -131,6 +131,7 @@ the app's own editing copy of a file (an EPS after its `%%EOF`, a `.ai`'s `AIPri
 
 | Date | Change | Summary |
 |---|---|---|
+| 2026-10-29 | minor | `.ai` and Illustrator EPS import: the editing data is read up to an eighth of installed memory, from 512 MiB to 2 GiB on 64-bit builds (it was 512 MiB on every machine), so a layered file whose embedded rasters unpack past 512 MiB keeps its layers, groups and text on a machine with 8 GiB or more instead of opening as its page or PDF part (#3510, #4628). Measured: 1.1 GB of editing data peaked 1.1 GB above the start; data past the limit is still refused (a 2 MB file unpacking to 2.2 GB stops in under a second). 32-bit and web builds keep 256 MiB. Own-fixture tests `import::ai::tests::editing_data_past_512_mib_is_read_where_memory_allows`, `the_editing_data_limit_follows_installed_memory`; scores and hours unchanged |
 | 2026-10-29 | minor | Affinity import: attached Recolor Adjustment (RcRA) layers import as color overlays with HSL lightness (#4420); scores unchanged. |
 | 2026-10-26 | minor | EPS Save and Save As: an opened EPS saves back to its own path as EPS, keeping the chosen artboard's page frame and transparent margins and the complete editable VectorCraft document; Save's TIFF preview is capped at 1024 px (Export keeps 2048); partly read, preview-only or damaged-editing-data sources still need confirmation before they are overwritten; Save As leaves out the export-only crop and selection options. Synthetic regressions in `tests::save` (engine and UI) and `dialogs::save_options::tests`, listed under [EPS Save and source protection](#eps-save-and-source-protection); it does not write Illustrator's own editing data (#868), and G9's real-file import fidelity is unchanged; scores and hours unchanged |
 | 2026-10-25 | minor | `.ai` import: the editing data is read up to 512 MiB on 64-bit builds (the limit was 256 MiB), so a file whose embedded rasters unpack to 300 MiB or more keeps its layers, groups and text instead of opening as its PDF part (part of #3510); 32-bit and web builds keep 256 MiB. Own-fixture test `import::ai::tests::editing_data_past_256_mib_is_read`; scores and hours unchanged |