PDF/EPS export: EPS export and PostScript printing expand pattern tile images once

EPS export expanded the pattern fills whose tile images need flattening and
then flattened the result with flatten_document, which expanded them again
with a new object limit. PostScript printing did the same, on the layers
shown rather than the layers that print. EPS export now flattens the
expanded document without expanding it again (flatten_expanded), and
PostScript printing leaves the expansion to the flattener, which runs on
the layers that print, and only checks for such pattern fills to warn.

crates/engine/src/cmd/fileio/eps.rs

@@ -12,7 +12,7 @@ use vectorcraft_doc::Document;
 use vectorcraft_eps::{EpsOptions, Level, Overprint, Preview, Raster};
 use vectorcraft_geom::{Point, Rect};
 
-use super::super::flatten::{FlattenOptions, expand_transparent_tile_images, flatten_document};
+use super::super::flatten::{FlattenOptions, expand_transparent_tile_images, flatten_expanded};
 use super::super::{CommandSpec, EngineError, Result, Session, bad, bool_or, cmd, has_doc, str_param};
 use super::{ArtboardPick, Encoded, FormatOption};
 
@@ -302,7 +302,7 @@ fn encode_document(doc: &Document, p: &Value, saving: bool) -> Result<Encoded> {
     let expanded = expand_transparent_tile_images(doc, &set.flatten)?;
     let source = expanded.as_ref().unwrap_or(doc);
     let transparent = source.layers.iter().any(|l| set.flatten.shows_transparency(source, l));
-    let flat = flatten_document(source, &set.flatten)?;
+    let flat = flatten_expanded(source, &set.flatten)?;
     let drawn = flat.as_ref().unwrap_or(source);
     let mut enc = Encoded { warnings, joiner: Some("_"), ..Encoded::default() };
     if transparent && (flat.is_some() || expanded.is_some()) {

crates/engine/src/cmd/flatten.rs

@@ -49,7 +49,7 @@ mod presets;
 mod report;
 
 pub(crate) use analysis::{outlined_text, see_through_images};
-pub(crate) use apply::{expand_transparent_tile_images, flatten_document};
+pub(crate) use apply::{expand_transparent_tile_images, flatten_document, flatten_expanded, has_transparent_tile_images};
 pub use commands::specs;
 pub use options::FlattenOptions;
 pub use presets::{FlattenerPreset, PRESET_EXTS, PRESET_FORMAT};

crates/engine/src/cmd/flatten/apply.rs

@@ -292,16 +292,21 @@ pub(super) fn opaque(n: &mut Node) {
     }
 }
 
+/// Whether `doc` has pattern fills whose image tiles need transparency flattening with `o`
+/// ([`expand_transparent_tile_images`]).
+pub(crate) fn has_transparent_tile_images(doc: &Document, o: &FlattenOptions) -> bool {
+    let keys = o.no_transparency.clone().unwrap_or_default();
+    let colorize = |d: &Document, n: &Node| o.counts_colorize(d, n);
+    !doc.patterns.is_empty() && crate::cmd::expand::has_transparent_image_patterns(doc, &keys, &colorize)
+}
+
 /// Expand pattern fills whose image tiles need transparency flattening.
 pub(crate) fn expand_transparent_tile_images(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
-    if doc.patterns.is_empty() {
+    if !has_transparent_tile_images(doc, o) {
         return Ok(None);
     }
     let keys = o.no_transparency.clone().unwrap_or_default();
     let colorize = |d: &Document, n: &Node| o.counts_colorize(d, n);
-    if !crate::cmd::expand::has_transparent_image_patterns(doc, &keys, &colorize) {
-        return Ok(None);
-    }
     let mut d = doc.clone();
     let layers: Vec<NodeId> =
         d.layers.iter().filter(|l| l.visible && !matches!(l.kind, NodeKind::Layer { template: true, .. })).map(|l| l.id).collect();
@@ -319,10 +324,15 @@ pub(crate) fn expand_transparent_tile_images(doc: &Document, o: &FlattenOptions)
 /// expanded when `o.no_transparency` counts them. `None` when nothing changes.
 pub(crate) fn flatten_document(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
     let expanded = expand_transparent_tile_images(doc, o)?;
-    let doc = expanded.as_ref().unwrap_or(doc);
+    Ok(flatten_expanded(expanded.as_ref().unwrap_or(doc), o)?.or(expanded))
+}
+
+/// [`flatten_document`] for `doc` whose pattern tile images were expanded already
+/// ([`expand_transparent_tile_images`]).
+pub(crate) fn flatten_expanded(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
     let outlining = o.text_to_outlines || o.strokes_to_outlines || !o.preserve_overprints;
     if !outlining && !doc.layers.iter().any(|l| o.shows_transparency(doc, l) || o.draws_see_through_image(l)) {
-        return Ok(expanded);
+        return Ok(None);
     }
     let mut roots = vec![];
     for l in doc.layers.iter().filter(|l| l.visible && !matches!(l.kind, NodeKind::Layer { template: true, .. })) {
@@ -331,7 +341,7 @@ pub(crate) fn flatten_document(doc: &Document, o: &FlattenOptions) -> Result<Opt
     }
     let plan = plan(doc, &roots, o, true)?;
     if plan.flat.is_empty() && !plan.kept.iter().any(|id| doc.node(*id).is_some_and(|n| kept_work(n, o))) {
-        return Ok(expanded);
+        return Ok(None);
     }
     let mut d = doc.clone();
     apply(&mut d, &mut vectorcraft_doc::Selection::default(), plan, o)?;

crates/engine/src/cmd/print.rs

@@ -11,7 +11,7 @@ use vectorcraft_pdf::{PrintLayers, PrintOptions, PrintPreview, PrintSettings};
 
 use super::fileio::pdf::{merge, pdf_error};
 use super::fileio::{extension, write_or_return};
-use super::flatten::expand_transparent_tile_images;
+use super::flatten::has_transparent_tile_images;
 use super::{CommandSpec, EngineError, Result, Session, bad, bool_or, cmd, has_doc, str_param};
 
 const SETUP: &str = "print.setup";
@@ -181,13 +181,11 @@ fn print(s: &mut Session, p: &Value) -> Result<Value> {
     let flat = super::rasterfx::flatten_raster_effects(doc);
     let doc = flat.as_ref().unwrap_or(doc);
     let bitmap = set.advanced.print_as_bitmap && set.output.mode == vectorcraft_pdf::OutputMode::Composite;
-    let expanded = if postscript && !bitmap {
-        let options = super::printadvanced::flattener(PRINT, &set, &s.prefs.flattener_presets)?;
-        options.as_ref().map(|o| expand_transparent_tile_images(doc, o)).transpose()?.flatten()
-    } else {
-        None
-    };
-    let doc = expanded.as_ref().unwrap_or(doc);
+    // The flattener expands pattern fills whose tile images need flattening, in the layers that print.
+    let tile_images = postscript
+        && !bitmap
+        && super::printadvanced::flattener(PRINT, &set, &s.prefs.flattener_presets)?
+            .is_some_and(|o| has_transparent_tile_images(&vectorcraft_pdf::printed_document(doc, &set), &o));
     // Print as Bitmap and the flattener preset (Advanced) render and flatten it first.
     let prepared = super::printadvanced::prepare(PRINT, doc, &set, &s.prefs.flattener_presets, s.prefs.black_output == "rich")?;
     let path = str_param(p, "path");
@@ -210,7 +208,7 @@ fn print(s: &mut Session, p: &Value) -> Result<Value> {
         None => Level::default(),
     };
     let mut warnings = vec![];
-    if !bitmap && doc.layers.iter().any(|l| l.shows_transparency()) {
+    if !bitmap && (tile_images || doc.layers.iter().any(|l| l.shows_transparency())) {
         warnings.push("transparency is flattened into opaque art and images (PostScript has none): see flattenerPreset".to_string());
     }
     if !set.color.profile.trim().is_empty() && set.output.mode == vectorcraft_pdf::OutputMode::Composite {

crates/engine/src/tests/legacy_ai.rs

@@ -820,6 +820,15 @@ fn flattening_that_keeps_colorize_leaves_colorized_pattern_tile_images_alone() {
     assert!(crate::cmd::flatten::flatten_document(d, &o).unwrap().is_none(), "the pattern isn't expanded for a colorize paint alone");
 }
 
+/// EPS export expands pattern tile images once: flattening the expanded
+/// document leaves the patterns the expansion kept, past its object limit, as they are.
+#[test]
+fn flattening_an_expanded_document_does_not_expand_patterns_again() {
+    let s = colorized_pattern_over_blue();
+    let d = &s.doc().unwrap().doc;
+    assert!(crate::cmd::flatten::flatten_expanded(d, &crate::cmd::FlattenOptions::default()).unwrap().is_none());
+}
+
 #[test]
 fn eps_flattens_colorized_image_in_a_see_through_pattern_tile_group_and_warns() {
     let mut s = colorized_pattern_over_blue();