PDF/EPS export: flatten see-through and colorized pattern tile images

PDF 1.3, PDF/X-1a, PDF/X-3, EPS and PostScript printing flatten
transparency with `flatten_document`, which only looked at art in the
layers. An image with see-through pixels, a colorize paint, opacity or
a blending mode drawn only through a pattern fill stayed in the pattern
definition, so the flattener returned early and the writer drew the tile
opaque and uncolorized (PDF 1.3 export failed on the transparency left).

`flatten_document` now first expands pattern fills whose tile image
needs flattening into their tiles, as legacy .ai save does. See-through
tile pixels count only where the caller counts see-through images (PDF
1.3); EPS level 3 already masks them out. The EPS and PostScript print
transparency warnings now see those tile images too.

crates/engine/src/cmd/expand.rs

@@ -456,18 +456,32 @@ pub(crate) fn expand_transparent_image_patterns(d: &mut Document, root: NodeId,
         if !l.appearance.items.iter().any(|i| pattern_fill(d, i).is_some()) {
             return None;
         }
-        let shape = d.ancestry(l.id).unwrap_or_default().iter().filter_map(|a| d.node(*a)).any(blended);
-        let transparent = |d: &Document, i: &AppearanceItem| {
-            pattern_fill(d, i).is_some_and(|(f, def, _)| {
-                let on = shape || f.opacity < 1.0 || f.blend != BlendMode::Normal;
-                def.art.iter().any(|a| transparent_image(a, see_through, on))
-            })
-        };
-        expand_patterns_where(d, l, transparent)
+        expand_patterns_where(d, l, |d, i| transparent_pattern_fill(d, l, i, see_through))
     })?;
     Ok(n)
 }
 
+/// Whether a path or compound path of `d` has a pattern fill that
+/// [`expand_transparent_image_patterns`] would expand with `see_through`.
+pub(crate) fn has_transparent_image_patterns(d: &Document, see_through: &BTreeSet<String>) -> bool {
+    let mut found = false;
+    d.walk(|n| {
+        if !found && matches!(n.kind, NodeKind::Path { .. } | NodeKind::Compound { .. }) {
+            found = n.appearance.items.iter().any(|i| transparent_pattern_fill(d, n, i, see_through));
+        }
+    });
+    found
+}
+
+/// Whether `i`, an item of leaf `l`, is a pattern fill whose tile art holds a transparent image
+/// ([`transparent_image`]), counting the fill's and the shape's opacity and blending mode.
+fn transparent_pattern_fill(d: &Document, l: &Node, i: &AppearanceItem, see_through: &BTreeSet<String>) -> bool {
+    let Some((f, def, _)) = pattern_fill(d, i) else { return false };
+    let shape = d.ancestry(l.id).unwrap_or_default().iter().filter_map(|a| d.node(*a)).any(blended);
+    let on = shape || f.opacity < 1.0 || f.blend != BlendMode::Normal;
+    def.art.iter().any(|a| transparent_image(a, see_through, on))
+}
+
 /// Whether `n` has opacity or a blending mode of its own.
 fn blended(n: &Node) -> bool {
     n.opacity < 1.0 || n.blend != BlendMode::Normal

crates/engine/src/cmd/fileio/eps.rs

@@ -12,7 +12,7 @@ use vectorcraft_doc::Document;
 use vectorcraft_eps::{EpsOptions, Level, Overprint, Preview, Raster};
 use vectorcraft_geom::{Point, Rect};
 
-use super::super::flatten::{FlattenOptions, flatten_document};
+use super::super::flatten::{FlattenOptions, expand_transparent_tile_images, flatten_document};
 use super::super::{CommandSpec, EngineError, Result, Session, bad, bool_or, cmd, has_doc, str_param};
 use super::{ArtboardPick, Encoded, FormatOption};
 
@@ -299,11 +299,13 @@ fn encode_document(doc: &Document, p: &Value, saving: bool) -> Result<Encoded> {
         return Err(bad(C, "the document has no artboard"));
     }
     let regions: Vec<Rect> = pages.iter().map(|(_, region, _)| *region).collect();
-    let transparent = doc.layers.iter().any(|l| l.shows_transparency());
-    let flat = flatten_document(doc, &set.flatten)?;
-    let drawn = flat.as_ref().unwrap_or(doc);
+    let expanded = expand_transparent_tile_images(doc, &set.flatten)?;
+    let source = expanded.as_ref().unwrap_or(doc);
+    let transparent = source.layers.iter().any(|l| l.shows_transparency());
+    let flat = flatten_document(source, &set.flatten)?;
+    let drawn = flat.as_ref().unwrap_or(source);
     let mut enc = Encoded { warnings, joiner: Some("_"), ..Encoded::default() };
-    if transparent && flat.is_some() {
+    if transparent && (flat.is_some() || expanded.is_some()) {
         enc.warnings.push("transparency is flattened into opaque art and images (EPS has none): see the flattener preset".into());
     }
     let preview_limit = if saving { SAVE_PREVIEW } else { MAX_PREVIEW };

crates/engine/src/cmd/fileio/pdf.rs

@@ -317,9 +317,9 @@ pub const DEFAULT_FLATTENER: &str = "High Resolution";
 
 /// `doc` with its transparency flattened for a PDF 1.3 file (asked for, or PDF/X-1a and PDF/X-3,
 /// whose files have none), with `flattenerPreset` (default High Resolution) and `p`'s `flattener`
-/// options over it; images with see-through pixels and colorized images count as transparency,
-/// rasterized areas are clipped to their regions, and discarded overprints are dropped. `None`
-/// when nothing changes (or the file keeps transparency).
+/// options over it; images with see-through pixels and colorized images, including those in
+/// pattern tiles, count as transparency. Rasterized areas are clipped to their regions, and
+/// discarded overprints are dropped. `None` when nothing changes (or the file keeps transparency).
 fn flatten_for(cmd: &str, doc: &Document, set: &PdfSettings, p: &Value) -> Result<Option<Document>> {
     if !set.pdf13() {
         return Ok(None);

crates/engine/src/cmd/flatten.rs

@@ -49,7 +49,7 @@ mod presets;
 mod report;
 
 pub(crate) use analysis::{outlined_text, see_through_images};
-pub(crate) use apply::flatten_document;
+pub(crate) use apply::{expand_transparent_tile_images, flatten_document};
 pub use commands::specs;
 pub use options::FlattenOptions;
 pub use presets::{FlattenerPreset, PRESET_EXTS, PRESET_FORMAT};

crates/engine/src/cmd/flatten/apply.rs

@@ -292,13 +292,35 @@ pub(super) fn opaque(n: &mut Node) {
     }
 }
 
+/// Expand pattern fills whose image tiles need transparency flattening.
+pub(crate) fn expand_transparent_tile_images(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
+    if doc.patterns.is_empty() {
+        return Ok(None);
+    }
+    let keys = o.no_transparency.clone().unwrap_or_default();
+    if !crate::cmd::expand::has_transparent_image_patterns(doc, &keys) {
+        return Ok(None);
+    }
+    let mut d = doc.clone();
+    let layers: Vec<NodeId> = d.layers.iter().map(|l| l.id).collect();
+    let mut changed = 0;
+    for id in layers {
+        changed += crate::cmd::expand::expand_transparent_image_patterns(&mut d, id, &keys)?;
+    }
+    Ok((changed > 0).then_some(d))
+}
+
 /// `doc` with everything it draws flattened with `o`, as formats without transparency (EPS)
 /// write it: every visible object of the visible layers, locked ones too (a clipping layer's
-/// clipping path stays). `None` when that changes nothing.
+/// clipping path stays). Pattern tile images with opacity, blending or colorize are expanded;
+/// see-through tile pixels are expanded when `o.no_transparency` counts them. `None` when nothing
+/// changes.
 pub(crate) fn flatten_document(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
+    let expanded = expand_transparent_tile_images(doc, o)?;
+    let doc = expanded.as_ref().unwrap_or(doc);
     let outlining = o.text_to_outlines || o.strokes_to_outlines || !o.preserve_overprints;
     if !outlining && !doc.layers.iter().any(|l| l.shows_transparency() || o.draws_see_through_image(l)) {
-        return Ok(None);
+        return Ok(expanded);
     }
     let mut roots = vec![];
     for l in doc.layers.iter().filter(|l| l.visible && !matches!(l.kind, NodeKind::Layer { template: true, .. })) {
@@ -307,7 +329,7 @@ pub(crate) fn flatten_document(doc: &Document, o: &FlattenOptions) -> Result<Opt
     }
     let plan = plan(doc, &roots, o, true)?;
     if plan.flat.is_empty() && !plan.kept.iter().any(|id| doc.node(*id).is_some_and(|n| kept_work(n, o))) {
-        return Ok(None);
+        return Ok(expanded);
     }
     let mut d = doc.clone();
     apply(&mut d, &mut vectorcraft_doc::Selection::default(), plan, o)?;

crates/engine/src/cmd/print.rs

@@ -11,6 +11,7 @@ use vectorcraft_pdf::{PrintLayers, PrintOptions, PrintPreview, PrintSettings};
 
 use super::fileio::pdf::{merge, pdf_error};
 use super::fileio::{extension, write_or_return};
+use super::flatten::expand_transparent_tile_images;
 use super::{CommandSpec, EngineError, Result, Session, bad, bool_or, cmd, has_doc, str_param};
 
 const SETUP: &str = "print.setup";
@@ -179,6 +180,14 @@ fn print(s: &mut Session, p: &Value) -> Result<Value> {
     // Raster effects print as images at the document's raster effects resolution.
     let flat = super::rasterfx::flatten_raster_effects(doc);
     let doc = flat.as_ref().unwrap_or(doc);
+    let bitmap = set.advanced.print_as_bitmap && set.output.mode == vectorcraft_pdf::OutputMode::Composite;
+    let expanded = if postscript && !bitmap {
+        let options = super::printadvanced::flattener(PRINT, &set, &s.prefs.flattener_presets)?;
+        options.as_ref().map(|o| expand_transparent_tile_images(doc, o)).transpose()?.flatten()
+    } else {
+        None
+    };
+    let doc = expanded.as_ref().unwrap_or(doc);
     // Print as Bitmap and the flattener preset (Advanced) render and flatten it first.
     let prepared = super::printadvanced::prepare(PRINT, doc, &set, &s.prefs.flattener_presets, s.prefs.black_output == "rich")?;
     let path = str_param(p, "path");
@@ -201,7 +210,6 @@ fn print(s: &mut Session, p: &Value) -> Result<Value> {
         None => Level::default(),
     };
     let mut warnings = vec![];
-    let bitmap = set.advanced.print_as_bitmap && set.output.mode == vectorcraft_pdf::OutputMode::Composite;
     if !bitmap && doc.layers.iter().any(|l| l.shows_transparency()) {
         warnings.push("transparency is flattened into opaque art and images (PostScript has none): see flattenerPreset".to_string());
     }

crates/engine/src/tests/legacy_ai.rs

@@ -704,3 +704,100 @@ fn only_the_transparent_pattern_fill_of_a_leaf_is_expanded() {
     assert_eq!(patterns.len(), 1, "{patterns:?}");
     assert!(images > 0);
 }
+
+#[test]
+fn pdf13_flattens_see_through_pattern_tile_images() {
+    let mut s = pattern_over_blue(&checker([0, 200, 0, 255], [255, 255, 255, 0]), 1.0);
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    let r = s.execute("document.exportPdf", &json!({"compatibility": "1.3"})).unwrap();
+    let bytes = vectorcraft_format::base64_decode(r["dataBase64"].as_str().unwrap()).unwrap();
+    let page = vectorcraft_pdf::import(&bytes).unwrap();
+    let after = [colour_at(&page, 72.0, 72.0), colour_at(&page, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
+}
+
+#[test]
+fn pdf13_flattens_colorized_pattern_tile_images() {
+    let mut s = colorized_pattern_over_blue();
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    let r = s.execute("document.exportPdf", &json!({"compatibility": "1.3"})).unwrap();
+    let bytes = vectorcraft_format::base64_decode(r["dataBase64"].as_str().unwrap()).unwrap();
+    let page = vectorcraft_pdf::import(&bytes).unwrap();
+    let after = [colour_at(&page, 72.0, 72.0), colour_at(&page, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
+}
+
+#[test]
+fn eps_level3_writes_see_through_pattern_tile_images() {
+    let mut s = pattern_over_blue(&checker([0, 200, 0, 255], [255, 255, 255, 0]), 1.0);
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    let r = s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3})).unwrap();
+    s.execute("document.open", &json!({"name": "p.eps", "dataBase64": r["dataBase64"], "editingData": false})).unwrap();
+    let page = &s.doc().unwrap().doc;
+    let after = [colour_at(page, 72.0, 72.0), colour_at(page, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
+}
+
+#[test]
+fn eps_flattens_colorized_pattern_tile_images_and_warns() {
+    let mut s = colorized_pattern_over_blue();
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    let r = s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3})).unwrap();
+    assert!(warnings(&r).iter().any(|w| w.contains("transparency is flattened")), "{:?}", warnings(&r));
+    s.execute("document.open", &json!({"name": "p.eps", "dataBase64": r["dataBase64"], "editingData": false})).unwrap();
+    let after = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
+}
+
+#[test]
+fn eps_warns_for_colorized_image_nested_in_pattern_tile_group() {
+    let mut s = colorized_pattern_over_blue();
+    let d = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().doc);
+    let id = d.alloc_id();
+    let art = std::mem::take(&mut d.patterns[0].art);
+    d.patterns[0].art.push(std::sync::Arc::new(vectorcraft_doc::Node::group(id, art)));
+    let r = s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3})).unwrap();
+    assert!(warnings(&r).iter().any(|w| w.contains("transparency is flattened")), "{:?}", warnings(&r));
+}
+
+#[test]
+fn postscript_print_warns_for_colorized_pattern_tile_image() {
+    let mut s = colorized_pattern_over_blue();
+    let r = s.execute("file.print", &json!({"format": "postscript"})).unwrap();
+    assert!(warnings(&r).iter().any(|w| w.contains("transparency is flattened")), "{:?}", warnings(&r));
+}
+
+#[test]
+fn eps_pattern_does_not_flatten_a_separate_see_through_image() {
+    let mut output = vec![];
+    for tile in [checker([0, 200, 0, 255], [255, 230, 0, 255]), checker([0, 200, 0, 255], [255, 255, 255, 0])] {
+        let mut s = pattern_over_blue(&tile, 1.0);
+        s.execute("shape.rectangle", &json!({"x": 8, "y": 8, "width": 24, "height": 24})).unwrap();
+        s.execute("paint.setFill", &json!({"color": "#ff0000"})).unwrap();
+        let p = json!({"name": "i.png", "dataBase64": vectorcraft_format::base64_encode(&png(20, 20, [0, 0, 0, 128])), "at": [20, 20]});
+        s.execute("file.place", &p).unwrap();
+        let r =
+            s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3, "flattenerPreset": "high"})).unwrap();
+        let image_warning = warnings(&r).iter().any(|w| w.contains("partly transparent"));
+        s.execute("document.open", &json!({"name": "p.eps", "dataBase64": r["dataBase64"], "editingData": false})).unwrap();
+        output.push((colour_at(&s.doc().unwrap().doc, 20.0, 20.0), image_warning));
+    }
+    assert_eq!(output[0], output[1], "EPS layer image changed when a see-through pattern tile was present: {output:?}");
+}
+
+#[test]
+fn opaque_pattern_tile_images_do_not_change_during_export_flattening() {
+    let s = pattern_over_blue(&checker([0, 200, 0, 255], [255, 230, 0, 255]), 1.0);
+    let d = &s.doc().unwrap().doc;
+    assert!(crate::cmd::flatten::flatten_document(d, &crate::cmd::FlattenOptions::default()).unwrap().is_none());
+    let o = crate::cmd::FlattenOptions { no_transparency: Some(crate::cmd::flatten::see_through_images(d)), ..Default::default() };
+    assert!(crate::cmd::flatten::flatten_document(d, &o).unwrap().is_none());
+}