PDF/EPS export: pattern tile images in live art and hidden shapes flatten as drawn

Blends, envelopes, repeats and Intertwine groups that hold a see-through
or colorized pattern tile image are expanded into the art the writers
draw for them before the tiles are expanded, so those tiles are
flattened too. Random Transform effects in that art are baked first.

Hidden shapes, shapes in hidden groups or layers and template layers are
skipped: expanding a hidden shape made its tiles show in the file, and
art that is not written no longer makes a flattened copy of the document.

crates/engine/src/cmd/expand.rs

@@ -449,33 +449,107 @@ fn expand_pattern_fills(s: &mut Session) -> Result<()> {
 /// pattern fills stay. A tile image counts when its key is in `see_through` (it has see-through
 /// pixels), when it has a colorize paint that `colorize` counts, or when it, a group
 /// in the tile art, the fill, the shape or a group the shape is in has opacity or a blending
-/// mode. Returns how many shapes changed.
+/// mode. Blends, envelopes, repeats and Intertwine groups that hold such a fill are expanded
+/// first ([`expand_live_with_transparent_image_patterns`]). Hidden shapes and shapes in hidden
+/// groups or layers and in template layers stay as they are. Returns how many objects changed.
 pub(crate) fn expand_transparent_image_patterns(
     d: &mut Document,
     root: NodeId,
     see_through: &BTreeSet<String>,
     colorize: &dyn Fn(&Document, &Node) -> bool,
 ) -> Result<usize> {
+    let live = expand_live_with_transparent_image_patterns(d, root, see_through, colorize)?;
     let (_, n) = replace_leaves(d, root, |d, l| {
         // Most leaves have no pattern fill: skip the ancestry walk for them.
         if !l.appearance.items.iter().any(|i| pattern_fill(d, i).is_some()) {
             return None;
         }
+        // The expanded art would be shown.
+        if !d.ancestry(l.id).unwrap_or_default().iter().filter_map(|a| d.node(*a)).all(drawn) {
+            return None;
+        }
         expand_patterns_where(d, l, |d, i| transparent_pattern_fill(d, l, i, see_through, colorize))
     })?;
-    Ok(n)
+    Ok(live + n)
 }
 
-/// Whether a path or compound path of `d` has a pattern fill that
-/// [`expand_transparent_image_patterns`] would expand with `see_through` and `colorize`.
+/// Whether a layer of `d` holds a pattern fill that [`expand_transparent_image_patterns`] would
+/// expand with `see_through` and `colorize` ([`holds_transparent_image_patterns`]).
 pub(crate) fn has_transparent_image_patterns(d: &Document, see_through: &BTreeSet<String>, colorize: &dyn Fn(&Document, &Node) -> bool) -> bool {
-    let mut found = false;
-    d.walk(|n| {
-        if !found && matches!(n.kind, NodeKind::Path { .. } | NodeKind::Compound { .. }) {
-            found = n.appearance.items.iter().any(|i| transparent_pattern_fill(d, n, i, see_through, colorize));
+    d.layers.iter().any(|l| holds_transparent_image_patterns(d, l, see_through, colorize))
+}
+
+/// Whether `n`, art of `d`, is or holds a shown path or compound path (in groups, blends,
+/// envelopes, repeats and Intertwine groups) with a pattern fill that
+/// [`transparent_pattern_fill`] counts. Guides, clipping paths and template layers don't count.
+fn holds_transparent_image_patterns(d: &Document, n: &Node, see_through: &BTreeSet<String>, colorize: &dyn Fn(&Document, &Node) -> bool) -> bool {
+    if !drawn(n) {
+        return false;
+    }
+    match &n.kind {
+        NodeKind::Path { guide: true, .. } | NodeKind::Path { clipping: true, .. } => false,
+        NodeKind::Path { .. } | NodeKind::Compound { .. } => {
+            n.appearance.items.iter().any(|i| transparent_pattern_fill(d, n, i, see_through, colorize))
         }
-    });
-    found
+        _ => n.children().is_some_and(|ch| ch.iter().any(|c| holds_transparent_image_patterns(d, c, see_through, colorize))),
+    }
+}
+
+/// Whether `n` is shown and isn't a template layer.
+fn drawn(n: &Node) -> bool {
+    n.visible && !matches!(n.kind, NodeKind::Layer { template: true, .. })
+}
+
+/// Expand the shown blends, envelopes, repeats and Intertwine groups under `root` that hold a
+/// pattern fill [`holds_transparent_image_patterns`] counts into the art the exporters write for
+/// them ([`expand_live_deep`](vectorcraft_render::effects::expand_live_deep), under one budget for
+/// everything nested in each), so the shapes they draw are leaves. Returns how many were expanded.
+fn expand_live_with_transparent_image_patterns(
+    d: &mut Document,
+    root: NodeId,
+    see_through: &BTreeSet<String>,
+    colorize: &dyn Fn(&Document, &Node) -> bool,
+) -> Result<usize> {
+    fn outermost(d: &Document, n: &Node, see_through: &BTreeSet<String>, colorize: &dyn Fn(&Document, &Node) -> bool, out: &mut Vec<NodeId>) {
+        if !drawn(n) {
+            return;
+        }
+        match &n.kind {
+            NodeKind::Blend { .. } | NodeKind::Envelope { .. } | NodeKind::Repeat(_) | NodeKind::Intertwine { .. } => {
+                if holds_transparent_image_patterns(d, n, see_through, colorize) {
+                    out.push(n.id);
+                }
+            }
+            NodeKind::Group { children, .. } | NodeKind::Layer { children, .. } => {
+                children.iter().for_each(|c| outermost(d, c, see_through, colorize, out));
+            }
+            _ => {}
+        }
+    }
+    /// Fresh ids for everything under `n`: copies of the same art share theirs.
+    fn fresh_ids(d: &mut Document, n: &mut Node) {
+        for c in n.children_mut().into_iter().flatten() {
+            let c = Arc::make_mut(c);
+            c.id = d.alloc_id();
+            fresh_ids(d, c);
+        }
+    }
+    let mut live = vec![];
+    if let Some(n) = d.node(root) {
+        outermost(d, n, see_through, colorize, &mut live);
+    }
+    let mut count = 0;
+    for id in &live {
+        let Some(n) = d.node(*id) else { continue };
+        // Random Transform effects take their seed from the ids: bake them first ([`bake_seeded`]),
+        // and leave the object live when that can't be done.
+        let Some(mut g) = bake_seeded(&vectorcraft_render::effects::expand_live_deep(Some(d), n)) else { continue };
+        g.id = *id;
+        fresh_ids(d, &mut g);
+        *d.node_mut(*id).ok_or(EngineError::NoNode(*id))? = g;
+        count += 1;
+    }
+    Ok(count)
 }
 
 /// Whether `i`, an item of leaf `l`, is a pattern fill whose tile art holds a transparent image

crates/engine/src/cmd/flatten/apply.rs

@@ -303,7 +303,8 @@ pub(crate) fn expand_transparent_tile_images(doc: &Document, o: &FlattenOptions)
         return Ok(None);
     }
     let mut d = doc.clone();
-    let layers: Vec<NodeId> = d.layers.iter().map(|l| l.id).collect();
+    let layers: Vec<NodeId> =
+        d.layers.iter().filter(|l| l.visible && !matches!(l.kind, NodeKind::Layer { template: true, .. })).map(|l| l.id).collect();
     let mut changed = 0;
     for id in layers {
         changed += crate::cmd::expand::expand_transparent_image_patterns(&mut d, id, &keys, &colorize)?;

crates/engine/src/tests/legacy_ai.rs

@@ -817,3 +817,63 @@ fn opaque_pattern_tile_images_do_not_change_during_export_flattening() {
     let o = crate::cmd::FlattenOptions { no_transparency: Some(crate::cmd::flatten::see_through_images(d)), ..Default::default() };
     assert!(crate::cmd::flatten::flatten_document(d, &o).unwrap().is_none());
 }
+
+/// The id of the square [`pattern_over_blue`] fills with its pattern.
+fn pattern_square(s: &Session) -> vectorcraft_doc::NodeId {
+    let mut id = None;
+    s.doc().unwrap().doc.walk(|n| {
+        if n.appearance.items.iter().any(|i| matches!(i.paint(), vectorcraft_color::Paint::Pattern { .. })) {
+            id = Some(n.id);
+        }
+    });
+    id.unwrap()
+}
+
+/// PDF 1.3 draws `s` as the canvas draws it at two points of the pattern square.
+fn assert_pdf13_draws_pattern_square_as_canvas(s: &mut Session) {
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    let r = s.execute("document.exportPdf", &json!({"compatibility": "1.3"})).unwrap();
+    let bytes = vectorcraft_format::base64_decode(r["dataBase64"].as_str().unwrap()).unwrap();
+    let page = vectorcraft_pdf::import(&bytes).unwrap();
+    let after = [colour_at(&page, 72.0, 72.0), colour_at(&page, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
+}
+
+#[test]
+fn pdf13_flattens_colorized_pattern_tile_images_in_a_repeat() {
+    let mut s = colorized_pattern_over_blue();
+    let square = pattern_square(&s);
+    s.execute("object.repeat.grid", &json!({"ids": [square.0]})).unwrap();
+    assert!(s.doc().unwrap().doc.layers[0].count() > 3);
+    let d = &s.doc().unwrap().doc;
+    let mut repeats = 0;
+    d.walk(|n| repeats += usize::from(matches!(n.kind, vectorcraft_doc::NodeKind::Repeat(_))));
+    assert_eq!(repeats, 1);
+    assert!(crate::cmd::flatten::flatten_document(d, &crate::cmd::FlattenOptions::default()).unwrap().is_some());
+    assert_pdf13_draws_pattern_square_as_canvas(&mut s);
+}
+
+#[test]
+fn pdf13_leaves_a_hidden_shape_with_a_colorized_pattern_tile_image_hidden() {
+    let mut s = colorized_pattern_over_blue();
+    let square = pattern_square(&s);
+    std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().doc).node_mut(square).unwrap().visible = false;
+    assert_pdf13_draws_pattern_square_as_canvas(&mut s);
+}
+
+#[test]
+fn flattening_skips_pattern_tile_images_on_hidden_and_template_layers() {
+    for template in [false, true] {
+        let mut s = colorized_pattern_over_blue();
+        let d = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().doc);
+        let layer = std::sync::Arc::make_mut(&mut d.layers[0]);
+        match &mut layer.kind {
+            vectorcraft_doc::NodeKind::Layer { template: t, .. } if template => *t = true,
+            _ => layer.visible = false,
+        }
+        let d = &s.doc().unwrap().doc;
+        assert!(crate::cmd::flatten::flatten_document(d, &crate::cmd::FlattenOptions::default()).unwrap().is_none(), "template {template}");
+    }
+}