Flattening: an option leaves colorized images to writers that draw them

Flattening counted every colorized image as transparency, in the art and
in pattern tiles. A new flattener option, keeps_colorize, stops counting
an image colorized with a solid paint whose pixels are grayscale or 1-bit,
for a format whose writer draws such images in their ink. Opacity,
blending modes, opacity masks and effects still flatten them, and pattern
tiles are still expanded for those. The option is off, so exports flatten
as before; the EPS writer that draws colorized images turns it on.

The EPS pattern tile tests now check the exported colors, and a tile
group with opacity for the warning.

crates/doc/src/node/ops.rs

@@ -686,6 +686,12 @@ impl Node {
     /// stroke opacity below 100%, a blend mode, an opacity mask, effects, transparent gradient
     /// stops or a colorized image. Hidden objects show none.
     pub fn shows_transparency(&self) -> bool {
+        self.shows_transparency_with(&|_| true)
+    }
+
+    /// [`Self::shows_transparency`], counting a colorized image by itself only when `colorize`
+    /// accepts it, for files that write some colorized images in their colorize paint (EPS).
+    pub fn shows_transparency_with(&self, colorize: &dyn Fn(&Node) -> bool) -> bool {
         use vectorcraft_color::Paint;
         let see_through = |p: &Paint| matches!(p, Paint::Gradient(g) if g.gradient.stops.iter().any(|s| s.opacity < 1.0));
         let effects = |fx: &[crate::Effect]| fx.iter().any(|e| e.visible);
@@ -693,14 +699,14 @@ impl Node {
             && (self.opacity < 1.0
                 || self.blend != BlendMode::Normal
                 || self.extras().mask.as_ref().is_some_and(|m| !m.disabled)
-                || (matches!(self.kind, NodeKind::Image(_)) && self.extras().colorize.is_some())
+                || (matches!(self.kind, NodeKind::Image(_)) && self.extras().colorize.is_some() && colorize(self))
                 || effects(&self.appearance.effects)
                 || self
                     .appearance
                     .items
                     .iter()
                     .any(|i| i.visible() && (i.opacity() < 1.0 || i.blend() != BlendMode::Normal || see_through(i.paint()) || effects(i.effects())))
-                || self.children().is_some_and(|ch| ch.iter().any(|c| c.shows_transparency())))
+                || self.children().is_some_and(|ch| ch.iter().any(|c| c.shows_transparency_with(colorize))))
     }
 }
 

crates/engine/src/cmd/expand.rs

@@ -447,27 +447,32 @@ fn expand_pattern_fills(s: &mut Session) -> Result<()> {
 /// Expand the pattern fills under `root` that paint a transparent or colorized image into the tiles they paint
 /// ([`expand_patterns_where`]), so each tile image becomes an object of its own; the other
 /// pattern fills stay. A tile image counts when its key is in `see_through` (it has see-through
-/// pixels), when it has a colorize paint, or when it, a group
+/// pixels), when it has a colorize paint that `colorize` counts, or when it, a group
 /// in the tile art, the fill, the shape or a group the shape is in has opacity or a blending
 /// mode. Returns how many shapes changed.
-pub(crate) fn expand_transparent_image_patterns(d: &mut Document, root: NodeId, see_through: &BTreeSet<String>) -> Result<usize> {
+pub(crate) fn expand_transparent_image_patterns(
+    d: &mut Document,
+    root: NodeId,
+    see_through: &BTreeSet<String>,
+    colorize: &dyn Fn(&Document, &Node) -> bool,
+) -> Result<usize> {
     let (_, n) = replace_leaves(d, root, |d, l| {
         // Most leaves have no pattern fill: skip the ancestry walk for them.
         if !l.appearance.items.iter().any(|i| pattern_fill(d, i).is_some()) {
             return None;
         }
-        expand_patterns_where(d, l, |d, i| transparent_pattern_fill(d, l, i, see_through))
+        expand_patterns_where(d, l, |d, i| transparent_pattern_fill(d, l, i, see_through, colorize))
     })?;
     Ok(n)
 }
 
 /// Whether a path or compound path of `d` has a pattern fill that
-/// [`expand_transparent_image_patterns`] would expand with `see_through`.
-pub(crate) fn has_transparent_image_patterns(d: &Document, see_through: &BTreeSet<String>) -> bool {
+/// [`expand_transparent_image_patterns`] would expand with `see_through` and `colorize`.
+pub(crate) fn has_transparent_image_patterns(d: &Document, see_through: &BTreeSet<String>, colorize: &dyn Fn(&Document, &Node) -> bool) -> bool {
     let mut found = false;
     d.walk(|n| {
         if !found && matches!(n.kind, NodeKind::Path { .. } | NodeKind::Compound { .. }) {
-            found = n.appearance.items.iter().any(|i| transparent_pattern_fill(d, n, i, see_through));
+            found = n.appearance.items.iter().any(|i| transparent_pattern_fill(d, n, i, see_through, colorize));
         }
     });
     found
@@ -475,11 +480,17 @@ pub(crate) fn has_transparent_image_patterns(d: &Document, see_through: &BTreeSe
 
 /// Whether `i`, an item of leaf `l`, is a pattern fill whose tile art holds a transparent image
 /// ([`transparent_image`]), counting the fill's and the shape's opacity and blending mode.
-fn transparent_pattern_fill(d: &Document, l: &Node, i: &AppearanceItem, see_through: &BTreeSet<String>) -> bool {
+fn transparent_pattern_fill(
+    d: &Document,
+    l: &Node,
+    i: &AppearanceItem,
+    see_through: &BTreeSet<String>,
+    colorize: &dyn Fn(&Document, &Node) -> bool,
+) -> bool {
     let Some((f, def, _)) = pattern_fill(d, i) else { return false };
     let shape = d.ancestry(l.id).unwrap_or_default().iter().filter_map(|a| d.node(*a)).any(blended);
     let on = shape || f.opacity < 1.0 || f.blend != BlendMode::Normal;
-    def.art.iter().any(|a| transparent_image(a, see_through, on))
+    def.art.iter().any(|a| transparent_image(d, a, see_through, colorize, on))
 }
 
 /// Whether `n` has opacity or a blending mode of its own.
@@ -487,13 +498,14 @@ fn blended(n: &Node) -> bool {
     n.opacity < 1.0 || n.blend != BlendMode::Normal
 }
 
-/// Whether `n` is or holds an image with see-through pixels (key in `keys`), a colorize paint,
-/// opacity or a blending mode of its own or of a group in `n`, or anywhere when `on`.
-fn transparent_image(n: &Node, keys: &BTreeSet<String>, on: bool) -> bool {
+/// Whether `n` (art of `d`) is or holds an image with see-through pixels (key in `keys`), a
+/// colorize paint that `colorize` counts, opacity or a blending mode of its own or of a group in
+/// `n`, or anywhere when `on`.
+fn transparent_image(d: &Document, n: &Node, keys: &BTreeSet<String>, colorize: &dyn Fn(&Document, &Node) -> bool, on: bool) -> bool {
     let on = on || blended(n);
     match &n.kind {
-        NodeKind::Image(im) => on || keys.contains(&im.key) || n.extras().colorize.is_some(),
-        _ => n.children().is_some_and(|ch| ch.iter().any(|c| transparent_image(c, keys, on))),
+        NodeKind::Image(im) => on || keys.contains(&im.key) || (n.extras().colorize.is_some() && colorize(d, n)),
+        _ => n.children().is_some_and(|ch| ch.iter().any(|c| transparent_image(d, c, keys, colorize, on))),
     }
 }
 

crates/engine/src/cmd/fileio/eps.rs

@@ -301,7 +301,7 @@ fn encode_document(doc: &Document, p: &Value, saving: bool) -> Result<Encoded> {
     let regions: Vec<Rect> = pages.iter().map(|(_, region, _)| *region).collect();
     let expanded = expand_transparent_tile_images(doc, &set.flatten)?;
     let source = expanded.as_ref().unwrap_or(doc);
-    let transparent = source.layers.iter().any(|l| l.shows_transparency());
+    let transparent = source.layers.iter().any(|l| set.flatten.shows_transparency(source, l));
     let flat = flatten_document(source, &set.flatten)?;
     let drawn = flat.as_ref().unwrap_or(source);
     let mut enc = Encoded { warnings, joiner: Some("_"), ..Encoded::default() };

crates/engine/src/cmd/fileio/legacy_ai.rs

@@ -167,7 +167,7 @@ fn expand_image_patterns(d: &mut Document) -> Result<()> {
     let keys = super::super::flatten::see_through_images(d);
     let layers: Vec<NodeId> = d.layers.iter().map(|l| l.id).collect();
     for id in layers {
-        super::super::expand::expand_transparent_image_patterns(d, id, &keys)?;
+        super::super::expand::expand_transparent_image_patterns(d, id, &keys, &|_, _| true)?;
     }
     Ok(())
 }

crates/engine/src/cmd/flatten/apply.rs

@@ -74,7 +74,7 @@ pub(super) fn plan(src: &Document, roots: &[NodeId], o: &FlattenOptions, render:
     let mut sc = Scratch { brushes: vectorcraft_brush::library(&baked), doc: baked.clone() };
     let plain: Vec<Node> = art.iter().map(|n| sc.plain(n)).collect();
     let reaches: Vec<Option<Rect>> = plain.iter().map(reach).collect();
-    let see_through: Vec<bool> = plain.iter().map(|n| n.shows_transparency() || o.draws_see_through_image(n)).collect();
+    let see_through: Vec<bool> = plain.iter().map(|n| o.shows_transparency(&baked, n) || o.draws_see_through_image(n)).collect();
     let cmyk = src.color_mode == ColorMode::Cmyk;
     let mut out = Plan { flat: vec![], kept: vec![], transparent: roots.iter().zip(&see_through).filter(|(_, t)| **t).map(|(id, _)| *id).collect() };
     // Baking above stopped early when cancelled; each group is a place to stop (a plan changes nothing).
@@ -298,28 +298,29 @@ pub(crate) fn expand_transparent_tile_images(doc: &Document, o: &FlattenOptions)
         return Ok(None);
     }
     let keys = o.no_transparency.clone().unwrap_or_default();
-    if !crate::cmd::expand::has_transparent_image_patterns(doc, &keys) {
+    let colorize = |d: &Document, n: &Node| o.counts_colorize(d, n);
+    if !crate::cmd::expand::has_transparent_image_patterns(doc, &keys, &colorize) {
         return Ok(None);
     }
     let mut d = doc.clone();
     let layers: Vec<NodeId> = d.layers.iter().map(|l| l.id).collect();
     let mut changed = 0;
     for id in layers {
-        changed += crate::cmd::expand::expand_transparent_image_patterns(&mut d, id, &keys)?;
+        changed += crate::cmd::expand::expand_transparent_image_patterns(&mut d, id, &keys, &colorize)?;
     }
     Ok((changed > 0).then_some(d))
 }
 
 /// `doc` with everything it draws flattened with `o`, as formats without transparency (EPS)
 /// write it: every visible object of the visible layers, locked ones too (a clipping layer's
-/// clipping path stays). Pattern tile images with opacity, blending or colorize are expanded;
-/// see-through tile pixels are expanded when `o.no_transparency` counts them. `None` when nothing
-/// changes.
+/// clipping path stays). Pattern tile images with opacity, blending or a colorize paint that
+/// `o` counts ([`FlattenOptions::counts_colorize`]) are expanded; see-through tile pixels are
+/// expanded when `o.no_transparency` counts them. `None` when nothing changes.
 pub(crate) fn flatten_document(doc: &Document, o: &FlattenOptions) -> Result<Option<Document>> {
     let expanded = expand_transparent_tile_images(doc, o)?;
     let doc = expanded.as_ref().unwrap_or(doc);
     let outlining = o.text_to_outlines || o.strokes_to_outlines || !o.preserve_overprints;
-    if !outlining && !doc.layers.iter().any(|l| l.shows_transparency() || o.draws_see_through_image(l)) {
+    if !outlining && !doc.layers.iter().any(|l| o.shows_transparency(doc, l) || o.draws_see_through_image(l)) {
         return Ok(expanded);
     }
     let mut roots = vec![];

crates/engine/src/cmd/flatten/options.rs

@@ -2,7 +2,7 @@ use super::analysis::draws_image;
 use super::apply::{apply, kept_work, plan};
 use super::presets::{FlattenerPreset, saved_preset};
 use super::report::target_roots;
-use super::{BTreeSet, Deserialize, EngineError, Node, Result, Serialize, Session, Value, bad, json, str_param};
+use super::{BTreeSet, Deserialize, Document, EngineError, Node, NodeKind, Paint, Result, Serialize, Session, Value, bad, json, str_param};
 
 /// Object → Flatten Transparency settings: a preset's, adjusted by the command's params.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -34,6 +34,10 @@ pub struct FlattenOptions {
     /// amounts). `None` for ordinary flattening. Not a preset option.
     #[serde(skip)]
     pub no_transparency: Option<BTreeSet<String>>,
+    /// The file writes a colorized image in its colorize paint (EPS), so a colorize paint by
+    /// itself doesn't count as transparency. Not a preset option.
+    #[serde(skip)]
+    pub keeps_colorize: bool,
 }
 
 impl Default for FlattenOptions {
@@ -81,6 +85,7 @@ impl FlattenOptions {
             preserve_alpha: false,
             preserve_overprints: true,
             no_transparency: None,
+            keeps_colorize: false,
         }
     }
 
@@ -138,6 +143,21 @@ impl FlattenOptions {
         Ok(o)
     }
 
+    /// Whether `n`, art of `d`, shows transparency these options flatten
+    /// ([`Node::shows_transparency`], colorized images as [`Self::counts_colorize`] says).
+    pub(crate) fn shows_transparency(&self, d: &Document, n: &Node) -> bool {
+        n.shows_transparency_with(&|c| self.counts_colorize(d, c))
+    }
+
+    /// Whether colorized image `n`, art of `d`, counts as transparency by itself. With
+    /// [`Self::keeps_colorize`], an image colorized with a solid paint whose pixels are grayscale
+    /// or 1-bit doesn't: the EPS writer draws it in that paint.
+    pub(crate) fn counts_colorize(&self, d: &Document, n: &Node) -> bool {
+        let NodeKind::Image(im) = &n.kind else { return true };
+        let solid = matches!(n.extras().colorize.as_deref(), Some(Paint::Solid { .. }));
+        !self.keeps_colorize || !solid || d.image_blob(im).is_some_and(|b| b.gray_source().is_none())
+    }
+
     /// `n` shows an image whose see-through pixels count as transparency.
     pub(super) fn draws_see_through_image(&self, n: &Node) -> bool {
         self.no_transparency.as_ref().is_some_and(|keys| draws_image(n, keys))

crates/engine/src/tests/legacy_ai.rs

@@ -546,7 +546,7 @@ fn a_colorized_grayscale_image_pattern_fill_is_expanded() {
     let keys = crate::cmd::flatten::see_through_images(d);
     assert!(keys.is_empty());
     let layer = d.layers[0].id;
-    assert_eq!(crate::cmd::expand::expand_transparent_image_patterns(d, layer, &keys).unwrap(), 1);
+    assert_eq!(crate::cmd::expand::expand_transparent_image_patterns(d, layer, &keys, &|_, _| true).unwrap(), 1);
     let mut patterns = 0;
     let mut tile_images = 0;
     d.walk(|n| {
@@ -688,7 +688,7 @@ fn only_the_transparent_pattern_fill_of_a_leaf_is_expanded() {
     let d = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().doc);
     let keys = crate::cmd::flatten::see_through_images(d);
     let layer = d.layers[0].id;
-    assert_eq!(crate::cmd::expand::expand_transparent_image_patterns(d, layer, &keys).unwrap(), 1);
+    assert_eq!(crate::cmd::expand::expand_transparent_image_patterns(d, layer, &keys, &|_, _| true).unwrap(), 1);
     let mut patterns = vec![];
     let mut images = 0;
     d.walk(|n| {
@@ -745,11 +745,10 @@ fn eps_level3_writes_see_through_pattern_tile_images() {
 }
 
 #[test]
-fn eps_flattens_colorized_pattern_tile_images_and_warns() {
+fn eps_writes_colorized_pattern_tile_images_in_their_colors() {
     let mut s = colorized_pattern_over_blue();
     let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
     let r = s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3})).unwrap();
-    assert!(warnings(&r).iter().any(|w| w.contains("transparency is flattened")), "{:?}", warnings(&r));
     s.execute("document.open", &json!({"name": "p.eps", "dataBase64": r["dataBase64"], "editingData": false})).unwrap();
     let after = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
     for (a, b) in before.iter().zip(after) {
@@ -758,14 +757,31 @@ fn eps_flattens_colorized_pattern_tile_images_and_warns() {
 }
 
 #[test]
-fn eps_warns_for_colorized_image_nested_in_pattern_tile_group() {
+fn flattening_that_keeps_colorize_leaves_colorized_pattern_tile_images_alone() {
+    let s = colorized_pattern_over_blue();
+    let d = &s.doc().unwrap().doc;
+    assert!(crate::cmd::flatten::flatten_document(d, &crate::cmd::FlattenOptions::default()).unwrap().is_some());
+    let o = crate::cmd::FlattenOptions { keeps_colorize: true, ..Default::default() };
+    assert!(crate::cmd::flatten::flatten_document(d, &o).unwrap().is_none(), "the pattern isn't expanded for a colorize paint alone");
+}
+
+#[test]
+fn eps_flattens_colorized_image_in_a_see_through_pattern_tile_group_and_warns() {
     let mut s = colorized_pattern_over_blue();
     let d = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().doc);
     let id = d.alloc_id();
     let art = std::mem::take(&mut d.patterns[0].art);
-    d.patterns[0].art.push(std::sync::Arc::new(vectorcraft_doc::Node::group(id, art)));
+    let mut group = vectorcraft_doc::Node::group(id, art);
+    group.opacity = 0.5;
+    d.patterns[0].art.push(std::sync::Arc::new(group));
+    let before = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
     let r = s.execute("document.exportEps", &json!({"previewFormat": "none", "useArtboards": true, "level": 3})).unwrap();
     assert!(warnings(&r).iter().any(|w| w.contains("transparency is flattened")), "{:?}", warnings(&r));
+    s.execute("document.open", &json!({"name": "p.eps", "dataBase64": r["dataBase64"], "editingData": false})).unwrap();
+    let after = [colour_at(&s.doc().unwrap().doc, 72.0, 72.0), colour_at(&s.doc().unwrap().doc, 88.0, 72.0)];
+    for (a, b) in before.iter().zip(after) {
+        assert!(a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= 4), "before {before:?} after {after:?}");
+    }
 }
 
 #[test]