PDF: colorized images count as transparency for PDF 1.3 and PDF/X

A colorized grayscale or 1-bit image draws its white pixels transparent and is written to
PDF with the image as a soft mask. Node::shows_transparency did not count it, so a PDF 1.3,
PDF/X-1a or PDF/X-3 export whose only transparency was a colorized image was not flattened
and kept the soft mask. A legacy .ai save also wrote such an image as opaque.

- shows_transparency counts a visible image with a colorize paint.
- The legacy .ai writer flattens or leaves out colorized images like other see-through images.

crates/doc/src/node/ops.rs

@@ -683,8 +683,8 @@ impl Node {
 
 impl Node {
     /// Whether this object shows any transparency, itself or anything inside it: object, fill or
-    /// stroke opacity below 100%, a blend mode, an opacity mask, effects or transparent gradient
-    /// stops. Hidden objects show none.
+    /// stroke opacity below 100%, a blend mode, an opacity mask, effects, transparent gradient
+    /// stops or a colorized image. Hidden objects show none.
     pub fn shows_transparency(&self) -> bool {
         use vectorcraft_color::Paint;
         let see_through = |p: &Paint| matches!(p, Paint::Gradient(g) if g.gradient.stops.iter().any(|s| s.opacity < 1.0));
@@ -693,6 +693,7 @@ impl Node {
             && (self.opacity < 1.0
                 || self.blend != BlendMode::Normal
                 || self.extras().mask.as_ref().is_some_and(|m| !m.disabled)
+                || (matches!(self.kind, NodeKind::Image(_)) && self.extras().colorize.is_some())
                 || effects(&self.appearance.effects)
                 || self
                     .appearance

crates/engine/src/cmd/fileio/legacy_ai.rs

@@ -174,8 +174,8 @@ fn expand_image_patterns(d: &mut Document) -> Result<()> {
 }
 
 /// The images of `d` a legacy `.ai` can't hold, in paint order with their places in it (as
-/// [`Document::walk`] counts). These are the images with see-through pixels, and the images
-/// that have opacity or a blending mode of their own or, with `flatten`, of a group they are in.
+/// [`Document::walk`] counts). These have see-through pixels or a colorize paint, or have opacity
+/// or a blending mode of their own or, with `flatten`, of a group they are in.
 /// With `flatten` only visible images are returned, because a hidden one (which the hidden objects
 /// option may write) is written over white. With `omit` all of them are returned.
 fn see_through_images(d: &Document, mode: SeeThrough) -> Vec<(usize, NodeId)> {
@@ -184,7 +184,11 @@ fn see_through_images(d: &Document, mode: SeeThrough) -> Vec<(usize, NodeId)> {
     let mut at = 0;
     d.walk(|n| {
         if let NodeKind::Image(im) = &n.kind
-            && (keys.contains(&im.key) || n.opacity < 1.0 || n.blend != BlendMode::Normal || (mode == SeeThrough::Flatten && transparent(d, n.id)))
+            && (keys.contains(&im.key)
+                || n.extras().colorize.is_some()
+                || n.opacity < 1.0
+                || n.blend != BlendMode::Normal
+                || (mode == SeeThrough::Flatten && transparent(d, n.id)))
         {
             out.push((at, n.id));
         }

crates/engine/src/cmd/fileio/pdf.rs

@@ -317,9 +317,9 @@ pub const DEFAULT_FLATTENER: &str = "High Resolution";
 
 /// `doc` with its transparency flattened for a PDF 1.3 file (asked for, or PDF/X-1a and PDF/X-3,
 /// whose files have none), with `flattenerPreset` (default High Resolution) and `p`'s `flattener`
-/// options over it; images with see-through pixels count as transparency, rasterized areas are
-/// clipped to their regions, and discarded overprints are dropped. `None` when nothing changes (or
-/// the file keeps transparency).
+/// options over it; images with see-through pixels and colorized images count as transparency,
+/// rasterized areas are clipped to their regions, and discarded overprints are dropped. `None`
+/// when nothing changes (or the file keeps transparency).
 fn flatten_for(cmd: &str, doc: &Document, set: &PdfSettings, p: &Value) -> Result<Option<Document>> {
     if !set.pdf13() {
         return Ok(None);

crates/engine/src/tests/colorize.rs

@@ -54,6 +54,84 @@ fn colorize(s: &Session, id: NodeId) -> Option<Paint> {
     s.doc().unwrap().doc.node(id).unwrap().extras().colorize.as_deref().cloned()
 }
 
+#[test]
+fn colorized_gray_image_is_flattened_for_pdf_13() {
+    let mut s = session();
+    let id = image(&mut s, "gray", gray(false));
+    assert!(!s.doc().unwrap().doc.layers[0].shows_transparency());
+    let plain = s.execute("document.exportPdf", &json!({"compatibility": "1.3"})).unwrap();
+    assert!(
+        plain["warnings"].as_array().unwrap().iter().all(|w| !w.as_str().unwrap().contains("transparency is flattened")),
+        "{}",
+        plain["warnings"]
+    );
+
+    s.execute("paint.setFill", &json!({"color": "#ff0000", "ids": [id.0]})).unwrap();
+    assert!(s.doc().unwrap().doc.layers[0].shows_transparency());
+    let tinted = s.execute("document.exportPdf", &json!({"compatibility": "1.3"})).unwrap();
+    assert!(
+        tinted["warnings"].as_array().unwrap().iter().any(|w| w.as_str().unwrap().contains("transparency is flattened")),
+        "{}",
+        tinted["warnings"]
+    );
+    let bytes = vectorcraft_format::base64_decode(tinted["dataBase64"].as_str().unwrap()).unwrap();
+    assert!(!String::from_utf8_lossy(&bytes).contains("/SMask"), "PDF 1.3 has no soft mask");
+}
+
+#[test]
+fn colorized_gray_image_is_flattened_for_pdf_x_1a() {
+    let mut s = session();
+    let id = image(&mut s, "gray", gray(false));
+    s.execute("paint.setFill", &json!({"color": "#ff0000", "ids": [id.0]})).unwrap();
+    let out = s.execute("document.exportPdf", &json!({"standard": "pdfX1a"})).unwrap();
+    assert!(out["warnings"].as_array().unwrap().iter().any(|w| w.as_str().unwrap().contains("transparency is flattened")), "{}", out["warnings"]);
+    let bytes = vectorcraft_format::base64_decode(out["dataBase64"].as_str().unwrap()).unwrap();
+    assert!(!String::from_utf8_lossy(&bytes).contains("/SMask"), "PDF/X-1a has no soft mask");
+}
+
+#[test]
+fn colorized_gray_image_is_omitted_from_legacy_ai_when_requested() {
+    let mut s = session();
+    let id = image(&mut s, "gray", gray(false));
+    s.execute("paint.setFill", &json!({"color": "#ff0000", "ids": [id.0]})).unwrap();
+    let out = s.execute("file.saveCopy", &json!({"format": "ai", "aiVersion": "7", "transparentImages": "omit"})).unwrap();
+    assert!(out["warnings"].as_array().unwrap().iter().any(|w| w.as_str().unwrap().contains("are left out")), "{}", out["warnings"]);
+}
+
+#[test]
+fn flattened_colorized_image_shows_the_art_under_white_pixels() {
+    let mut s = session();
+    let background = s.execute("shape.rectangle", &json!({"x": 0, "y": 0, "width": 40, "height": 20})).unwrap()["id"].as_u64().unwrap();
+    s.execute("paint.setFill", &json!({"color": "#0000ff", "ids": [background]})).unwrap();
+    s.execute("paint.setStroke", &json!({"none": true, "ids": [background]})).unwrap();
+    let id = image(&mut s, "gray", gray(false));
+    s.edit("Scale image", |d, _| {
+        if let NodeKind::Image(im) = &mut d.node_mut(id).unwrap().kind {
+            im.xf = vectorcraft_geom::Affine::scale(20.0);
+        }
+        Ok(())
+    })
+    .unwrap();
+    s.execute("paint.setFill", &json!({"color": "#ff0000", "ids": [id.0]})).unwrap();
+    s.execute("object.arrange.bringToFront", &json!({"ids": [id.0]})).unwrap();
+    let doc = &s.doc().unwrap().doc;
+    let mut options = crate::cmd::flatten::FlattenOptions::preset("low").unwrap();
+    options.no_transparency = Some(Default::default());
+    options.clip_complex_regions = true;
+    let flat = crate::cmd::flatten::flatten_document(doc, &options).unwrap().unwrap();
+    let region = vectorcraft_geom::Rect::new(0.0, 0.0, 40.0, 20.0);
+    let before = vectorcraft_render::Renderer::new().render_region(doc, region, 1.0, true);
+    let after = vectorcraft_render::Renderer::new().render_region(&flat, region, 1.0, true);
+    for (x, expected) in [(10, [255, 0, 0]), (30, [0, 0, 255])] {
+        let a = before.pixel(x, 10);
+        let b = after.pixel(x, 10);
+        for channel in 0..3 {
+            assert!((i16::from(a[channel]) - i16::from(b[channel])).abs() <= 10, "x={x}: {a:?} became {b:?}");
+            assert!((i16::from(b[channel]) - expected[channel]).abs() <= 10, "x={x}: {b:?}");
+        }
+    }
+}
+
 #[test]
 fn a_fill_color_colorizes_a_gray_image_and_none_takes_it_off() {
     let mut s = session();