PDF/X-1a and X-3: flattening keeps a CMYK document's inks where it rasterizes (#2160)

Exporting a CMYK document to PDF/X-1a or PDF/X-3 flattens transparency
first. Where the flattener has no single colour (an image with see-through
pixels over a flat fill, gradients, masks, raster effects) it renders the
area into an image, and that image was always an RGB PNG. The PDF writer
then separated it back to CMYK, so CMYK art behind the image came back with
other inks: a C100 M6 Y41 K3 panel printed C75 M14 Y35 K5 under a
transparent logo, with a hard edge against the rest of the panel, and K
moved into C, M and Y.

For files without transparency the flattener's images are opaque and
clipped to their regions anyway, so in a CMYK document it now draws them
as ink amounts over the paper (Renderer::render_region_inks) and stores a
CMYK TIFF, which the PDF writer keeps as DeviceCMYK samples. An area too
large for a CMYK image falls back to the PNG it made before. In-app
Flatten Transparency and other flattening, whose images keep alpha, are
unchanged.

Test: tests_pdfx::pdfx_1a_flattening_keeps_the_inks_behind_an_image_in_a_cmyk_document
exports a CMYK panel under a half see-through PNG to PDF/X-1a, opens the
file again and checks the inks behind the see-through half.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/engine/src/cmd/fileio/tests_pdfx.rs

@@ -89,3 +89,51 @@ fn choosing_a_pdfx_standard_sets_its_version_and_turns_off_what_it_forbids() {
         assert!(s.execute("document.exportPdf", &json!({"preset": preset})).is_ok(), "{preset}");
     }
 }
+
+/// The ink amounts (0..1) `doc` shows at `p` in its images (the topmost image there).
+fn image_inks_at(doc: &vectorcraft_doc::Document, p: vectorcraft_geom::Point) -> Option<[f32; 4]> {
+    fn visit(n: &vectorcraft_doc::Node, out: &mut Vec<vectorcraft_doc::ImageObject>) {
+        if let vectorcraft_doc::NodeKind::Image(im) = &n.kind {
+            out.push((**im).clone());
+        }
+        for c in n.children().into_iter().flatten() {
+            visit(c, out);
+        }
+    }
+    let mut images = vec![];
+    for l in &doc.layers {
+        visit(l, &mut images);
+    }
+    images.iter().rev().find_map(|im| {
+        let q = im.xf.inverse() * p;
+        if q.x < 0.0 || q.y < 0.0 || q.x >= im.width as f64 || q.y >= im.height as f64 {
+            return None;
+        }
+        let ink = doc.images.get(&im.key)?.sample_cmyk_object(q.x, q.y, (im.width, im.height), 1)?;
+        Some(ink.map(|v| v as f32 / 255.0))
+    })
+}
+
+#[test]
+fn pdfx_1a_flattening_keeps_the_inks_behind_an_image_in_a_cmyk_document() {
+    // #2160: a CMYK panel under a PNG with see-through pixels. The flattener rasterizes where the
+    // image reaches; the panel there keeps its ink numbers instead of going through RGB.
+    let _g = crate::tests_colormgmt::GLOBAL.lock().unwrap_or_else(|e| e.into_inner());
+    let mut s = Session::new();
+    s.execute("file.new", &json!({"width": 200, "height": 100, "colorMode": "cmyk"})).unwrap();
+    s.execute("shape.rectangle", &json!({"x": 10, "y": 10, "width": 180, "height": 80})).unwrap();
+    let teal = [1.0, 0.06, 0.41, 0.03];
+    s.execute("paint.setFill", &json!({"color": {"c": teal[0], "m": teal[1], "y": teal[2], "k": teal[3]}})).unwrap();
+    s.execute("paint.setStroke", &json!({"none": true})).unwrap();
+    let mut png = vec![];
+    image::RgbaImage::from_fn(40, 40, |x, _| image::Rgba([0, 120, 255, if x < 20 { 0 } else { 255 }]))
+        .write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png)
+        .unwrap();
+    s.execute("file.place", &json!({"name": "logos.png", "dataBase64": vectorcraft_format::base64_encode(&png), "at": [100, 50]})).unwrap();
+    let v = s.execute("document.exportPdf", &json!({"standard": "pdfX1a"})).unwrap();
+    let bytes = vectorcraft_format::base64_decode(v["dataBase64"].as_str().unwrap()).unwrap();
+    let back = vectorcraft_pdf::import(&bytes).unwrap();
+    // Behind the image's see-through half: the panel's own inks, K included.
+    let ink = image_inks_at(&back, vectorcraft_geom::Point::new(88.0, 50.0)).expect("a CMYK image behind the see-through pixels");
+    assert!(ink.iter().zip(teal).all(|(a, b)| (a - b).abs() <= 1.5 / 255.0), "{ink:?} is not {teal:?}");
+}