fix(render): blend large CMYK groups in every tile; decline nested isolated other-space groups

Review of #3777 (test report by mtlcl410).

- A group wider than about 512 device pixels lost its blend mode in the
  tiles that a paint covers completely: on the bare pass canvas, vello_cpu
  fills such a tile without the blend mode. At 300 dpi, magenta then black
  with Difference drew black in the middle of the group, not green. Each
  ink pass now draws inside a layer of its own.
- A nested isolated group that names another blending space (for example
  /CS /DeviceRGB) blends in that space (ISO 32000-2 §11.3.4). The
  interpreter now tells the device about such a group
  (Device::group_blends_outside_device_cmyk), and a pass that meets one
  declines, so the enclosing group is drawn as before. A non-isolated
  group's /CS is ignored (ISO 32000-2 §11.6.6, Table 145, CS: it takes its parent's space), so
  it stays in the enclosing group's CMYK.

Test: raster::tests::isolated_cmyk_groups_blend_at_any_size_and_respect_nested_spaces.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/render/src/raster/tests/isolated_cmyk_groups.rs

@@ -95,3 +95,76 @@ endstream endobj
     let rgb_group = render("/CS /DeviceRGB /I true", &rgb);
     assert_eq!(cmyk_group.rgba, rgb_group.rgba, "contents without tints are drawn as before");
 }
+
+/// Two cases from the review of #3777. A group wider than about two of vello_cpu's tiles lost
+/// its blend mode in the tiles a paint covered completely, which drew black over magenta at
+/// 300 dpi where Difference gives green; each pass now draws inside a layer. And a nested
+/// isolated group that names another blending space (here DeviceRGB) blends in that space
+/// (ISO 32000-2 §11.3.4), so the enclosing CMYK group is drawn as before; a non-isolated
+/// one ignores its /CS and blends in the enclosing group's CMYK (Table 145).
+#[test]
+fn isolated_cmyk_groups_blend_at_any_size_and_respect_nested_spaces() {
+    let render = |inner_cs: Option<&str>, scale: f32| {
+        // `inner_cs`: the nested group's /CS and /I entries.
+        let group = "0 1 0 0 k 0 0 200 200 re f /Diff gs 0 0 0 1 k 0 0 200 200 re f";
+        let (outer_body, extra) = match inner_cs {
+            None => (group.to_string(), String::new()),
+            Some(cs) => (
+                "/In Do".to_string(),
+                format!(
+                    "6 0 obj << /Type /XObject /Subtype /Form /BBox [0 0 200 200] /Group << /S /Transparency {cs} >> /Resources << /ExtGState << /Diff << /BM /Difference >> >> >> /Length {} >> stream\n{group}\nendstream endobj\n",
+                    group.len()
+                ),
+            ),
+        };
+        let page = "q 1 0 0 1 10 70 cm /X0 Do Q 1 0 1 0 k 10 10 200 50 re f";
+        let pdf = format!(
+            "%PDF-1.7
+1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj
+2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj
+3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 220 280] /Contents 4 0 R /Resources << /XObject << /X0 5 0 R >> >> >> endobj
+4 0 obj << /Length {} >> stream
+{page}
+endstream endobj
+5 0 obj << /Type /XObject /Subtype /Form /BBox [0 0 200 200] /Group << /S /Transparency /CS /DeviceCMYK /I true >> /Resources << /XObject << /In 6 0 R >> /ExtGState << /Diff << /BM /Difference >> >> >> /Length {} >> stream
+{outer_body}
+endstream endobj
+{extra}trailer << /Root 1 0 R >>
+%%EOF",
+            page.len(),
+            outer_body.len()
+        );
+        let mut r = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default());
+        let p = r.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale, tag: 0, ..Default::default() });
+        assert!(p.error.is_none(), "{:?}", p.error);
+        p
+    };
+    let px = |p: &RenderedPage, x: f32, y: f32, scale: f32| {
+        let (x, y) = ((x * scale) as u32, (y * scale) as u32);
+        p.rgba[((y * p.width + x) * 4) as usize..][..3].to_vec()
+    };
+    let distance = |a: &[u8], b: &[u8]| a.iter().zip(b).map(|(a, b)| a.abs_diff(*b)).max().unwrap_or(0);
+
+    // 200 pt at scale 4 is 800 px: the centre lies in tiles the paints cover completely.
+    for scale in [1.0, 4.0] {
+        let p = render(None, scale);
+        let strip = px(&p, 110.0, 245.0, scale);
+        for (x, y) in [(110.0, 110.0), (15.0, 15.0), (110.0, 205.0)] {
+            let group = px(&p, x, y, scale);
+            assert!(distance(&group, &strip) <= 3, "scale {scale} at ({x}, {y}): group {group:?}, strip {strip:?}");
+        }
+    }
+
+    // A nested DeviceRGB group: the whole CMYK group falls back, so the page is drawn exactly
+    // as when the enclosing group is not offered to the CMYK path at all.
+    let nested_rgb = render(Some("/CS /DeviceRGB /I true"), 1.0);
+    let green = px(&nested_rgb, 110.0, 245.0, 1.0);
+    assert!(distance(&px(&nested_rgb, 110.0, 110.0, 1.0), &green) > 100, "the nested group blends in RGB, not CMYK");
+    // A nested DeviceCMYK group stays in the CMYK path.
+    let nested_cmyk = render(Some("/CS /DeviceCMYK /I true"), 1.0);
+    assert!(distance(&px(&nested_cmyk, 110.0, 110.0, 1.0), &green) <= 3);
+    // A non-isolated nested group's /CS is ignored (ISO 32000-2 Table 145): it blends in the
+    // enclosing group's CMYK, even when it names DeviceRGB.
+    let non_isolated_rgb = render(Some("/CS /DeviceRGB /I false /K false"), 1.0);
+    assert!(distance(&px(&non_isolated_rgb, 110.0, 110.0, 1.0), &green) <= 3, "a non-isolated nested group inherits CMYK");
+}

vendor/hayro-interpret/src/cmyk_group.rs

@@ -93,3 +93,8 @@ impl<'a, 'b> CmykGroup<'a, 'b> {
 pub(crate) fn blends_in_device_cmyk(form: &Dict<'_>) -> bool {
     form.get::<Dict<'_>>(GROUP).and_then(|g| g.get::<Name<'_>>(CS)).is_some_and(|cs| matches!(cs.deref(), b"DeviceCMYK" | b"CMYK"))
 }
+
+/// Whether `form`'s transparency group names a blending colour space of its own (`/CS`).
+pub(crate) fn names_a_blending_space(form: &Dict<'_>) -> bool {
+    form.get::<Dict<'_>>(GROUP).is_some_and(|g| g.contains_key(CS))
+}

vendor/hayro-interpret/src/device.rs

@@ -115,6 +115,10 @@ pub trait Device<'a> {
     fn draw_isolated_cmyk_group(&mut self, _group: &CmykGroup<'a, '_>) -> bool {
         false
     }
+    /// PdfCraft patch (45): an isolated transparency group that names a blending colour space
+    /// other than `DeviceCMYK` is about to be drawn. A device drawing an enclosing group in CMYK
+    /// can no longer draw that group right, since this one blends in its own space.
+    fn group_blends_outside_device_cmyk(&mut self) {}
 }
 
 /// A device that discards all drawing operations.

vendor/hayro-interpret/src/x_object.rs

@@ -1,5 +1,5 @@
 use crate::cache::{Cache, DecodedImageCache, DecodedImageKey};
-use crate::cmyk_group::{CmykGroup, blends_in_device_cmyk};
+use crate::cmyk_group::{CmykGroup, blends_in_device_cmyk, names_a_blending_space};
 use crate::color::{ColorComponents, ColorSpace, DefaultColorSpaces, DeviceFamily, ToRgb};
 use crate::context::Context;
 use crate::device::Device;
@@ -215,6 +215,18 @@ pub(crate) fn draw_form_xobject<'a, 'b>(
     // its own, so a device able to blend in CMYK computes its interior apart and composites the
     // result through the layer pushed above (see `cmyk_group`). Knockout groups are left out:
     // their elements composite with the group's initial backdrop, not with each other.
+    // An isolated group that names another blending space blends in that space (§11.3.4); a
+    // device drawing an enclosing group in CMYK is told, so it can decline that group. A
+    // non-isolated group's /CS is ignored: it blends in its parent's space (ISO 32000-2
+    // Table 145, CS), so it stays in the enclosing group's CMYK.
+    if x_object.is_transparency_group
+        && x_object.is_isolated
+        && names_a_blending_space(&x_object.dict)
+        && !blends_in_device_cmyk(&x_object.dict)
+    {
+        device.group_blends_outside_device_cmyk();
+    }
+
     let blended_in_cmyk = own_layer
         && x_object.is_isolated
         && !x_object.is_knockout

vendor/hayro/src/renderer.rs

@@ -1157,7 +1157,12 @@ impl Renderer {
             erasing: false,
             erase_groups: 0,
         };
+        // The pass draws inside a layer of its own. On the bare canvas, vello_cpu fills a tile
+        // that a paint covers completely without the blend mode, so the middle of a group wider
+        // than about two tiles lost its blend (#3777).
+        renderer.ctx.push_layer(None, None, None, None, None);
         group.interpret(&mut renderer, offset);
+        renderer.ctx.pop_layer();
         if renderer.ink.unsupported {
             return None;
         }
@@ -1821,6 +1826,13 @@ impl<'a> Device<'a> for Renderer {
     /// with the group's opacity, soft mask and blend mode as before. Declined (and so drawn as
     /// before) under reading colours, inside a pass (a nested group then blends in the pass's
     /// own CMYK), and for contents a pass has no tints for.
+    /// PdfCraft patch (21): a nested group blending in another space can't be drawn in a pass.
+    fn group_blends_outside_device_cmyk(&mut self) {
+        if self.ink.pass.is_some() {
+            self.ink.unsupported = true;
+        }
+    }
+
     fn draw_isolated_cmyk_group(&mut self, group: &hayro_interpret::CmykGroup<'a, '_>) -> bool {
         if self.color_replacement.is_some() || self.ink.pass.is_some() {
             return false;