fix(render): knockout passes clear a masked image's coverage, and nothing for an empty mesh

Review of #4898 (FredApps):

- An image or nested group of another blend mode than the pass's composited
  its layer at opacity 0, on the assumption that its shape was cleared first.
  `knockout_image` skips images with an `/SMask` and stencil masks, so a
  white Difference square stayed under a blue Normal masked image in the
  Difference pass and turned it yellow. The layer now composites with
  `Compose::DestOut` at full opacity and no soft mask, clearing exactly its
  coverage.
- In another pass (and an erase pass), a shading cleared under its path
  before the mesh-extent check that makes an off-page mesh paint nothing,
  so a Difference `sh` of such a mesh erased the blue under it. The clear
  now makes the same decision (`shading_bbox`, `mesh_area`, shared with the
  paint path).

Tests: `it::knockout_backdrop` gains a masked image, a stencil and an
off-page mesh case; each fails with its half of the fix reverted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9

crates/render/tests/it/knockout_backdrop.rs

@@ -35,7 +35,9 @@ fn stream(dict: &str, content: &str) -> String {
 /// content `content`. Its resources have `/D` (`/BM /Difference`), `/N` (`/BM /Normal`), `/H`
 /// (`/ca 0.5`), `/W`, a white square from 40 to 160 in a group of its own, `/Nest`, a group
 /// holding a non-isolated knockout group with a white Difference square from 80 to 120, and
-/// `/Img`, a white 1 x 1 image.
+/// `/Img`, a white 1 x 1 image, `/Img2`, a blue 2 x 1 image whose soft mask is opaque on the
+/// left and transparent on the right, `/St`, a 2 x 1 stencil mask painting its left pixel, and
+/// the shading `/Sh0`, a mesh whose one triangle lies off the page.
 /// Returns a lookup of the colour at PDF point (x, y).
 fn render(group: &str, content: &str) -> impl Fn(u32, u32) -> [u8; 3] {
     let gs = "/ExtGState << /D << /BM /Difference >> /H << /ca 0.5 >> /N << /BM /Normal >> >>";
@@ -46,7 +48,7 @@ fn render(group: &str, content: &str) -> impl Fn(u32, u32) -> [u8; 3] {
             .to_string(),
         stream("", "1 0 1 rg 0 0 200 200 re f /Fk Do"),
         stream(
-            &format!("/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Group << /S /Transparency /CS /DeviceRGB {group} >> /Resources << {gs} /XObject << /W 6 0 R /Nest 7 0 R /Img 9 0 R >> >>"),
+            &format!("/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Group << /S /Transparency /CS /DeviceRGB {group} >> /Resources << {gs} /XObject << /W 6 0 R /Nest 7 0 R /Img 9 0 R /Img2 10 0 R /St 12 0 R >> /Shading << /Sh0 13 0 R >> >>"),
             content,
         ),
         stream("/Type /XObject /Subtype /Form /BBox [0 0 200 200] /Group << /S /Transparency >>", "1 1 1 rg 40 40 120 120 re f"),
@@ -56,6 +58,16 @@ fn render(group: &str, content: &str) -> impl Fn(u32, u32) -> [u8; 3] {
             "/D gs 1 1 1 rg 80 80 40 40 re f",
         ),
         stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /ASCIIHexDecode", "FFFFFF>"),
+        stream(
+            "/Type /XObject /Subtype /Image /Width 2 /Height 1 /ColorSpace /DeviceRGB /BitsPerComponent 8 /SMask 11 0 R /Filter /ASCIIHexDecode",
+            "0000FF0000FF>",
+        ),
+        stream("/Type /XObject /Subtype /Image /Width 2 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /ASCIIHexDecode", "FF00>"),
+        stream("/Type /XObject /Subtype /Image /Width 2 /Height 1 /ImageMask true /BitsPerComponent 1 /Filter /ASCIIHexDecode", "40>"),
+        stream(
+            "/ShadingType 4 /ColorSpace /DeviceRGB /BitsPerCoordinate 8 /BitsPerComponent 8 /BitsPerFlag 8 /Decode [300 310 300 310 0 1 0 1 0 1] /Filter /ASCIIHexDecode",
+            "000000FF0000 00FF00FF0000 0000FFFF0000>",
+        ),
     ];
     let mut renderer = PageRenderer::new(Arc::new(pdf(&objects)), RenderConfig::default());
     let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 1.0, tag: 0, preview: false });
@@ -131,3 +143,27 @@ fn a_knockout_group_inside_an_element_keeps_the_outer_groups_passes() {
     check(&nested, 30, 30, GREEN, "Difference square before the nested groups");
     check(&nested, 165, 165, GREEN, "Difference square after the nested groups");
 }
+
+/// A masked image (or, below, a stencil) in another pass than its own replaces what is under its mask
+/// there too: a blue image over a white Difference square stays blue where its mask is opaque,
+/// and leaves the square to blend with the page where it is transparent (review of #4886's fix).
+#[test]
+fn a_masked_image_replaces_what_is_under_its_mask_in_every_pass() {
+    let masked = render(NON_ISOLATED, "/D gs 1 1 1 rg 40 40 120 120 re f /N gs q 120 0 0 120 40 40 cm /Img2 Do Q");
+    check(&masked, 60, 100, [0, 0, 255], "image over the square, mask opaque");
+    check(&masked, 140, 100, GREEN, "square under the transparent half of the mask");
+}
+
+#[test]
+fn a_stencil_replaces_what_is_under_it_in_every_pass() {
+    let stencil = render(NON_ISOLATED, "/D gs 1 1 1 rg 40 40 120 120 re f /N gs 0 0 1 rg q 120 0 0 120 40 40 cm /St Do Q");
+    check(&stencil, 60, 100, [0, 0, 255], "stencil over the square");
+    check(&stencil, 140, 100, GREEN, "square beside the stencil");
+}
+
+/// A shading that paints nothing (a mesh off the page) replaces nothing in another pass either.
+#[test]
+fn a_shading_that_paints_nothing_clears_nothing_in_another_pass() {
+    let mesh = render(NON_ISOLATED, "0 0 1 rg 20 20 160 160 re f /D gs /Sh0 sh");
+    check(&mesh, 100, 100, [0, 0, 255], "blue under an off-page Difference mesh");
+}

vendor/hayro/src/renderer.rs

@@ -1130,6 +1130,25 @@ impl Renderer {
         }
     }
 
+    /// The device-space area a shading painted on `path` covers: the path's bounding box, with
+    /// the stroke width around it for a stroke, on the canvas.
+    fn shading_bbox(&self, path: &BezPath, is_stroke: bool) -> Rect {
+        let path_transform = self.ctx.transform();
+        let mut bbox = (*path_transform * path.clone()).bounding_box();
+        if is_stroke {
+            // Try to account for stroke in bbox.
+            let (a1, a2) = x_y_advances(path_transform);
+            let factor = a1.length().max(a2.length()) * self.ctx.stroke().width;
+            bbox = bbox.inflate(factor, factor);
+        }
+        bbox.intersect(Rect::new(
+            0.0,
+            0.0,
+            self.ctx.width() as f64,
+            self.ctx.height() as f64,
+        ))
+    }
+
     #[must_use]
     fn set_paint(&mut self, paint: &Paint<'_>, path: &BezPath, is_stroke: bool) -> PaintSetup {
         // PdfCraft patch (22): in a knockout erase pass every paint clears what is under its
@@ -1138,16 +1157,25 @@ impl Renderer {
         let other_pass =
             !self.erasing && element_mode.is_some_and(|mode| self.knockout_pass_skips(mode));
         if self.erasing || other_pass {
-            self.ctx.set_blend_mode(knockout_blend_mode());
-            self.ctx.set_paint(AlphaColor::<Srgb>::new([0.0, 0.0, 0.0, 0.0]));
-            // A shading's `/BBox` clips what it paints, so it clips what it clears too.
-            return PaintSetup::Clip(match paint {
+            // A shading's `/BBox` clips what it paints, so it clips what it clears too, and a
+            // mesh that paints nothing (below) clears nothing.
+            let clip = match paint {
                 Paint::Pattern(p) => match p.as_ref() {
-                    Pattern::Shading(s) => s.shading.clip_path.clone(),
+                    Pattern::Shading(s) => {
+                        let bbox = self.shading_bbox(path, is_stroke);
+                        let encoded = s.encode_within(Some(bbox.inflate(1.0, 1.0)));
+                        if mesh_area(bbox, &encoded).is_none() {
+                            return PaintSetup::Nothing;
+                        }
+                        s.shading.clip_path.clone()
+                    }
                     _ => None,
                 },
                 _ => None,
-            });
+            };
+            self.ctx.set_blend_mode(knockout_blend_mode());
+            self.ctx.set_paint(AlphaColor::<Srgb>::new([0.0, 0.0, 0.0, 0.0]));
+            return PaintSetup::Clip(clip);
         }
         let mut paint_transform = Affine::IDENTITY;
         let mut clip_path = None;
@@ -1166,21 +1194,7 @@ impl Renderer {
                 match *p {
                     Pattern::Shading(s) => {
                         clip_path = s.shading.clip_path.clone();
-                        let mut bbox = (*path_transform * path.clone()).bounding_box();
-
-                        if is_stroke {
-                            // Try to account for stroke in bbox.
-                            let (a1, a2) = x_y_advances(path_transform);
-                            let factor = a1.length().max(a2.length()) * self.ctx.stroke().width;
-                            bbox = bbox.inflate(factor, factor);
-                        }
-
-                        bbox = bbox.intersect(Rect::new(
-                            0.0,
-                            0.0,
-                            self.ctx.width() as f64,
-                            self.ctx.height() as f64,
-                        ));
+                        let mut bbox = self.shading_bbox(path, is_stroke);
 
                         // PdfCraft patch: mesh shadings are sampled only inside `bbox` (and a
                         // pixel around it: texels at a fractional edge look one pixel further).
@@ -1193,18 +1207,11 @@ impl Renderer {
                         // now covers the triangles alone (a pixel around them, which stays
                         // transparent and is what `Extend::Pad` repeats), and a mesh with no
                         // triangle on the canvas paints nothing at all.
-                        if let Some(extent) = encoded.extent {
-                            // An empty extent (no triangle in the area) is `Rect::ZERO`, which
-                            // sits at the canvas origin: test it before intersecting, or a
-                            // 1 × 1 transparent texture would still be painted over the whole
-                            // path (310 of them cost 230 ms per tile).
-                            if extent.width() <= 0.0 || extent.height() <= 0.0 {
+                        if encoded.extent.is_some() {
+                            let Some(area) = mesh_area(bbox, &encoded) else {
                                 return PaintSetup::Nothing;
-                            }
-                            bbox = bbox.intersect(extent.inflate(1.0, 1.0));
-                            if bbox.width() <= 0.0 || bbox.height() <= 0.0 {
-                                return PaintSetup::Nothing;
-                            }
+                            };
+                            bbox = area;
                             extend = peniko::Extend::Pad;
                         }
 
@@ -1745,12 +1752,14 @@ impl<'a> Device<'a> for Renderer {
             self.erase_groups += 1;
             return;
         }
-        // PdfCraft patch (26): an image or group of another pass than the current one composites
-        // nothing; what it covers was cleared before it (`knockout_image`, the erase pass).
-        let opacity = if self.knockout_pass_skips(blend_mode) {
-            0.0
+        // PdfCraft patch (26): an image or group of another pass than the current one clears
+        // what is under its shape instead of painting: its layer, drawn at full opacity with no
+        // soft mask, composites with `Compose::DestOut`. That clears the coverage of a masked
+        // image's or a stencil's mask too, which `knockout_image` leaves alone.
+        let (opacity, mask, layer_blend) = if self.knockout_pass_skips(blend_mode) {
+            (1.0, None, peniko::BlendMode::new(Mix::Normal, Compose::DestOut))
         } else {
-            opacity
+            (opacity, mask, convert_blend_mode(blend_mode))
         };
         let settings = *self.ctx.render_settings();
         let force_images_may_have_transparency = mask.is_some()
@@ -1762,7 +1771,7 @@ impl<'a> Device<'a> for Renderer {
                 .unwrap_or(false);
         self.ctx.push_layer(
             None,
-            Some(convert_blend_mode(blend_mode)),
+            Some(layer_blend),
             Some(opacity),
             // TODO: Deduplicate
             mask.map(|m| {
@@ -2207,6 +2216,22 @@ fn knockout_blend_mode() -> peniko::BlendMode {
     peniko::BlendMode::new(Mix::Normal, Compose::Copy)
 }
 
+/// PdfCraft patch (19): the part of `bbox` a shading encoded within it paints. A mesh paints
+/// only around its triangles, and nothing (`None`) when none is in `bbox`; its empty extent is
+/// `Rect::ZERO`, at the canvas origin, so it is tested before intersecting, or a 1 × 1
+/// transparent texture would still be painted over the whole path (310 of them cost 230 ms per
+/// tile). Any other shading paints all of `bbox`.
+fn mesh_area(bbox: Rect, encoded: &EncodedShadingPattern) -> Option<Rect> {
+    let Some(extent) = encoded.extent else {
+        return Some(bbox);
+    };
+    if extent.width() <= 0.0 || extent.height() <= 0.0 {
+        return None;
+    }
+    let area = bbox.intersect(extent.inflate(1.0, 1.0));
+    (area.width() > 0.0 && area.height() > 0.0).then_some(area)
+}
+
 fn convert_blend_mode(blend_mode: BlendMode) -> peniko::BlendMode {
     let mix = match blend_mode {
         BlendMode::Normal => Mix::Normal,