fix(render): a visibility expression decides what an OCMD shows (#2986)

An optional content membership dictionary may carry a visibility
expression (/VE, PDF 1.6): [/And e...], [/Or e...] or [/Not e], each e an
optional content group or a nested expression. It takes precedence
over /OCGs and /P (ISO 32000-2 8.11.2.2). hayro-interpret ignored it, so
content under such a dictionary stayed visible when its layers were
hidden in the Layers panel.

Vendored hayro-interpret patch (40) evaluates the expression, nested
up to 32 deep. A malformed expression (unknown operator, /Not without
exactly one operand, /And or /Or without any, an operand that is
neither a group nor an expression, too deep) falls back to /OCGs and
/P as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RUNQSPVXx3dDBabRZHAnjz

crates/render/src/raster.rs

@@ -3315,6 +3315,58 @@ trailer << /Root 1 0 R >>
         assert_eq!(px(off), vec![255, 255, 255, 255], "toggled-off layer must not draw");
     }
 
+    /// #2986: an optional content membership dictionary with a visibility expression (`/VE`,
+    /// ISO 32000-2 §8.11.2.2) shows its content only when the expression is true, and the
+    /// expression takes precedence over `/OCGs` and `/P`. It was ignored, so content under it
+    /// stayed visible when its layers were hidden. A malformed expression falls back to `/OCGs`.
+    #[test]
+    fn visibility_expressions_decide_what_a_membership_dictionary_shows() {
+        let render = |ocmd: &str, off: &[i32]| {
+            let pdf = format!(
+                "%PDF-1.7
+1 0 obj << /Type /Catalog /Pages 2 0 R /OCProperties << /OCGs [5 0 R 6 0 R] /D << /Order [5 0 R 6 0 R] >> >> >> endobj
+2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj
+3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Contents 4 0 R /Resources << /Properties << /M1 7 0 R >> >> >> endobj
+4 0 obj << /Length 44 >> stream
+/OC /M1 BDC 1 0 0 rg 0 0 100 100 re f EMC
+endstream endobj
+5 0 obj << /Type /OCG /Name (A) >> endobj
+6 0 obj << /Type /OCG /Name (B) >> endobj
+7 0 obj {ocmd} endobj
+trailer << /Root 1 0 R >>
+%%EOF"
+            );
+            let layers = off.iter().map(|n| (*n, 0, false)).collect();
+            let mut r = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig { layers: Arc::new(layers), ..Default::default() });
+            let p = r.render(RenderRequest { page: 0, kind: RequestKind::Pixels, tile: None, scale: 0.5, tag: 0, ..Default::default() });
+            p.rgba[..4] == [255, 0, 0, 255]
+        };
+        let and = "<< /Type /OCMD /VE [/And 5 0 R 6 0 R] >>";
+        assert!(render(and, &[]), "And: both on");
+        assert!(!render(and, &[5]), "And: one off");
+        let or = "<< /Type /OCMD /VE [/Or 5 0 R 6 0 R] >>";
+        assert!(render(or, &[5]), "Or: one on");
+        assert!(!render(or, &[5, 6]), "Or: both off");
+        let not = "<< /Type /OCMD /VE [/Not 5 0 R] >>";
+        assert!(!render(not, &[]), "Not: the group is on");
+        assert!(render(not, &[5]), "Not: the group is off");
+        let nested = "<< /Type /OCMD /VE [/And [/Not 5 0 R] 6 0 R] >>";
+        assert!(render(nested, &[5]), "nested: A off, B on");
+        assert!(!render(nested, &[]), "nested: A on");
+        // /VE wins over /OCGs and /P.
+        let both = "<< /Type /OCMD /OCGs [5 0 R] /P /AllOn /VE [/Not 5 0 R] >>";
+        assert!(render(both, &[5]), "the expression, not /P, decides");
+        // A malformed expression falls back to /OCGs and /P.
+        for bad in ["[/Xor 5 0 R]", "[/Not 5 0 R 6 0 R]", "[/And]", "[5 0 R]", "[/And (5)]"] {
+            let ocmd = format!("<< /Type /OCMD /OCGs [5 0 R] /VE {bad} >>");
+            assert!(render(&ocmd, &[]), "{bad}: /OCGs on");
+            assert!(!render(&ocmd, &[5]), "{bad}: /OCGs off");
+        }
+        // Nesting past the limit is malformed too, and does not recurse without end.
+        let deep = format!("<< /Type /OCMD /OCGs [5 0 R] /VE {}5 0 R{} >>", "[/Not ".repeat(41), "]".repeat(41));
+        assert!(!render(&deep, &[5]), "too deep: /OCGs decides");
+    }
+
     /// From `cargo xtask fuzz`: a tiling pattern with no /Resources inherits the page's, where
     /// its own name points back at it. It painted itself until the stack overflowed (vendored
     /// hayro-interpret patch: `MAX_PAINT_NESTING`). Ordinary patterns must still paint.

vendor/hayro-interpret/src/ocg.rs

@@ -1,5 +1,5 @@
-use hayro_syntax::object::dict::keys::{BASE_STATE, D, OCGS, OCMD, OCPROPERTIES, OFF, ON, P, TYPE};
-use hayro_syntax::object::{Array, Dict, Name, ObjectIdentifier};
+use hayro_syntax::object::dict::keys::{BASE_STATE, D, OCGS, OCMD, OCPROPERTIES, OFF, ON, P, TYPE, VE};
+use hayro_syntax::object::{Array, Dict, MaybeRef, Name, Object, ObjectIdentifier};
 use std::collections::HashSet;
 
 pub(crate) struct OcgState {
@@ -72,6 +72,14 @@ impl OcgState {
     }
 
     pub(crate) fn begin_ocmd(&mut self, ocmd: &Dict<'_>) {
+        // PdfCraft patch (40): a visibility expression takes precedence over /OCGs and /P
+        // (ISO 32000-2 §8.11.2.2); a malformed one is ignored and they decide instead.
+        if let Some(visible) = ocmd.get::<Array<'_>>(VE).and_then(|ve| self.visibility_expression(&ve, 0)) {
+            let visible = self.is_visible() && visible;
+            self.visibility_stack.push(visible);
+            return;
+        }
+
         let policy = ocmd
             .get::<Name<'_>>(P)
             .and_then(|n| OcmdPolicy::from_name(n.as_ref()))
@@ -104,6 +112,39 @@ impl OcgState {
         self.visibility_stack.push(visible);
     }
 
+    /// PdfCraft patch (40): evaluate a visibility expression (`/VE`, PDF 1.6): `[/And e…]`,
+    /// `[/Or e…]` or `[/Not e]`, each `e` an optional content group (by reference) or a nested
+    /// expression. `None` for a malformed expression: an unknown operator, a `/Not` without
+    /// exactly one operand, an `/And` or `/Or` without any, an operand that is neither, or
+    /// nesting deeper than `MAX_VISIBILITY_EXPRESSION_DEPTH`.
+    fn visibility_expression(&self, expr: &Array<'_>, depth: usize) -> Option<bool> {
+        if depth > MAX_VISIBILITY_EXPRESSION_DEPTH {
+            return None;
+        }
+        let mut items = expr.raw_iter();
+        let operator = match items.next()? {
+            MaybeRef::NotRef(Object::Name(name)) => name,
+            _ => return None,
+        };
+        let (mut count, mut all, mut any) = (0usize, true, false);
+        for item in items {
+            let value = match item {
+                MaybeRef::Ref(r) => !self.inactive_ocgs.contains(&r.into()),
+                MaybeRef::NotRef(Object::Array(nested)) => self.visibility_expression(&nested, depth + 1)?,
+                _ => return None,
+            };
+            count = count.saturating_add(1);
+            all &= value;
+            any |= value;
+        }
+        match (operator.as_ref(), count) {
+            (b"And", 1..) => Some(all),
+            (b"Or", 1..) => Some(any),
+            (b"Not", 1) => Some(!all),
+            _ => None,
+        }
+    }
+
     pub(crate) fn begin_ocg(&mut self, props: &Dict<'_>, ref_id: ObjectIdentifier) {
         match props.get::<Name<'_>>(TYPE).as_deref() {
             Some(OCMD) => self.begin_ocmd(props),
@@ -140,6 +181,9 @@ impl Default for OcgState {
     }
 }
 
+/// PdfCraft patch (40): how deep visibility expressions may nest; deeper ones are malformed.
+const MAX_VISIBILITY_EXPRESSION_DEPTH: usize = 32;
+
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
 enum BaseState {
     On,