hayro-syntax: cap the nesting of direct arrays and dictionaries (#772)

Skipping a direct array or dictionary, and the lenient repair of a
dictionary that has an object where a key belongs, recursed once per
level with no bound, so deep nesting in a trailer entry or a content
stream overflowed the stack. Count the depth on the Reader and fail
beyond 128 levels like any other malformed object.

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: Brandon Thomas <echelon@gmail.com>

crates/render/src/raster.rs

@@ -1661,7 +1661,11 @@ mod tests {
         let object = Object::from_bytes(data.as_bytes()).unwrap();
         assert_eq!(Function::new(&object).unwrap().eval([0.5].into_iter().collect()).unwrap().as_slice(), &[0.5]);
         let over = format!("<< /FunctionType 3 /Domain [0 1] /Functions [{data}] /Bounds [] /Encode [0 1] >>");
-        assert!(Function::new(&Object::from_bytes(over.as_bytes()).unwrap()).is_none());
+        // The 64 functions above nest exactly 128 levels (each dictionary and its /Functions
+        // array, then the leaf's /Domain array), the parser's nesting cap. 65 nest 130, so the
+        // parser refuses them before the function-depth guard sees them (a refusal either way).
+        // `function_limits_depth_guard_refuses_a_chain_of_references` tests the guard itself.
+        assert!(Object::from_bytes(over.as_bytes()).is_none(), "the parser refuses direct nesting past its cap");
 
         // One root plus 10,000 leaves is one node past the common budget. This input remains
         // below a megabyte and does not attempt excessive recursion or an allocation failure.
@@ -1677,6 +1681,43 @@ mod tests {
             assert_eq!(Function::new(&object).is_some(), children == 9999);
         }
     }
+    /// Stitching functions that each name the next through `N 0 R` nest no direct objects, so
+    /// the parser's nesting cap doesn't stop them and the function-depth guard does.
+    #[test]
+    fn function_limits_depth_guard_refuses_a_chain_of_references() {
+        use hayro::hayro_interpret::Function;
+        use hayro_syntax::object::{Object, ObjectIdentifier};
+        for functions in [64, 65] {
+            // Objects 4 onwards: each stitching function names the next one; the last is a leaf.
+            let objects: String = (4..4 + functions)
+                .map(|number| {
+                    let body = if number < 3 + functions {
+                        format!("<< /FunctionType 3 /Domain [0 1] /Functions [{} 0 R] /Bounds [] /Encode [0 1] >>", number + 1)
+                    } else {
+                        "<< /FunctionType 2 /Domain [0 1] /C0 [0] /C1 [1] /N 1 >>".to_owned()
+                    };
+                    format!("{number} 0 obj {body} endobj\n")
+                })
+                .collect();
+            let parsed = hayro_syntax::Pdf::new(
+                format!(
+                    "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\
+                     2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\
+                     3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 20 20] >> endobj\n\
+                     {objects}trailer << /Root 1 0 R >>\n%%EOF"
+                )
+                .into_bytes(),
+            )
+            .unwrap();
+            let object = parsed.xref().get::<Object<'_>>(ObjectIdentifier::new(4, 0)).unwrap();
+            let function = Function::new(&object);
+            if functions == 64 {
+                assert_eq!(function.unwrap().eval([0.5].into_iter().collect()).unwrap().as_slice(), &[0.5]);
+            } else {
+                assert!(function.is_none(), "65 functions are one past the function-depth guard");
+            }
+        }
+    }
     #[test]
     fn function_limits_stitching_cycles_are_refused_and_shared_children_work() {
         use hayro::hayro_interpret::Function;
@@ -3353,6 +3394,222 @@ trailer << /Root 1 0 R >>
         assert_eq!(&page.rgba[((38 * 40 + 1) * 4)..][..4], &[255, 0, 0, 255], "the rest of the page draws");
     }
 
+    /// `depth` direct arrays (`array`), dictionaries with a key (`dict`) or both in turn (`mixed`)
+    /// around `null`.
+    fn nested(depth: usize, kind: &str) -> String {
+        let is_dict = |level| kind == "dict" || (kind == "mixed" && level % 2 == 0);
+        let mut value = String::new();
+        for level in 0..depth {
+            value.push_str(if is_dict(level) { "<< /N " } else { "[" });
+        }
+        value.push_str("null");
+        for level in (0..depth).rev() {
+            value.push_str(if is_dict(level) { " >>" } else { "]" });
+        }
+        value
+    }
+
+    /// `depth` dictionaries nested without keys (`<< << >> >>`), closed or not.
+    fn keyless(depth: usize, closed: bool) -> String {
+        let mut value = "<< ".repeat(depth);
+        if closed {
+            value.push_str(&">> ".repeat(depth));
+        }
+        value
+    }
+
+    /// A 40 × 40 page that runs `content`, with a correct xref table and `trailer` at the end
+    /// of a trailer that also points at an `/Info` title.
+    fn nesting_pdf(trailer: &str, content: &str) -> Vec<u8> {
+        let mut bytes = b"%PDF-1.7\n".to_vec();
+        let mut offsets = vec![0];
+        for (index, body) in [
+            "<< /Type /Catalog /Pages 2 0 R >>".to_string(),
+            "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(),
+            "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 40 40] /Contents 4 0 R /Resources << >> >>".to_string(),
+            format!("<< /Length {} >> stream\n{content}\nendstream", content.len()),
+            "<< /Title (Nested) >>".to_string(),
+        ]
+        .iter()
+        .enumerate()
+        {
+            offsets.push(bytes.len());
+            bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", index + 1).as_bytes());
+        }
+        let xref = bytes.len();
+        bytes.extend_from_slice(b"xref\n0 6\n0000000000 65535 f \n");
+        for offset in &offsets[1..] {
+            bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
+        }
+        bytes.extend_from_slice(format!("trailer << /Size 6 /Root 1 0 R /Info 5 0 R {trailer} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes());
+        bytes
+    }
+
+    /// Opens `bytes` and renders its one page, which must render without an error.
+    fn nesting_render(bytes: Vec<u8>) -> RenderedPage {
+        let parsed = Pdf::new(bytes.clone()).expect("xref repair can recover the catalog when the trailer is malformed");
+        assert_eq!(parsed.pages().len(), 1);
+        let mut renderer = PageRenderer::new(Arc::new(bytes), RenderConfig::default());
+        let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() });
+        assert!(page.error.is_none(), "{:?}", page.error);
+        assert_eq!((page.width, page.height), (40, 40));
+        page
+    }
+
+    /// Direct arrays and dictionaries used to recurse without a bound while skipping even
+    /// unused trailer values and content operands. Exercise the vendored patch in workspace CI.
+    #[test]
+    fn deeply_nested_direct_objects_do_not_overflow_the_stack() {
+        use hayro_syntax::object::Object;
+        use hayro_syntax::reader::{Reader, ReaderExt};
+        let pdf = |unused: &str, content: &str| nesting_pdf(&format!("/Unused {unused}"), content);
+        let paint = "1 0 0 rg 0 0 4 4 re f";
+        for kind in ["array", "dict", "mixed"] {
+            // Run the crash reproducer first, so the unpatched red state is a stack overflow.
+            let page = nesting_render(pdf(&nested(100_000, kind), paint));
+            assert_eq!(&page.rgba[((38 * 40 + 1) * 4)..][..4], &[255, 0, 0, 255], "{kind}: repaired trailer");
+            for depth in [20, 128, 129] {
+                let value = nested(depth, kind);
+                let siblings = format!("{value} [] []");
+                let mut reader = Reader::new(siblings.as_bytes());
+                assert_eq!(reader.skip::<Object<'_>>(false).is_some(), depth <= 128, "{kind}: depth {depth}");
+                if depth > 128 {
+                    assert_eq!(reader.offset(), 0, "failed skips restore the offset");
+                    reader.jump(value.len());
+                }
+                // A failed skip must unwind its depth; successful siblings must not accumulate it.
+                for _ in 0..2 {
+                    reader.skip_white_spaces();
+                    assert!(reader.skip::<Object<'_>>(false).is_some());
+                }
+                let page = nesting_render(pdf(&value, paint));
+                assert_eq!(&page.rgba[((38 * 40 + 1) * 4)..][..4], &[255, 0, 0, 255], "{kind}: depth {depth}");
+            }
+            // Many siblings exercise the offset constructor and ensure depth is per nesting.
+            let siblings = "[] ".repeat(256);
+            let mut reader = Reader::new_with(siblings.as_bytes(), 0);
+            for _ in 0..256 {
+                assert!(reader.skip::<Object<'_>>(false).is_some());
+                reader.skip_white_spaces();
+            }
+        }
+        // Content-stream arrays take the Object path, rather than MaybeRef<Object>.
+        for depth in [100_000, 20, 128, 129] {
+            let value = nested(depth, "array");
+            let mut reader = Reader::new(value.as_bytes());
+            assert_eq!(reader.skip::<Object<'_>>(true).is_some(), depth <= 128, "content: depth {depth}");
+            let content = format!("{value} discard {paint}");
+            let page = nesting_render(pdf("null", &content));
+            if depth <= 128 {
+                assert_eq!(&page.rgba[((38 * 40 + 1) * 4)..][..4], &[255, 0, 0, 255], "content: depth {depth}");
+            }
+        }
+    }
+
+    /// A dictionary with an object where a key belongs is repaired by reading that object, not
+    /// skipping it, and that read recursed without a bound when the object was another such
+    /// dictionary: `<<` 100,000 times aborted the process. Here as a content-stream operand.
+    #[test]
+    fn dictionaries_nested_without_keys_in_content_do_not_overflow_the_stack() {
+        for closed in [true, false] {
+            let content = format!("1 0 0 rg 0 0 4 4 re f {} discard", keyless(100_000, closed));
+            let page = nesting_render(nesting_pdf("", &content));
+            assert_eq!(px40(&page, 1, 38), &[255, 0, 0, 255], "closed: {closed}");
+        }
+    }
+
+    /// The same, where a key of the trailer belongs: the xref table can't be used, and repair
+    /// finds the catalog.
+    #[test]
+    fn dictionaries_nested_without_keys_in_the_trailer_do_not_overflow_the_stack() {
+        for closed in [true, false] {
+            let page = nesting_render(nesting_pdf(&keyless(100_000, closed), "1 0 0 rg 0 0 4 4 re f"));
+            assert_eq!(px40(&page, 1, 38), &[255, 0, 0, 255], "closed: {closed}");
+        }
+    }
+
+    /// The same as the properties of marked content (`/Tag <<<<<< … BDC`).
+    #[test]
+    fn dictionaries_nested_without_keys_as_marked_content_properties_do_not_overflow_the_stack() {
+        for closed in [true, false] {
+            let properties = if closed { "<<".repeat(100_000) + &">>".repeat(100_000) } else { "<<".repeat(100_000) };
+            let content = format!("1 0 0 rg 0 0 4 4 re f /Tag {properties} BDC 0 0 1 rg 36 36 4 4 re f EMC");
+            let page = nesting_render(nesting_pdf("", &content));
+            assert_eq!(px40(&page, 1, 38), &[255, 0, 0, 255], "closed: {closed}");
+        }
+    }
+
+    /// Through the same three paths, dictionaries without keys are read up to the nesting cap
+    /// (128 dictionaries in all) and refused one past it.
+    #[test]
+    fn dictionaries_nested_without_keys_are_read_up_to_the_nesting_cap() {
+        use hayro_syntax::object::{FromBytes, Object};
+        use hayro_syntax::reader::{Reader, ReaderExt};
+        let (red, blue, white) = ([255, 0, 0, 255], [0, 0, 255, 255], [255, 255, 255, 255]);
+        for depth in [128, 129] {
+            let within = depth <= 128;
+            let value = keyless(depth, true);
+            assert_eq!(Object::from_bytes(value.as_bytes()).is_some(), within, "object: {depth}");
+            assert_eq!(Reader::new(value.as_bytes()).read_without_context::<Object<'_>>().is_some(), within, "operand: {depth}");
+            // Content reading stops at an operand it can't read, so the blue square is drawn
+            // only when the dictionaries were read.
+            let properties = "<<".repeat(depth) + &">>".repeat(depth);
+            for (path, operand) in [("operand", format!("{value} discard")), ("BDC", format!("/Tag {properties} BDC"))] {
+                let content = format!("1 0 0 rg 0 0 4 4 re f {operand} 0 0 1 rg 36 36 4 4 re f");
+                let page = nesting_render(nesting_pdf("", &content));
+                assert_eq!(px40(&page, 1, 38), &red, "{path}: {depth}");
+                assert_eq!(px40(&page, 38, 1), if within { &blue } else { &white }, "{path}: {depth}");
+            }
+            // The trailer is the outermost dictionary. Within the cap it is read (with its
+            // /Info); past it, repair recovers the catalog but not the trailer.
+            let bytes = nesting_pdf(&keyless(depth - 1, true), "1 0 0 rg 0 0 4 4 re f");
+            let parsed = Pdf::new(bytes.clone()).expect("the trailer or repair finds the catalog");
+            assert_eq!(parsed.metadata().title, within.then(|| b"Nested".to_vec()), "trailer: {depth}");
+            assert_eq!(px40(&nesting_render(bytes), 1, 38), &red, "trailer: {depth}");
+        }
+    }
+
+    /// The cap must also fit small stacks: skip, read, iterate down with the array iterators and
+    /// format (`{:?}`) the deepest nesting it admits, of arrays, dictionaries, both and
+    /// dictionaries without keys, on a 512 KiB thread in a debug build. Measured when this was
+    /// written (macOS arm64, dev profile): at most about 100 KiB, for `{:?}` of 128 dictionaries.
+    #[cfg(not(target_arch = "wasm32"))]
+    #[test]
+    fn nesting_at_the_cap_fits_a_512_kib_stack() {
+        use hayro_syntax::object::{FromBytes, Object};
+        use hayro_syntax::reader::{Reader, ReaderExt};
+        let worker = std::thread::Builder::new().stack_size(512 * 1024).spawn(|| {
+            // (kind, arrays and dictionaries `{:?}` shows, levels walked down to the end)
+            for (kind, arrays, dicts, levels) in [("array", 128, 0, 129), ("dict", 0, 128, 129), ("mixed", 64, 64, 129), ("keyless", 0, 1, 1)] {
+                let value = if kind == "keyless" { keyless(128, true) } else { nested(128, kind) };
+                assert!(Reader::new(value.as_bytes()).skip::<Object<'_>>(false).is_some(), "{kind}: skip");
+                assert!(Reader::new(value.as_bytes()).skip::<Object<'_>>(true).is_some(), "{kind}: skip in content");
+                assert!(Reader::new(value.as_bytes()).read_without_context::<Object<'_>>().is_some(), "{kind}: operand");
+                let object = Object::from_bytes(value.as_bytes()).expect("the cap admits 128 levels");
+                let shown = format!("{object:?}");
+                assert_eq!((shown.matches("Array(").count(), shown.matches("Dict(").count()), (arrays, dicts), "{kind}: {{:?}}");
+                let (mut walked, mut next) = (0, Some(object));
+                while let Some(object) = next.take() {
+                    walked += 1;
+                    next = match object {
+                        Object::Array(array) => {
+                            assert_eq!(array.raw_iter().count(), 1, "{kind}: raw_iter");
+                            assert!(array.flex_iter().next::<Object<'_>>().is_some(), "{kind}: flex_iter");
+                            array.iter::<Object<'_>>().next()
+                        }
+                        Object::Dict(dict) => dict.get::<Object<'_>>("N"),
+                        _ => None,
+                    };
+                }
+                assert_eq!(walked, levels, "{kind}: iterate");
+            }
+        });
+        // A stack overflow aborts the process; a failed check is a panic, shown as it is.
+        if let Err(panic) = worker.expect("spawn a 512 KiB thread").join() {
+            std::panic::resume_unwind(panic);
+        }
+    }
+
     /// From the nightly `cargo xtask fuzz`: many small inline images, each with "EI" (followed
     /// by a space) inside its data, and no text anywhere. To decide whether such an "EI" ends the
     /// data, the parser re-read the rest of the stream as content, and that re-read did the same

vendor/hayro-interpret/src/function/type3.rs

@@ -174,8 +174,10 @@ mod tests {
             Some(smallvec![0.5])
         );
         let data = function_limits_stitching(65);
-        let object = Object::from_bytes(data.as_bytes()).unwrap();
-        assert!(Function::new(&object).is_none());
+        // Direct nesting beyond the hayro-syntax cap (PdfCraft patch) is refused while parsing,
+        // before this guard; the guard is tested with indirect references in pdfcraft-render
+        // (`function_limits_depth_guard_refuses_a_chain_of_references`).
+        assert!(Object::from_bytes(data.as_bytes()).is_none());
     }
 
     fn function_limits_fanout(children: usize) -> String {

vendor/hayro-syntax/src/byte_reader.rs

@@ -2,6 +2,10 @@
 
 use core::ops::Range;
 
+/// PdfCraft patch: how deeply direct arrays and dictionaries may nest (the same cap as `pdfcraft-cos`; a
+/// chain of 64 direct stitching functions is exactly this deep).
+const MAX_CONTAINER_DEPTH: usize = 128;
+
 /// A reader for reading bytes and PDF objects.
 #[derive(Clone, Debug)]
 pub struct Reader<'a> {
@@ -9,19 +13,34 @@ pub struct Reader<'a> {
     pub data: &'a [u8],
     /// The current byte-offset.
     pub offset: usize,
+    // PdfCraft patch: bound direct-object nesting while skipping arrays and dictionaries
+    // and while parsing dictionaries (`Dict::skip`, `Array::skip`, `parse_dict_with`).
+    skip_depth: usize,
 }
 
 impl<'a> Reader<'a> {
     /// Create a new reader.
     #[inline]
     pub fn new(data: &'a [u8]) -> Self {
-        Self { data, offset: 0 }
+        Self { data, offset: 0, skip_depth: 0 }
     }
 
     /// Create a new reader at the given offset.
     #[inline]
     pub fn new_with(data: &'a [u8], offset: usize) -> Self {
-        Self { data, offset }
+        Self { data, offset, skip_depth: 0 }
+    }
+
+    pub(crate) fn enter_container(&mut self) -> Option<()> {
+        if self.skip_depth >= MAX_CONTAINER_DEPTH {
+            return None;
+        }
+        self.skip_depth += 1;
+        Some(())
+    }
+
+    pub(crate) fn leave_container(&mut self) {
+        self.skip_depth -= 1;
     }
 
     /// Returns `true` if the reader has reached the end of the data.

vendor/hayro-syntax/src/object/array.rs

@@ -71,8 +71,10 @@ object!(Array<'a>, Array);
 impl Skippable for Array<'_> {
     fn skip(r: &mut Reader<'_>, is_content_stream: bool) -> Option<()> {
         r.forward_tag(b"[")?;
+        r.enter_container()?;
 
-        loop {
+        // PdfCraft patch: leave even when an element's skip fails through `?`.
+        let result = (|| loop {
             r.skip_white_spaces_and_comments();
 
             if let Some(()) = r.forward_tag(b"]") {
@@ -82,7 +84,9 @@ impl Skippable for Array<'_> {
             } else {
                 r.skip::<MaybeRef<Object<'_>>>(false)?;
             }
-        }
+        })();
+        r.leave_container();
+        result
     }
 }
 

vendor/hayro-syntax/src/object/dict.rs

@@ -177,8 +177,10 @@ impl Debug for Dict<'_> {
 impl Skippable for Dict<'_> {
     fn skip(r: &mut Reader<'_>, is_content_stream: bool) -> Option<()> {
         r.forward_tag(b"<<")?;
+        r.enter_container()?;
 
-        loop {
+        // PdfCraft patch: leave even when a value or garbage fails through `?`.
+        let result = (|| loop {
             r.skip_white_spaces_and_comments();
 
             if let Some(()) = r.forward_tag(b">>") {
@@ -198,7 +200,9 @@ impl Skippable for Dict<'_> {
                     r.skip::<MaybeRef<Object<'_>>>(is_content_stream)?;
                 }
             }
-        }
+        })();
+        r.leave_container();
+        result
     }
 }
 
@@ -362,7 +366,26 @@ pub(crate) fn probe_dict<'a>(
     })
 }
 
+// PdfCraft patch: an object where a key belongs is read, not skipped, so a dictionary
+// holding another one there (`<< << …`) recursed without a bound. Count each parsed
+// dictionary like a skipped one, and leave on every exit of the parse below.
 fn parse_dict_with<'a, F>(
+    r: &mut Reader<'a>,
+    ctx: &ReaderContext<'a>,
+    start_tag: Option<&[u8]>,
+    end_tag: &[u8],
+    on_entry: F,
+) -> Option<(&'a [u8], bool)>
+where
+    F: FnMut(Name<'a>, usize, &Reader<'a>) -> Option<()>,
+{
+    r.enter_container()?;
+    let result = parse_dict_with_inner(r, ctx, start_tag, end_tag, on_entry);
+    r.leave_container();
+    result
+}
+
+fn parse_dict_with_inner<'a, F>(
     r: &mut Reader<'a>,
     ctx: &ReaderContext<'a>,
     start_tag: Option<&[u8]>,