PDF: preserve catalogs and full references in layer variants

ROADMAP.md

@@ -11,6 +11,10 @@
 
 DesignCraft aims at full Adobe InDesign parity — and to be better: faster, open (documented JSON format + IDML), scriptable by agents (MCP), and available on the web.
 
+- **2026-10-23** · PDF layer visibility variants retain catalog/configuration entries and full
+  object references; bounded incremental updates replace the existing catalog. Synthetic
+  regression tests cover metadata, output-intent references and nonzero generations. No readiness percentage change.
+
 ## Status (2026-10-01)
 
 **Working today**

crates/engine/src/cmd/pdflayers.rs

@@ -63,31 +63,84 @@ pub(crate) mod tests {
 
     /// A one-page PDF (100×100 pt) with two layers: "Blue" (left half) and "Red" (right half).
     pub(crate) fn two_layer_pdf() -> Vec<u8> {
+        two_layer_pdf_generation(0)
+    }
+
+    fn two_layer_pdf_generation(generation: i32) -> Vec<u8> {
         let content = "/OC /a BDC 0 0 1 rg 0 0 50 100 re f EMC /OC /b BDC 1 0 0 rg 50 0 50 100 re f EMC";
         let objs = [
-            "<</Type/Catalog/Pages 2 0 R/OCProperties<</OCGs[5 0 R 6 0 R]/D<</Order[5 0 R 6 0 R]>>>>>>".to_string(),
+            "<</Type/Catalog/Lang(en-GB)/Metadata 7 0 R/OutputIntents[8 0 R]/Pages 2 0 R/OCProperties<</OCGs[5 0 R 6 0 R]/D<</Order[5 0 R 6 0 R]/Name(Default)/Locked[5 0 R]>>>>>>".to_string(),
             "<</Type/Pages/Kids[3 0 R]/Count 1>>".to_string(),
             "<</Type/Page/Parent 2 0 R/MediaBox[0 0 100 100]/Resources<</Properties<</a 5 0 R/b 6 0 R>>>>/Contents 4 0 R>>".to_string(),
             format!("<</Length {}>>\nstream\n{content}\nendstream", content.len()),
             "<</Type/OCG/Name(Blue)>>".to_string(),
             "<</Type/OCG/Name(Red)>>".to_string(),
+            "<</Type/Metadata/Subtype/XML/Length 4>>\nstream\n<x/>\nendstream".to_string(),
+            "<</Type/OutputIntent/S/GTS_PDFX/OutputConditionIdentifier(Test)>>".to_string(),
         ];
         let mut out = b"%PDF-1.7\n".to_vec();
         let mut offs = Vec::new();
         for (i, o) in objs.iter().enumerate() {
             offs.push(out.len());
-            out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes());
+            out.extend_from_slice(format!("{} {generation} obj\n{}\nendobj\n", i + 1, o.replace(" 0 R", &format!(" {generation} R"))).as_bytes());
         }
         let x = out.len();
         let mut t = format!("xref\n0 {}\n0000000000 65535 f\r\n", objs.len() + 1);
         for o in offs {
-            t.push_str(&format!("{o:010} 00000 n\r\n"));
+            t.push_str(&format!("{o:010} {generation:05} n\r\n"));
         }
-        t.push_str(&format!("trailer\n<</Size {}/Root 1 0 R>>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1));
+        t.push_str(&format!("trailer\n<</Size {}/Root 1 {generation} R>>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1));
         out.extend_from_slice(t.as_bytes());
         out
     }
 
+    #[test]
+    fn pdf_layer_update_preserves_catalog_configuration_and_generations() {
+        for generation in [0, 1] {
+            let original = two_layer_pdf_generation(generation);
+            let updated = designcraft_render::pdf_hide_layers(&original, &["Red".into()]).unwrap();
+            assert_eq!(designcraft_render::pdf_layers(&updated), vec![("Blue".into(), true), ("Red".into(), false)]);
+            let appended = std::str::from_utf8(&updated[original.len()..]).unwrap();
+            for token in [
+                "/Lang(en-GB)".to_string(),
+                format!("/Metadata 7 {generation} R"),
+                format!("/OutputIntents[8 {generation} R]"),
+                format!("/Locked[5 {generation} R]"),
+                "/Name(Default)".into(),
+                format!("/Root 1 {generation} R"),
+            ] {
+                assert!(appended.contains(&token), "missing {token}: {appended}");
+            }
+            assert!(!appended.contains("9999000"));
+            let all_on = designcraft_render::pdf_hide_layers(&updated, &[]).unwrap();
+            assert!(designcraft_render::pdf_layers(&all_on).iter().all(|(_, visible)| *visible));
+        }
+    }
+
+    #[test]
+    fn pdf_layer_update_accepts_xref_streams_and_rejects_unsupported_updates() {
+        let original = two_layer_pdf();
+        let text = std::str::from_utf8(&original).unwrap();
+        let offset: usize = text.rsplit_once("startxref").unwrap().1.split_whitespace().next().unwrap().parse().unwrap();
+        let mut stream_pdf = original[..offset].to_vec();
+        let mut entries = vec![0, 0, 0, 0, 0, 255, 255];
+        for number in 1..=9 {
+            let at = if number == 9 { offset } else { text.find(&format!("{number} 0 obj")).unwrap() };
+            entries.push(1);
+            entries.extend_from_slice(&u32::try_from(at).unwrap().to_be_bytes());
+            entries.extend_from_slice(&[0, 0]);
+        }
+        stream_pdf.extend_from_slice(format!("9 0 obj\n<</Type/XRef/Size 10/Root 1 0 R/W[1 4 2]/Length {}>>\nstream\n", entries.len()).as_bytes());
+        stream_pdf.extend_from_slice(&entries);
+        stream_pdf.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{offset}\n%%EOF\n").as_bytes());
+        let updated = designcraft_render::pdf_hide_layers(&stream_pdf, &["Red".into()]).unwrap();
+        assert_eq!(designcraft_render::pdf_layers(&updated), vec![("Blue".into(), true), ("Red".into(), false)]);
+        for unsupported in ["/Encrypt null", "/XRefStm 0"] {
+            let altered = text.replace("/Size 9", &format!("{unsupported}/Size 9"));
+            assert!(designcraft_render::pdf_hide_layers(altered.as_bytes(), &["Red".into()]).is_none());
+        }
+    }
+
     #[test]
     fn placed_pdf_layers_hide() {
         let mut s = Session::new();

crates/render/src/pdf_layers.rs

@@ -6,20 +6,17 @@ use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 
 use craft_pdf::syntax::Pdf;
-use craft_pdf::syntax::object::{Array, Dict, Name, ObjectIdentifier};
+use craft_pdf::syntax::object::{Array, Dict, MaybeRef, Name, ObjRef, Object, ObjectIdentifier};
 
-/// (object number, name, on by default) of each layer.
-fn ocgs(pdf: &Pdf) -> Vec<(i32, String, bool)> {
+/// (object reference, name, on by default) of each layer.
+fn ocgs(pdf: &Pdf) -> Vec<(ObjRef, String, bool)> {
     let xref = pdf.xref();
     let Some(cat) = xref.get::<Dict<'_>>(xref.root_id()) else { return vec![] };
     let Some(ocp) = cat.get::<Dict<'_>>(b"OCProperties") else { return vec![] };
     let Some(list) = ocp.get::<Array<'_>>(b"OCGs") else { return vec![] };
     let cfg = ocp.get::<Dict<'_>>(b"D");
-    let ids_in = |key: &[u8]| -> Vec<i32> {
-        cfg.as_ref()
-            .and_then(|c| c.get::<Array<'_>>(key))
-            .map(|a| a.raw_iter().filter_map(|o| o.as_obj_ref()).map(|r| r.obj_number).collect())
-            .unwrap_or_default()
+    let ids_in = |key: &[u8]| -> Vec<ObjRef> {
+        cfg.as_ref().and_then(|c| c.get::<Array<'_>>(key)).map(|a| a.raw_iter().filter_map(|o| o.as_obj_ref()).collect()).unwrap_or_default()
     };
     let base_off = cfg.as_ref().and_then(|c| c.get::<Name<'_>>(b"BaseState")).is_some_and(|n| n.as_ref() == b"OFF");
     let (on, off) = (ids_in(b"ON"), ids_in(b"OFF"));
@@ -31,8 +28,8 @@ fn ocgs(pdf: &Pdf) -> Vec<(i32, String, bool)> {
                 .get::<craft_pdf::syntax::object::String<'_>>(b"Name")
                 .map(|s| String::from_utf8_lossy(s.as_bytes()).to_string())
                 .unwrap_or_default();
-            let visible = if base_off { on.contains(&r.obj_number) } else { !off.contains(&r.obj_number) };
-            Some((r.obj_number, name, visible))
+            let visible = if base_off { on.contains(&r) } else { !off.contains(&r) };
+            Some((r, name, visible))
         })
         .collect()
 }
@@ -42,62 +39,179 @@ pub fn pdf_layers(bytes: &[u8]) -> Vec<(String, bool)> {
     craft_pdf::open_slice(bytes, None).map(|p| ocgs(&p).into_iter().map(|(_, n, v)| (n, v)).collect()).unwrap_or_default()
 }
 
-/// `bytes` with exactly the layers named in `hidden` off (the others on). `None` when the file
-/// has no such layers or can't be read.
+// Copy untouched dictionary entries as raw PDF tokens. In particular, do not reinterpret
+// strings, numbers or unknown extension keys. The parser bounds container nesting.
+fn patch_dict(dict: &Dict<'_>, updates: &[(&[u8], &[u8])]) -> Option<Vec<u8>> {
+    use craft_pdf::syntax::reader::{Reader, ReaderExt};
+    let data = if dict.is_empty() { b"<<>>".as_slice() } else { dict.data() };
+    if data.len() > 4 * 1024 * 1024 {
+        return None;
+    }
+    let mut reader = Reader::new(data);
+    reader.forward_tag(b"<<")?;
+    let mut out = b"<<".to_vec();
+    let mut ended = false;
+    for _ in 0..100_000 {
+        reader.skip_white_spaces_and_comments();
+        if reader.forward_tag(b">>").is_some() {
+            ended = true;
+            break;
+        }
+        let start = reader.offset();
+        let name = reader.read_without_context::<Name<'_>>()?;
+        reader.skip_white_spaces_and_comments();
+        reader.skip::<MaybeRef<Object<'_>>>(false)?;
+        if !updates.iter().any(|(key, _)| *key == name.as_ref()) {
+            out.push(b' ');
+            out.extend_from_slice(data.get(start..reader.offset())?);
+        }
+    }
+    if !ended {
+        return None;
+    }
+    for (key, value) in updates {
+        if !value.is_empty() {
+            out.extend_from_slice(b" /");
+            out.extend_from_slice(key);
+            out.push(b' ');
+            out.extend_from_slice(value);
+        }
+    }
+    out.extend_from_slice(b" >>");
+    Some(out)
+}
+
+fn trailer_at(bytes: &[u8], offset: usize) -> Option<Dict<'_>> {
+    use craft_pdf::syntax::reader::{Reader, ReaderExt};
+    let data = bytes.get(offset..)?;
+    let mut reader = Reader::new(data);
+    if data.starts_with(b"xref") {
+        let pos = data.get(..data.len().min(4 * 1024 * 1024))?.windows(7).position(|w| w == b"trailer")?;
+        reader.skip_bytes(pos.checked_add(7)?)?;
+    } else {
+        // An xref stream's dictionary also holds the trailer entries.
+        reader.read_without_context::<ObjectIdentifier>()?;
+    }
+    reader.skip_white_spaces_and_comments();
+    reader.read_without_context::<Dict<'_>>()
+}
+
+/// `bytes` with exactly the layers named in `hidden` off (the others on).
+/// Returns `None` for unreadable, encrypted, hybrid-reference or over-budget updates.
+/// Untouched catalog/configuration entries and full object references are preserved.
 pub fn pdf_hide_layers(bytes: &[u8], hidden: &[String]) -> Option<Vec<u8>> {
+    crate::check_pdf_input_size(bytes.len()).ok()?;
     let pdf = craft_pdf::open_slice(bytes, None).ok()?;
     let layers = ocgs(&pdf);
     if layers.is_empty() {
         return None;
     }
-    let xref = pdf.xref();
-    let cat = xref.get::<Dict<'_>>(xref.root_id())?;
-    let pages = cat.get_ref(b"Pages")?;
-    let all: Vec<String> = layers.iter().map(|(n, _, _)| format!("{n} 0 R")).collect();
-    let off: Vec<String> = layers.iter().filter(|(_, name, _)| hidden.contains(name)).map(|(n, _, _)| format!("{n} 0 R")).collect();
-    // A new catalog under a fresh object number, with the original's page tree.
-    let root = 9_999_000;
-    let mut out = bytes.to_vec();
-    if !out.ends_with(b"\n") {
-        out.push(b'\n');
+    let tail = String::from_utf8_lossy(bytes.get(bytes.len().saturating_sub(1024)..)?);
+    let prev = tail.rsplit_once("startxref")?.1.split_whitespace().next()?.parse::<usize>().ok()?;
+    let trailer = trailer_at(bytes, prev)?;
+    // Do not copy decrypted dictionaries into an encrypted file, or ignore a hybrid
+    // xref table. Follow bounded Prev links because encryption can be inherited.
+    let mut offset = Some(prev);
+    for _ in 0..32 {
+        let Some(at) = offset else { break };
+        let t = trailer_at(bytes, at)?;
+        if t.contains_key(b"Encrypt") || t.contains_key(b"XRefStm") {
+            return None;
+        }
+        offset = t.get::<usize>(b"Prev");
+        if offset.is_some_and(|n| n >= at) {
+            return None;
+        }
     }
+    if offset.is_some() {
+        return None;
+    }
+    let xref = pdf.xref();
+    let root = xref.root_id();
+    if root.obj_number <= 0 || !(0..=65535).contains(&root.gen_number) || trailer.get::<usize>(b"Size")? <= usize::try_from(root.obj_number).ok()? {
+        return None;
+    }
+    let cat = xref.get::<Dict<'_>>(root)?;
+    let ocp = cat.get::<Dict<'_>>(b"OCProperties")?;
+    let cfg = ocp.get::<Dict<'_>>(b"D");
+    let mut off = b"[".to_vec();
+    for (id, name, _) in &layers {
+        if hidden.contains(name) {
+            off.extend_from_slice(format!("{} {} R ", id.obj_number, id.gen_number).as_bytes());
+        }
+    }
+    off.push(b']');
+    let empty = Dict::empty();
+    let cfg = patch_dict(cfg.as_ref().unwrap_or(&empty), &[(b"BaseState", b"/ON"), (b"ON", b"[]"), (b"OFF", &off)])?;
+    let ocp = patch_dict(&ocp, &[(b"D", &cfg)])?;
+    let catalog = patch_dict(&cat, &[(b"OCProperties", &ocp)])?;
+    let root_ref = format!("{} {} R", root.obj_number, root.gen_number);
+    let prev_value = prev.to_string();
+    let trailer = patch_dict(
+        &trailer,
+        &[
+            (b"Root", root_ref.as_bytes()),
+            (b"Prev", prev_value.as_bytes()),
+            (b"Type", b""),
+            (b"Length", b""),
+            (b"Filter", b""),
+            (b"DecodeParms", b""),
+            (b"W", b""),
+            (b"Index", b""),
+        ],
+    )?;
+    let mut out = bytes.to_vec();
+    out.push(b'\n');
     let at = out.len();
-    out.extend_from_slice(
-        format!(
-            "{root} 0 obj\n<</Type/Catalog/Pages {} {} R/OCProperties<</OCGs[{}]/D<</BaseState/ON/OFF[{}]>>>>>>\nendobj\n",
-            pages.obj_number,
-            pages.gen_number,
-            all.join(" "),
-            off.join(" ")
-        )
-        .as_bytes(),
-    );
+    // Redefine the existing catalog: no arbitrary fresh object number or Size increase.
+    out.extend_from_slice(format!("{} {} obj\n", root.obj_number, root.gen_number).as_bytes());
+    out.extend_from_slice(&catalog);
+    out.extend_from_slice(b"\nendobj\n");
     let xref_at = out.len();
-    let prev = {
-        let tail = String::from_utf8_lossy(&bytes[bytes.len().saturating_sub(1024)..]).to_string();
-        tail.rsplit_once("startxref").and_then(|(_, t)| t.split_whitespace().next()).and_then(|n| n.parse::<usize>().ok())?
-    };
-    out.extend_from_slice(
-        format!("xref\n{root} 1\n{at:010} 00000 n\r\ntrailer\n<</Size {}/Root {root} 0 R/Prev {prev}>>\nstartxref\n{xref_at}\n%%EOF\n", root + 1)
-            .as_bytes(),
-    );
+    out.extend_from_slice(format!("xref\n{} 1\n{at:010} {:05} n\r\ntrailer\n", root.obj_number, root.gen_number).as_bytes());
+    out.extend_from_slice(&trailer);
+    out.extend_from_slice(format!("\nstartxref\n{xref_at}\n%%EOF\n").as_bytes());
     Some(out)
 }
 
 /// Shared copies of layer variants (by source address and hidden set).
 pub(crate) fn layered(data: &Arc<Vec<u8>>, hidden: &[String]) -> Arc<Vec<u8>> {
     type Key = (usize, Vec<String>);
-    static CACHE: Mutex<Option<HashMap<Key, Arc<Vec<u8>>>>> = Mutex::new(None);
+    static CACHE: Mutex<Option<HashMap<Key, (Arc<Vec<u8>>, Arc<Vec<u8>>)>>> = Mutex::new(None);
     let key = (Arc::as_ptr(data) as usize, hidden.to_vec());
     let mut c = CACHE.lock().unwrap_or_else(|e| e.into_inner());
     let map = c.get_or_insert_with(HashMap::new);
     if let Some(v) = map.get(&key) {
-        return v.clone();
+        return v.1.clone();
     }
     if map.len() > 64 {
         map.clear();
     }
     let v = pdf_hide_layers(data, hidden).map(Arc::new).unwrap_or_else(|| data.clone());
-    map.insert(key, v.clone());
+    map.insert(key, (data.clone(), v.clone()));
     v
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use craft_pdf::syntax::reader::{Reader, ReaderExt};
+
+    #[test]
+    fn dictionary_patch_preserves_raw_values_and_bounds_updates() {
+        let data = b"<</Unknown[(x) 12 1 R <ff>]/Base#53tate/OFF/Name(a(b)c)>>";
+        let dict = Reader::new(data).read_without_context::<Dict<'_>>().unwrap();
+        let out = patch_dict(&dict, &[(b"BaseState", b"/ON")]).unwrap();
+        let text = std::str::from_utf8(&out).unwrap();
+        assert!(text.contains("/Unknown[(x) 12 1 R <ff>]"));
+        assert!(text.contains("/Name(a(b)c)"));
+        assert!(!text.contains("/OFF"));
+        assert!(text.contains("/BaseState /ON"));
+        assert_eq!(patch_dict(&Dict::empty(), &[(b"BaseState", b"/ON")]).unwrap(), b"<< /BaseState /ON >>");
+        let mut many = b"<< /Big (".to_vec();
+        many.resize(4 * 1024 * 1024 + 1, b'x');
+        many.extend_from_slice(b") >>");
+        let dict = Reader::new(&many).read_without_context::<Dict<'_>>().unwrap();
+        assert!(patch_dict(&dict, &[]).is_none());
+    }
+}

docs/gaps.md

@@ -190,6 +190,9 @@ Kind: **F** feature · **U** UI/UX · **T** typography · **FF** file format ·
 - **Estimate:** 20–35 h.
 
 ### 19. Placed graphics depth (FF)
+- **Safety:** PDF visibility variants preserve catalog/configuration entries and nonzero-generation
+  references, redefining the existing catalog rather than choosing an arbitrary object number.
+  Encrypted, hybrid-reference and over-budget rewrites are rejected; no print-fidelity claim.
 - **Missing:** EPS is shown through its preview only (no PostScript interpreter); PSD layer
   comps beyond visibility; the Clipping Path Options dialog (tracing into a clip that keeps the
   frame, Threshold/Tolerance/Inset Frame preview, picking an alpha channel or a path saved in a