diff --git a/crates/organize/src/dedupe.rs b/crates/organize/src/dedupe.rs new file mode 100644 index 000000000..96a327168 --- /dev/null +++ b/crates/organize/src/dedupe.rs @@ -0,0 +1,323 @@ +//! Merging duplicate resources after pages are copied (execution plan M4.5). +//! +//! Combining a document with itself, or inserting pages from a file that uses the same fonts, +//! would otherwise store every font, image and colour profile once per source. Here, objects +//! that are byte-for-byte the same resource after copying are merged into one, and references +//! are rewritten to it. +//! +//! - Only **resource-like** objects are merged: streams (font files, images, form XObjects, ICC +//! profiles, functions…), font, font-descriptor, graphics-state, pattern, shading, function and +//! encoding dictionaries, and colour-space arrays. Objects whose *identity* matters (pages, +//! annotations, form fields, layers, outline items, structure elements) are never merged. +//! - Two objects are equal when they are structurally identical with references compared after +//! merging, so a font dictionary merges once its font file has. This runs to a fixpoint. +//! - Only `candidates` (objects created by the operation) are ever replaced or rewritten. Existing +//! objects can serve as the merge target but are not changed, so an incremental save stays +//! small. +//! - Merged objects may now be shared by several pages. Editing code must copy on write (cos +//! objects are immutable values, so a changed resource is always a new object). + +use std::collections::{HashMap, HashSet}; +use std::hash::{Hash, Hasher}; + +use printcraft_cos::{Document, ObjRef, Object}; + +const RESOURCE_TYPES: &[&[u8]] = &[b"Font", b"FontDescriptor", b"ExtGState", b"Pattern", b"Encoding", b"CMap", b"Halftone"]; +const COLOR_SPACE_FAMILIES: &[&[u8]] = &[b"ICCBased", b"Indexed", b"Separation", b"DeviceN", b"CalRGB", b"CalGray", b"Lab", b"Pattern"]; + +fn is_resource(o: &Object) -> bool { + match o { + Object::Stream(s) => !matches!(s.dict.get(b"Type"), Some(Object::Name(n)) if n == b"XRef" || n == b"ObjStm"), + Object::Dict(d) => { + matches!(d.get(b"Type"), Some(Object::Name(n)) if RESOURCE_TYPES.contains(&n.as_slice())) + || d.contains(b"ShadingType") + || d.contains(b"FunctionType") + } + Object::Array(a) => matches!(a.first(), Some(Object::Name(n)) if COLOR_SPACE_FAMILIES.contains(&n.as_slice())), + _ => false, + } +} + +/// Resource dictionary categories whose entries are shareable resources. `/Properties` is left +/// out on purpose: it names optional content groups, whose identity matters. +const RESOURCE_CATEGORIES: &[&[u8]] = &[b"ExtGState", b"ColorSpace", b"Pattern", b"Shading", b"XObject", b"Font"]; + +/// References listed in any `/Resources` dictionary reachable inside `o` (pages, forms, Type 3 +/// fonts, annotation appearances), including the `/Resources` dictionary itself when indirect. +fn collect_resource_refs(doc: &Document, o: &Object, out: &mut HashSet) { + let dict = match o { + Object::Dict(d) => d, + Object::Stream(s) => &s.dict, + Object::Array(a) => { + for x in a { + collect_resource_refs(doc, x, out); + } + return; + } + _ => return, + }; + for (k, v) in dict.iter() { + if k.as_slice() == b"Resources" { + if let Object::Ref(r) = v { + out.insert(*r); + } + let res = doc.resolve(v); + if let Some(res) = res.as_dict() { + for cat in RESOURCE_CATEGORIES { + let entries = res.get(cat).map(|c| doc.resolve(c)); + if let Some(entries) = entries.as_ref().and_then(|e| e.as_dict()) { + out.extend(entries.iter().filter_map(|(_, v)| if let Object::Ref(r) = v { Some(*r) } else { None })); + } + } + } + } else if matches!(v, Object::Dict(_) | Object::Array(_)) { + collect_resource_refs(doc, v, out); + } + } +} + +/// Union-find style lookup of the object a reference now stands for. +fn canon(map: &HashMap, mut r: ObjRef) -> ObjRef { + while let Some(n) = map.get(&r) { + r = *n; + } + r +} + +fn hash_obj(o: &Object, map: &HashMap, h: &mut impl Hasher) { + match o { + Object::Null => 0u8.hash(h), + Object::Bool(b) => (1u8, b).hash(h), + Object::Int(i) => (2u8, i).hash(h), + Object::Real(f) => (3u8, f.to_bits()).hash(h), + Object::String(s) => (4u8, &s.bytes).hash(h), + Object::Name(n) => (5u8, n).hash(h), + Object::Array(a) => { + (6u8, a.len()).hash(h); + for x in a { + hash_obj(x, map, h); + } + } + Object::Dict(d) => { + // Key order does not change meaning: combine per-entry hashes order-independently. + let mut acc = 0u64; + for (k, v) in d.iter() { + let mut eh = std::collections::hash_map::DefaultHasher::new(); + k.hash(&mut eh); + hash_obj(v, map, &mut eh); + acc = acc.wrapping_add(eh.finish()); + } + (7u8, d.len(), acc).hash(h); + } + Object::Stream(s) => { + 8u8.hash(h); + hash_obj(&Object::Dict(s.dict.clone()), map, h); + s.raw.as_slice().hash(h); + } + Object::Ref(r) => { + let c = canon(map, *r); + (9u8, c.num, c.generation).hash(h); + } + } +} + +fn eq_obj(a: &Object, b: &Object, map: &HashMap) -> bool { + match (a, b) { + (Object::Array(x), Object::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| eq_obj(p, q, map)), + (Object::Dict(x), Object::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.get(k).is_some_and(|w| eq_obj(v, w, map))), + (Object::Stream(x), Object::Stream(y)) => { + (std::sync::Arc::ptr_eq(&x.raw, &y.raw) || x.raw == y.raw) && eq_obj(&Object::Dict(x.dict.clone()), &Object::Dict(y.dict.clone()), map) + } + (Object::Ref(x), Object::Ref(y)) => canon(map, *x) == canon(map, *y), + (Object::Real(x), Object::Real(y)) => x.to_bits() == y.to_bits(), + (Object::String(x), Object::String(y)) => x.bytes == y.bytes, + _ => a == b, + } +} + +fn rewrite(o: &Object, map: &HashMap) -> Object { + match o { + Object::Ref(r) => Object::Ref(canon(map, *r)), + Object::Array(a) => Object::Array(a.iter().map(|x| rewrite(x, map)).collect()), + Object::Dict(d) => { + let mut out = d.clone(); + for (k, v) in d.iter() { + out.set(k.clone(), rewrite(v, map)); + } + Object::Dict(out) + } + Object::Stream(s) => { + let mut s = s.clone(); + if let Object::Dict(d) = rewrite(&Object::Dict(s.dict.clone()), map) { + s.dict = d; + } + Object::Stream(s) + } + other => other.clone(), + } +} + +fn contains_ref(o: &Object, map: &HashMap) -> bool { + match o { + Object::Ref(r) => map.contains_key(r), + Object::Array(a) => a.iter().any(|x| contains_ref(x, map)), + Object::Dict(d) => d.iter().any(|(_, v)| contains_ref(v, map)), + Object::Stream(s) => s.dict.iter().any(|(_, v)| contains_ref(v, map)), + _ => false, + } +} + +/// Merge duplicate resources among `candidates` (into each other or into equal objects already +/// in `doc`), rewrite references inside `candidates`, and free the merged-away objects. +/// When `index_existing` is false, only candidates are compared (a brand-new document). +/// Returns how many objects were removed. +pub fn dedupe_resources(doc: &mut Document, candidates: &[ObjRef], index_existing: bool) -> usize { + let cand: HashSet = candidates.iter().copied().collect(); + let mut sorted: Vec = candidates.to_vec(); + sorted.sort_unstable_by_key(|r| r.num); + // Existing objects first, so candidates merge into them rather than the reverse. + let mut all: Vec<(ObjRef, std::sync::Arc)> = Vec::new(); + if index_existing { + for num in doc.object_numbers() { + let r = ObjRef::new(num, doc.generation(num)); + if !cand.contains(&r) { + all.push((r, doc.get(r))); + } + } + } + all.extend(sorted.iter().map(|r| (*r, doc.get(*r)))); + let mut listed = HashSet::new(); + for (_, o) in &all { + collect_resource_refs(doc, o, &mut listed); + } + let universe: Vec<(ObjRef, std::sync::Arc)> = + all.into_iter().filter(|(r, o)| is_resource(o) || (listed.contains(r) && !matches!(**o, Object::Null))).collect(); + + let mut map: HashMap = HashMap::new(); + // A fixpoint: each round can make referring objects equal. Depth is small in practice. + for _round in 0..16 { + let mut buckets: HashMap> = HashMap::new(); + let mut merged = 0; + for (i, (r, o)) in universe.iter().enumerate() { + if map.contains_key(r) { + continue; + } + let mut h = std::collections::hash_map::DefaultHasher::new(); + hash_obj(o, &map, &mut h); + let bucket = buckets.entry(h.finish()).or_default(); + let target = bucket.iter().map(|j| &universe[*j]).find(|(_, other)| eq_obj(o, other, &map)).map(|(t, _)| *t); + match target { + Some(t) if cand.contains(r) => { + map.insert(*r, t); + merged += 1; + } + _ => bucket.push(i), + } + } + if merged == 0 { + break; + } + } + if map.is_empty() { + return 0; + } + for r in &sorted { + if map.contains_key(r) { + continue; + } + let o = doc.get(*r); + if contains_ref(&o, &map) { + let new = rewrite(&o, &map); + doc.set(*r, new); + } + } + for r in map.keys() { + doc.free(*r); + } + map.len() +} + +#[cfg(test)] +mod tests { + use super::*; + use printcraft_cos::{Dict, Stream}; + use std::sync::Arc; + + fn stream(data: &[u8]) -> Object { + Object::Stream(Stream { dict: Dict::new(), raw: Arc::new(data.to_vec()) }) + } + + fn font(doc: &mut Document, file: ObjRef, name: &str) -> ObjRef { + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("Font")); + d.set(b"BaseFont".to_vec(), Object::name(name)); + d.set(b"FontFile".to_vec(), Object::Ref(file)); + doc.add(Object::Dict(d)) + } + + #[test] + fn merges_bottom_up_and_keeps_identity_objects() { + let mut doc = Document::new_empty(); + let f1 = doc.add(stream(b"glyphs")); + let f2 = doc.add(stream(b"glyphs")); + let f3 = doc.add(stream(b"other")); + let a = font(&mut doc, f1, "A"); + let b = font(&mut doc, f2, "A"); // equal to `a` once f2 merges into f1 + let c = font(&mut doc, f3, "A"); + let mut annot = Dict::new(); + annot.set(b"Type".to_vec(), Object::name("Annot")); + let n1 = doc.add(Object::Dict(annot.clone())); + let n2 = doc.add(Object::Dict(annot)); + let mut page = Dict::new(); + page.set(b"Fonts".to_vec(), Object::Array(vec![Object::Ref(a), Object::Ref(b), Object::Ref(c)])); + let p = doc.add(Object::Dict(page)); + let removed = dedupe_resources(&mut doc, &[f1, f2, f3, a, b, c, n1, n2, p], false); + assert_eq!(removed, 2); + let page = doc.get(p); + let fonts = page.as_dict().unwrap().get(b"Fonts").unwrap().as_array().unwrap().clone(); + assert_eq!(fonts, vec![Object::Ref(a), Object::Ref(a), Object::Ref(c)]); + assert!(!doc.object_numbers().contains(&f2.num) && !doc.object_numbers().contains(&b.num)); + assert!(doc.object_numbers().contains(&n2.num), "annotations are never merged"); + } + + #[test] + fn untyped_resources_are_found_through_resource_dictionaries() { + let mut doc = Document::new_empty(); + let mut gs = Dict::new(); + gs.set(b"CA".to_vec(), Object::Real(0.5)); + let g1 = doc.add(Object::Dict(gs.clone())); + let g2 = doc.add(Object::Dict(gs.clone())); + let g3 = doc.add(Object::Dict(gs)); // not listed as a resource: left alone + let page = |doc: &mut Document, g: ObjRef| { + let mut ext = Dict::new(); + ext.set(b"G0".to_vec(), Object::Ref(g)); + let mut res = Dict::new(); + res.set(b"ExtGState".to_vec(), Object::Dict(ext)); + let mut p = Dict::new(); + p.set(b"Type".to_vec(), Object::name("Page")); + p.set(b"Resources".to_vec(), Object::Dict(res)); + doc.add(Object::Dict(p)) + }; + let p1 = page(&mut doc, g1); + let p2 = page(&mut doc, g2); + assert_eq!(dedupe_resources(&mut doc, &[g1, g2, g3, p1, p2], false), 1); + assert!(doc.object_numbers().contains(&g3.num)); + let nums = doc.object_numbers(); + assert!(nums.contains(&p1.num) && nums.contains(&p2.num), "pages are never merged"); + assert_eq!(doc.get(p1), doc.get(p2), "both pages now use the one graphics state"); + } + + #[test] + fn existing_objects_are_targets_but_never_changed() { + let mut doc = Document::new_empty(); + let old = doc.add(stream(b"icc")); + let new = doc.add(stream(b"icc")); + let mut holder = Dict::new(); + holder.set(b"CS".to_vec(), Object::Array(vec![Object::name("ICCBased"), Object::Ref(new)])); + let h = doc.add(Object::Dict(holder)); + assert_eq!(dedupe_resources(&mut doc, &[new, h], true), 1); + assert!(doc.object_numbers().contains(&old.num)); + let cs = doc.get(h).as_dict().unwrap().get(b"CS").unwrap().clone(); + assert_eq!(cs, Object::Array(vec![Object::name("ICCBased"), Object::Ref(old)])); + } +} diff --git a/crates/organize/src/import.rs b/crates/organize/src/import.rs index 206c8f15e..716098efc 100644 --- a/crates/organize/src/import.rs +++ b/crates/organize/src/import.rs @@ -183,8 +183,15 @@ impl Copier<'_> { /// Copy pages `src_pages` (0-based, in the given order) of `src` into `dst`, inserting them at /// position `at`. Returns the new page references in order. +/// +/// Resources identical to ones already in `dst` (the same fonts, images, colour profiles) are +/// shared rather than stored twice (see `dedupe`). pub fn import_pages(dst: &mut Document, src: &Document, src_pages: &[usize], at: usize) -> Result, OrganizeError> { - import_pages_mapped(dst, src, src_pages, at).map(|(pages, _)| pages) + let first_new = dst.object_numbers().last().map_or(1, |n| n + 1); + let pages = import_pages_mapped(dst, src, src_pages, at)?.0; + let created: Vec = dst.object_numbers().into_iter().filter(|n| *n >= first_new).map(|n| ObjRef::new(n, dst.generation(n))).collect(); + crate::dedupe::dedupe_resources(dst, &created, true); + Ok(pages) } /// `import_pages`, also returning the source-page → destination-page map. @@ -406,7 +413,8 @@ fn register_fields(dst: &mut Document, fields: &[ObjRef]) -> Result<(), Organize /// Document information (title, author…) is carried over. pub fn extract_pages(src: &Document, pages: &[usize]) -> Result { let mut out = Document::new_empty(); - import_pages(&mut out, src, pages, 0)?; + // One source: nothing to deduplicate. + import_pages_mapped(&mut out, src, pages, 0)?; for key in crate::INFO_KEYS { if let Some(v) = crate::info(src, key) { crate::set_info(&mut out, key, &v)?; @@ -466,6 +474,9 @@ pub fn combine(sources: &[(&str, &Document)]) -> Result } collect_attachments(&mut out, src, &mut attachments); } + // Sources often share fonts, images and profiles (or are the same file): store them once. + let all: Vec = out.object_numbers().into_iter().map(|n| ObjRef::new(n, out.generation(n))).collect(); + crate::dedupe::dedupe_resources(&mut out, &all, false); add_outline(&mut out, &marks)?; set_attachments(&mut out, attachments)?; Ok(out) diff --git a/crates/organize/src/lib.rs b/crates/organize/src/lib.rs index aa677d930..5914bc531 100644 --- a/crates/organize/src/lib.rs +++ b/crates/organize/src/lib.rs @@ -11,8 +11,10 @@ use printcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; +mod dedupe; mod import; +pub use dedupe::dedupe_resources; pub use import::{SplitBy, combine, extract_pages, import_pages, split, split_ranges}; #[derive(Debug, thiserror::Error, PartialEq)] diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 93f54154f..5413a9ee6 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -455,3 +455,28 @@ fn combine_nests_source_bookmarks_and_keeps_attachments() { let keys: Vec = tree.get(b"Names").and_then(|n| n.as_array()).unwrap().chunks(2).map(|p| p[0].as_string().unwrap().to_text()).collect(); assert_eq!(keys, ["notes.txt", "notes.txt (2)"]); } + +#[test] +fn combine_and_insert_share_identical_resources() { + let a = doc_a(); + let fonts = |d: &Document| { + d.object_numbers() + .into_iter() + .filter(|n| d.get(ObjRef::new(*n, d.generation(*n))).as_dict().is_some_and(|x| x.name(b"Type") == Some(b"Font"))) + .count() + }; + assert_eq!(fonts(&a), 1); + // The same document three times: one font, and every page still renders its own text. + let out = full_roundtrip(&crate::combine(&[("one", &a), ("two", &a), ("three", &a)]).unwrap()); + assert_eq!(fonts(&out), 1); + assert_eq!(labels(&out), ["A1", "A2", "A3", "A1", "A2", "A3", "A1", "A2", "A3"]); + // Inserting pages that use the same font reuses the font already there, without touching it. + let mut d = doc_a(); + let font_before = + d.object_numbers().into_iter().find(|n| d.get(ObjRef::new(*n, 0)).as_dict().is_some_and(|x| x.name(b"Type") == Some(b"Font"))).unwrap(); + crate::import_pages(&mut d, &doc_a(), &[0, 1], 3).unwrap(); + assert_eq!(fonts(&d), 1); + assert!(!d.modified_objects().contains(&font_before)); + let d = full_roundtrip(&d); + assert_eq!(labels(&d), ["A1", "A2", "A3", "A1", "A2"]); +}