From 0fd627aa44b268d935f04e1b094d46ba84815865 Mon Sep 17 00:00:00 2001 From: LabDennis <341377186+LabDennis@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:12:09 +0200 Subject: [PATCH] perf(assets): index duplicate candidates by content (#727) --- crates/doc/src/assets.rs | 64 ++++++++++++++++++++++++++++++++++++-- crates/format/src/lib.rs | 33 ++++++++++++++++++-- docs/file-format-parity.md | 5 +-- 3 files changed, 96 insertions(+), 6 deletions(-) diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rs index 24ba5c243..cff196fc0 100644 --- a/crates/doc/src/assets.rs +++ b/crates/doc/src/assets.rs @@ -5,6 +5,7 @@ //! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`]. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::hash::BuildHasher; use std::sync::Arc; use crate::story::Story; @@ -246,10 +247,19 @@ impl Document { if !self.asset_refs(&mut |_| {}) { return 0; } - let mut reps: HashMap<(usize, &str, u32), Vec<&Arc>> = HashMap::new(); + let fingerprint = std::collections::hash_map::RandomState::new(); + let mut hashes = HashMap::new(); + let mut reps: HashMap<_, Vec<&Arc>> = HashMap::new(); let mut map = HashMap::new(); for a in self.assets.values() { - let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default(); + // Empty, unlinked assets never compare equal. Don't build a quadratic bucket + // of them, and hash a shared immutable buffer only once per compaction. + if a.data.is_empty() && a.link.is_none() { + continue; + } + let hash = *hashes.entry(Arc::as_ptr(&a.data)).or_insert_with(|| fingerprint.hash_one(a.data.as_slice())); + let bucket = reps.entry((hash, a.mime.as_str(), a.page, a.link.as_deref())).or_default(); + // A fingerprint only selects candidates; equality still checks the actual file. match bucket.iter().find(|r| r.same_file(a)) { Some(r) => { map.insert(a.id, r.id); @@ -280,6 +290,56 @@ mod tests { Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), link: None, data: Arc::new(data.to_vec()), pixels: None, page: 0 } } + #[test] + fn equal_size_assets_keep_distinct_contents_and_merge_exact_copies() { + let mut d = Document::new(&Default::default()); + for id in 1..=256u64 { + let mut bytes = vec![42; 1024]; + bytes[1016..].copy_from_slice(&id.to_le_bytes()); + d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes))); + d.assets.insert(AssetId(id + 256), Arc::new(asset(id + 256, &bytes))); + } + assert_eq!(d.merge_duplicate_assets(), 256); + assert_eq!(d.assets.len(), 256); + for id in 1..=256u64 { + assert_eq!(d.assets[&AssetId(id)].data[1016..], id.to_le_bytes()); + } + } + + #[test] + fn compaction_preserves_asset_identity_and_missing_data_rules() { + let mut d = Document::new(&Default::default()); + let mut cases = vec![asset(1, b"same"), asset(2, b"same"), asset(3, b"same"), asset(4, b"same")]; + cases[1].mime = "application/pdf".into(); + cases[2].page = 1; + cases[3].link = Some("/linked.png".into()); + cases.extend([asset(5, b""), asset(6, b""), asset(7, b""), asset(8, b"")]); + cases[6].link = Some("/missing.png".into()); + cases[7].link = Some("/missing.png".into()); + for a in cases { + d.assets.insert(a.id, Arc::new(a)); + } + assert_eq!(d.merge_duplicate_assets(), 1); + assert_eq!(d.assets.keys().copied().collect::>(), (1..=7).map(AssetId).collect::>()); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + #[ignore = "manual bounded timing comparison; no timing assertion"] + fn equal_size_assets_timing() { + for count in [500, 1000, 2000] { + let mut d = Document::new(&Default::default()); + for id in 1..=count as u64 { + let mut bytes = vec![42; 8192]; + bytes[8184..].copy_from_slice(&id.to_le_bytes()); + d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes))); + } + let start = std::time::Instant::now(); + assert_eq!(d.merge_duplicate_assets(), 0); + eprintln!("{count} distinct 8 KiB assets: {:?}", start.elapsed()); + } + } + fn graphic_item(id: u64, asset: u64, crop: f64) -> Item { let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0))); it.content = Content::Graphic(Graphic { diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 408bb3ccc..c1b6d805a 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -11,6 +11,7 @@ #![forbid(unsafe_code)] use std::collections::{HashMap, HashSet}; +use std::hash::BuildHasher; use std::io::{Cursor, Read, Write}; use std::sync::Arc; @@ -107,12 +108,13 @@ fn load_zip(bytes: &[u8]) -> Result { let names: Vec = z.file_names().map(str::to_string).collect(); // Files saved with one copy per placement: identical copies share one buffer as they are // read, so memory holds each distinct file once. - let mut loaded: HashMap>>> = HashMap::new(); + let fingerprint = std::collections::hash_map::RandomState::new(); + let mut loaded: HashMap>>> = HashMap::new(); for name in names { let Some(stem) = name.strip_prefix("assets/") else { continue }; let Some(id) = stem.split('.').next().and_then(|s| s.parse::().ok()) else { continue }; if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) { - let same = loaded.entry(data.len()).or_default(); + let same = loaded.entry(fingerprint.hash_one(data.as_slice())).or_default(); let data = match same.iter().find(|d| ***d == data) { Some(d) => d.clone(), None => { @@ -338,6 +340,33 @@ mod tests { } } + #[test] + fn equal_size_archive_assets_preserve_content_and_share_only_identical_buffers() { + let mut d = Document::new(&NewDocument::default()); + let mut pairs = Vec::new(); + for n in 0..128u64 { + let mut bytes = vec![42; 1024]; + bytes[1016..].copy_from_slice(&n.to_le_bytes()); + let a = png_asset(&mut d, &bytes); + let b = png_asset(&mut d, &bytes); + // Distinct links must remain independently relinkable even when their bytes match. + Arc::make_mut(d.assets.get_mut(&a).unwrap()).link = Some(format!("/a/{n}.png")); + Arc::make_mut(d.assets.get_mut(&b).unwrap()).link = Some(format!("/b/{n}.png")); + place(&mut d, a, 1.0); + place(&mut d, b, 2.0); + pairs.push((a, b)); + } + let back = load(&save_uncompacted(&d)).unwrap(); + assert_eq!(back.assets.len(), 256); + for (n, (a, b)) in pairs.into_iter().enumerate() { + assert_eq!(back.assets[&a].data[1016..], (n as u64).to_le_bytes()); + assert!(Arc::ptr_eq(&back.assets[&a].data, &back.assets[&b].data)); + assert_eq!(back.assets[&a].link, d.assets[&a].link); + assert_eq!(back.assets[&b].link, d.assets[&b].link); + } + assert_eq!(back.spreads, d.spreads); + } + #[test] fn older_files_with_one_asset_per_placement_open_with_one() { let mut d = Document::new(&NewDocument::default()); diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index fc8fbc3a5..06f59de94 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File format parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (shared print spooler; Windows CI passes) · **Target:** Adobe InDesign 2026 (21.6) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (content-indexed asset deduplication, #727) · **Target:** Adobe InDesign 2026 (21.6) Every format InDesign 2026 reads or writes, with DesignCraft's support, fidelity and tests. The InDesign list comes from its bundle's `Info.plist` (`CFBundleDocumentTypes`, @@ -22,7 +22,7 @@ score below). | InDesign document | `.indd` | read/write (native) | ✗ (#21; converters proposed in #98, #114) | ✗ | — | 30% | | InDesign template | `.indt` | read/write | ✗ | ✗ | — | 2% | | InDesign Markup | `.idml` | read/write | ✓ | ✓ | Synthetic fixtures in `crates/idml/src/tests.rs` (49 tests); exported files verified by opening in InDesign 2026 by hand; indd-utils' 98-document run imports 811 spreads / 2,866 stories / 5,812 items. Open: #193, #338, #339, #340, #211. No real-file corpus test | 20% | -| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests | — | +| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests. Asset loading and compaction index by content hash, then confirm byte equality; equal-sized distinct files no longer all share one search bucket (#727). Separate links, MIME types and PDF pages retain their identity | — | | InDesign book | `.indb` | read/write | ✗ (own `.dcbook`) | ✗ | Books are shallow (see gaps.md) | 2% | | InDesign library | `.indl`, `.inlx` | read/write | ✗ (own `.dclib`) | ✗ | | 1% | | Snippet | `.idms` | read/write | ✗ (own snippets) | ✗ | | 1% | @@ -107,4 +107,5 @@ for real work** (estimated). INDD alone moves this dimension by up to 30 points. | Date | Change | Summary | |---|---|---| +| 2026-10-13 | minor | Asset loading and compaction use content fingerprints with exact equality checks; tests cover equal-sized distinct buffers, shared copies, independent links and missing data (#727) | | 2026-10-10 | major | Created from InDesign 2026's declared document types and filters, `plan/indesign/08-file-formats.md` and the code |