diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rs index fec7cbab0..0b1813103 100644 --- a/crates/doc/src/assets.rs +++ b/crates/doc/src/assets.rs @@ -5,6 +5,7 @@ //! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`]. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::hash::BuildHasher; use std::sync::Arc; use crate::story::Story; @@ -246,10 +247,19 @@ impl Document { if !self.asset_refs(&mut |_| {}) { return 0; } - let mut reps: HashMap<(usize, &str, u32), Vec<&Arc>> = HashMap::new(); + let fingerprint = std::collections::hash_map::RandomState::new(); + let mut hashes = HashMap::new(); + let mut reps: HashMap<_, Vec<&Arc>> = HashMap::new(); let mut map = HashMap::new(); for a in self.assets.values() { - let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default(); + // Empty, unlinked assets never compare equal. Don't build a quadratic bucket + // of them, and hash a shared immutable buffer only once per compaction. + if a.data.is_empty() && a.link.is_none() { + continue; + } + let hash = *hashes.entry(Arc::as_ptr(&a.data)).or_insert_with(|| fingerprint.hash_one(a.data.as_slice())); + let bucket = reps.entry((hash, a.mime.as_str(), a.page, a.link.as_deref())).or_default(); + // A fingerprint only selects candidates; equality still checks the actual file. match bucket.iter().find(|r| r.same_file(a)) { Some(r) => { map.insert(a.id, r.id); @@ -280,6 +290,56 @@ mod tests { Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), data: Arc::new(data.to_vec()), ..Default::default() } } + #[test] + fn equal_size_assets_keep_distinct_contents_and_merge_exact_copies() { + let mut d = Document::new(&Default::default()); + for id in 1..=256u64 { + let mut bytes = vec![42; 1024]; + bytes[1016..].copy_from_slice(&id.to_le_bytes()); + d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes))); + d.assets.insert(AssetId(id + 256), Arc::new(asset(id + 256, &bytes))); + } + assert_eq!(d.merge_duplicate_assets(), 256); + assert_eq!(d.assets.len(), 256); + for id in 1..=256u64 { + assert_eq!(d.assets[&AssetId(id)].data[1016..], id.to_le_bytes()); + } + } + + #[test] + fn compaction_preserves_asset_identity_and_missing_data_rules() { + let mut d = Document::new(&Default::default()); + let mut cases = vec![asset(1, b"same"), asset(2, b"same"), asset(3, b"same"), asset(4, b"same")]; + cases[1].mime = "application/pdf".into(); + cases[2].page = 1; + cases[3].link = Some("/linked.png".into()); + cases.extend([asset(5, b""), asset(6, b""), asset(7, b""), asset(8, b"")]); + cases[6].link = Some("/missing.png".into()); + cases[7].link = Some("/missing.png".into()); + for a in cases { + d.assets.insert(a.id, Arc::new(a)); + } + assert_eq!(d.merge_duplicate_assets(), 1); + assert_eq!(d.assets.keys().copied().collect::>(), (1..=7).map(AssetId).collect::>()); + } + + #[test] + #[cfg(not(target_arch = "wasm32"))] + #[ignore = "manual bounded timing comparison; no timing assertion"] + fn equal_size_assets_timing() { + for count in [500, 1000, 2000] { + let mut d = Document::new(&Default::default()); + for id in 1..=count as u64 { + let mut bytes = vec![42; 8192]; + bytes[8184..].copy_from_slice(&id.to_le_bytes()); + d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes))); + } + let start = std::time::Instant::now(); + assert_eq!(d.merge_duplicate_assets(), 0); + eprintln!("{count} distinct 8 KiB assets: {:?}", start.elapsed()); + } + } + fn graphic_item(id: u64, asset: u64, crop: f64) -> Item { let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0))); it.content = Content::Graphic(Graphic { diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 293ad8f48..fce6921c3 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -13,6 +13,7 @@ #![forbid(unsafe_code)] use std::collections::{HashMap, HashSet}; +use std::hash::BuildHasher; use std::io::{Cursor, Read, Write}; use std::sync::Arc; @@ -155,12 +156,13 @@ fn load_zip(bytes: &[u8]) -> Result { let names: Vec = z.file_names().map(str::to_string).collect(); // Files saved with one copy per placement: identical copies share one buffer as they are // read, so memory holds each distinct file once. - let mut loaded: HashMap>>> = HashMap::new(); + let fingerprint = std::collections::hash_map::RandomState::new(); + let mut loaded: HashMap>>> = HashMap::new(); for name in names { let Some(stem) = name.strip_prefix("assets/") else { continue }; let Some(id) = stem.split('.').next().and_then(|s| s.parse::().ok()) else { continue }; if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) { - let same = loaded.entry(data.len()).or_default(); + let same = loaded.entry(fingerprint.hash_one(data.as_slice())).or_default(); let data = match same.iter().find(|d| ***d == data) { Some(d) => d.clone(), None => { @@ -568,6 +570,33 @@ mod tests { } } + #[test] + fn equal_size_archive_assets_preserve_content_and_share_only_identical_buffers() { + let mut d = Document::new(&NewDocument::default()); + let mut pairs = Vec::new(); + for n in 0..128u64 { + let mut bytes = vec![42; 1024]; + bytes[1016..].copy_from_slice(&n.to_le_bytes()); + let a = png_asset(&mut d, &bytes); + let b = png_asset(&mut d, &bytes); + // Distinct links must remain independently relinkable even when their bytes match. + Arc::make_mut(d.assets.get_mut(&a).unwrap()).link = Some(format!("/a/{n}.png")); + Arc::make_mut(d.assets.get_mut(&b).unwrap()).link = Some(format!("/b/{n}.png")); + place(&mut d, a, 1.0); + place(&mut d, b, 2.0); + pairs.push((a, b)); + } + let back = load(&save_uncompacted(&d)).unwrap(); + assert_eq!(back.assets.len(), 256); + for (n, (a, b)) in pairs.into_iter().enumerate() { + assert_eq!(back.assets[&a].data[1016..], (n as u64).to_le_bytes()); + assert!(Arc::ptr_eq(&back.assets[&a].data, &back.assets[&b].data)); + assert_eq!(back.assets[&a].link, d.assets[&a].link); + assert_eq!(back.assets[&b].link, d.assets[&b].link); + } + assert_eq!(back.spreads, d.spreads); + } + #[test] fn older_files_with_one_asset_per_placement_open_with_one() { let mut d = Document::new(&NewDocument::default()); diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c19df6b5c..d32492602 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File format parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (.indd and .indt open through indd-utils; XLSX cached values preserve numeric precision and text; shared print spooler; Windows CI passes) · **Target:** Adobe InDesign 2026 (21.6) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (.indd and .indt open through indd-utils; XLSX cached values preserve numeric precision and text; shared print spooler; Windows CI passes; content-indexed asset deduplication, #727) · **Target:** Adobe InDesign 2026 (21.6) Every format InDesign 2026 reads or writes, with DesignCraft's support, fidelity and tests. The InDesign list comes from its bundle's `Info.plist` (`CFBundleDocumentTypes`, @@ -22,7 +22,7 @@ score below). | InDesign document | `.indd` | read/write (native) | ✓ (converted to IDML in memory by [storytold/indd-utils](https://github.com/storytold/indd-utils), then the IDML importer; #21, #114) | ✗ | Converter warnings join the open result; unreadable files are open errors. Synthetic-bytes tests only; indd-utils reports 96.9% of IDML values reproduced on ~650 pairs. No real-file corpus test | 30% | | InDesign template | `.indt` | read/write | ✓ (as `.indd`) | ✗ | As `.indd` | 2% | | InDesign Markup | `.idml` | read/write | ✓ | ✓ | Synthetic fixtures in `crates/idml/src/tests.rs` (51 tests); exported files verified by opening in InDesign 2026 by hand; indd-utils' 98-document run imports 811 spreads / 2,866 stories / 5,812 items. Open: #193, #338, #339, #340, #211. No real-file corpus test | 20% | -| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests | — | +| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests. Asset loading and compaction index by content hash, then confirm byte equality; equal-sized distinct files fall into separate search buckets (#727). Separate links, MIME types and PDF pages retain their identity | — | | InDesign book | `.indb` | read/write | ✗ (own `.dcbook`) | ✗ | Books are shallow (see gaps.md) | 2% | | InDesign library | `.indl`, `.inlx` | read/write | ✗ (own `.dclib`) | ✗ | | 1% | | Snippet | `.idms` | read/write | ✗ (own snippets) | ✗ | | 1% | @@ -108,6 +108,7 @@ up to 30 points. | Date | Change | Summary | |---|---|---| +| 2026-10-13 | minor | Asset loading and compaction use content fingerprints with exact equality checks; tests cover equal-sized distinct buffers, shared copies, independent links and missing data (#727) | | 2026-10-13 | minor | `.indd` and `.indt` read through storytold/indd-utils; INDD remaining work is a real-file corpus (10–30 h); score unchanged until it is measured | | 2026-10-12 | minor | XLSX table placement and data merge preserve small numeric values and cached formula strings; synthetic regressions in `textimport::xlsx::tests`. Scores unchanged. | | 2026-10-10 | major | Created from InDesign 2026's declared document types and filters, `plan/indesign/08-file-formats.md` and the code |