Merge #854: index duplicate asset candidates by content (#727)

# Conflicts:
#	docs/file-format-parity.md

crates/doc/src/assets.rs

@@ -5,6 +5,7 @@
 //! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`].
 
 use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::hash::BuildHasher;
 use std::sync::Arc;
 
 use crate::story::Story;
@@ -246,10 +247,19 @@ impl Document {
         if !self.asset_refs(&mut |_| {}) {
             return 0;
         }
-        let mut reps: HashMap<(usize, &str, u32), Vec<&Arc<Asset>>> = HashMap::new();
+        let fingerprint = std::collections::hash_map::RandomState::new();
+        let mut hashes = HashMap::new();
+        let mut reps: HashMap<_, Vec<&Arc<Asset>>> = HashMap::new();
         let mut map = HashMap::new();
         for a in self.assets.values() {
-            let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default();
+            // Empty, unlinked assets never compare equal. Don't build a quadratic bucket
+            // of them, and hash a shared immutable buffer only once per compaction.
+            if a.data.is_empty() && a.link.is_none() {
+                continue;
+            }
+            let hash = *hashes.entry(Arc::as_ptr(&a.data)).or_insert_with(|| fingerprint.hash_one(a.data.as_slice()));
+            let bucket = reps.entry((hash, a.mime.as_str(), a.page, a.link.as_deref())).or_default();
+            // A fingerprint only selects candidates; equality still checks the actual file.
             match bucket.iter().find(|r| r.same_file(a)) {
                 Some(r) => {
                     map.insert(a.id, r.id);
@@ -280,6 +290,56 @@ mod tests {
         Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), data: Arc::new(data.to_vec()), ..Default::default() }
     }
 
+    #[test]
+    fn equal_size_assets_keep_distinct_contents_and_merge_exact_copies() {
+        let mut d = Document::new(&Default::default());
+        for id in 1..=256u64 {
+            let mut bytes = vec![42; 1024];
+            bytes[1016..].copy_from_slice(&id.to_le_bytes());
+            d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes)));
+            d.assets.insert(AssetId(id + 256), Arc::new(asset(id + 256, &bytes)));
+        }
+        assert_eq!(d.merge_duplicate_assets(), 256);
+        assert_eq!(d.assets.len(), 256);
+        for id in 1..=256u64 {
+            assert_eq!(d.assets[&AssetId(id)].data[1016..], id.to_le_bytes());
+        }
+    }
+
+    #[test]
+    fn compaction_preserves_asset_identity_and_missing_data_rules() {
+        let mut d = Document::new(&Default::default());
+        let mut cases = vec![asset(1, b"same"), asset(2, b"same"), asset(3, b"same"), asset(4, b"same")];
+        cases[1].mime = "application/pdf".into();
+        cases[2].page = 1;
+        cases[3].link = Some("/linked.png".into());
+        cases.extend([asset(5, b""), asset(6, b""), asset(7, b""), asset(8, b"")]);
+        cases[6].link = Some("/missing.png".into());
+        cases[7].link = Some("/missing.png".into());
+        for a in cases {
+            d.assets.insert(a.id, Arc::new(a));
+        }
+        assert_eq!(d.merge_duplicate_assets(), 1);
+        assert_eq!(d.assets.keys().copied().collect::<Vec<_>>(), (1..=7).map(AssetId).collect::<Vec<_>>());
+    }
+
+    #[test]
+    #[cfg(not(target_arch = "wasm32"))]
+    #[ignore = "manual bounded timing comparison; no timing assertion"]
+    fn equal_size_assets_timing() {
+        for count in [500, 1000, 2000] {
+            let mut d = Document::new(&Default::default());
+            for id in 1..=count as u64 {
+                let mut bytes = vec![42; 8192];
+                bytes[8184..].copy_from_slice(&id.to_le_bytes());
+                d.assets.insert(AssetId(id), Arc::new(asset(id, &bytes)));
+            }
+            let start = std::time::Instant::now();
+            assert_eq!(d.merge_duplicate_assets(), 0);
+            eprintln!("{count} distinct 8 KiB assets: {:?}", start.elapsed());
+        }
+    }
+
     fn graphic_item(id: u64, asset: u64, crop: f64) -> Item {
         let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0)));
         it.content = Content::Graphic(Graphic {

crates/format/src/lib.rs

@@ -13,6 +13,7 @@
 #![forbid(unsafe_code)]
 
 use std::collections::{HashMap, HashSet};
+use std::hash::BuildHasher;
 use std::io::{Cursor, Read, Write};
 use std::sync::Arc;
 
@@ -155,12 +156,13 @@ fn load_zip(bytes: &[u8]) -> Result<Document, FormatError> {
     let names: Vec<String> = z.file_names().map(str::to_string).collect();
     // Files saved with one copy per placement: identical copies share one buffer as they are
     // read, so memory holds each distinct file once.
-    let mut loaded: HashMap<usize, Vec<Arc<Vec<u8>>>> = HashMap::new();
+    let fingerprint = std::collections::hash_map::RandomState::new();
+    let mut loaded: HashMap<u64, Vec<Arc<Vec<u8>>>> = HashMap::new();
     for name in names {
         let Some(stem) = name.strip_prefix("assets/") else { continue };
         let Some(id) = stem.split('.').next().and_then(|s| s.parse::<u64>().ok()) else { continue };
         if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) {
-            let same = loaded.entry(data.len()).or_default();
+            let same = loaded.entry(fingerprint.hash_one(data.as_slice())).or_default();
             let data = match same.iter().find(|d| ***d == data) {
                 Some(d) => d.clone(),
                 None => {
@@ -568,6 +570,33 @@ mod tests {
         }
     }
 
+    #[test]
+    fn equal_size_archive_assets_preserve_content_and_share_only_identical_buffers() {
+        let mut d = Document::new(&NewDocument::default());
+        let mut pairs = Vec::new();
+        for n in 0..128u64 {
+            let mut bytes = vec![42; 1024];
+            bytes[1016..].copy_from_slice(&n.to_le_bytes());
+            let a = png_asset(&mut d, &bytes);
+            let b = png_asset(&mut d, &bytes);
+            // Distinct links must remain independently relinkable even when their bytes match.
+            Arc::make_mut(d.assets.get_mut(&a).unwrap()).link = Some(format!("/a/{n}.png"));
+            Arc::make_mut(d.assets.get_mut(&b).unwrap()).link = Some(format!("/b/{n}.png"));
+            place(&mut d, a, 1.0);
+            place(&mut d, b, 2.0);
+            pairs.push((a, b));
+        }
+        let back = load(&save_uncompacted(&d)).unwrap();
+        assert_eq!(back.assets.len(), 256);
+        for (n, (a, b)) in pairs.into_iter().enumerate() {
+            assert_eq!(back.assets[&a].data[1016..], (n as u64).to_le_bytes());
+            assert!(Arc::ptr_eq(&back.assets[&a].data, &back.assets[&b].data));
+            assert_eq!(back.assets[&a].link, d.assets[&a].link);
+            assert_eq!(back.assets[&b].link, d.assets[&b].link);
+        }
+        assert_eq!(back.spreads, d.spreads);
+    }
+
     #[test]
     fn older_files_with_one_asset_per_placement_open_with_one() {
         let mut d = Document::new(&NewDocument::default());

docs/file-format-parity.md

@@ -1,6 +1,6 @@
 # File format parity
 
-> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (.indd and .indt open through indd-utils; XLSX cached values preserve numeric precision and text; shared print spooler; Windows CI passes) · **Target:** Adobe InDesign 2026 (21.6)
+> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (.indd and .indt open through indd-utils; XLSX cached values preserve numeric precision and text; shared print spooler; Windows CI passes; content-indexed asset deduplication, #727) · **Target:** Adobe InDesign 2026 (21.6)
 
 Every format InDesign 2026 reads or writes, with DesignCraft's support, fidelity and tests. The
 InDesign list comes from its bundle's `Info.plist` (`CFBundleDocumentTypes`,
@@ -22,7 +22,7 @@ score below).
 | InDesign document | `.indd` | read/write (native) | ✓ (converted to IDML in memory by [storytold/indd-utils](https://github.com/storytold/indd-utils), then the IDML importer; #21, #114) | ✗ | Converter warnings join the open result; unreadable files are open errors. Synthetic-bytes tests only; indd-utils reports 96.9% of IDML values reproduced on ~650 pairs. No real-file corpus test | 30% |
 | InDesign template | `.indt` | read/write | ✓ (as `.indd`) | ✗ | As `.indd` | 2% |
 | InDesign Markup | `.idml` | read/write | ✓ | ✓ | Synthetic fixtures in `crates/idml/src/tests.rs` (51 tests); exported files verified by opening in InDesign 2026 by hand; indd-utils' 98-document run imports 811 spreads / 2,866 stories / 5,812 items. Open: #193, #338, #339, #340, #211. No real-file corpus test | 20% |
-| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests | — |
+| DesignCraft native | `.designcraft` | — | ✓ | ✓ | zip + JSON, versioned; round-trip tests. Asset loading and compaction index by content hash, then confirm byte equality; equal-sized distinct files fall into separate search buckets (#727). Separate links, MIME types and PDF pages retain their identity | — |
 | InDesign book | `.indb` | read/write | ✗ (own `.dcbook`) | ✗ | Books are shallow (see gaps.md) | 2% |
 | InDesign library | `.indl`, `.inlx` | read/write | ✗ (own `.dclib`) | ✗ | | 1% |
 | Snippet | `.idms` | read/write | ✗ (own snippets) | ✗ | | 1% |
@@ -108,6 +108,7 @@ up to 30 points.
 
 | Date | Change | Summary |
 |---|---|---|
+| 2026-10-13 | minor | Asset loading and compaction use content fingerprints with exact equality checks; tests cover equal-sized distinct buffers, shared copies, independent links and missing data (#727) |
 | 2026-10-13 | minor | `.indd` and `.indt` read through storytold/indd-utils; INDD remaining work is a real-file corpus (10–30 h); score unchanged until it is measured |
 | 2026-10-12 | minor | XLSX table placement and data merge preserve small numeric values and cached formula strings; synthetic regressions in `textimport::xlsx::tests`. Scores unchanged. |
 | 2026-10-10 | major | Created from InDesign 2026's declared document types and filters, `plan/indesign/08-file-formats.md` and the code |