From daf1a0ccb3875a6435a1d56cd4cc18c199847738 Mon Sep 17 00:00:00 2001 From: aiongg Date: Sat, 10 Oct 2026 23:28:24 +0000 Subject: [PATCH] Assets: one asset per image, and saves leave out unused ones Placing a file that the document already holds reuses its asset (same bytes, type and PDF page), in file.place, place.load, media and poster placement, graphic table cells, math, data merge image fields, IDML import (a linked file is read once) and paste/duplicate between documents (which also maps ids that collide in the target). Saving merges identical assets into the first and leaves out assets nothing references; the open document keeps them for undo. Opening a file saved with one copy per placement shares the identical bytes as they are read and merges the assets. Package and Copy Links write only used files, and links.list lists only used assets. Refs #391 --- crates/doc/src/assets.rs | 342 ++++++++++++++++++++++++ crates/doc/src/lib.rs | 2 + crates/engine/src/cmd/datamerge/fill.rs | 41 +-- crates/engine/src/cmd/file.rs | 12 +- crates/engine/src/cmd/links.rs | 85 ++++++ crates/engine/src/cmd/mathexpr.rs | 15 +- crates/engine/src/cmd/media.rs | 9 +- crates/engine/src/cmd/object.rs | 20 +- crates/engine/src/cmd/package.rs | 6 +- crates/engine/src/cmd/table.rs | 16 +- crates/format/src/lib.rs | 132 +++++++-- crates/idml/src/import.rs | 41 ++- crates/idml/src/tests.rs | 1 + crates/idml/src/tests/assets.rs | 107 ++++++++ 14 files changed, 747 insertions(+), 82 deletions(-) create mode 100644 crates/doc/src/assets.rs create mode 100644 crates/idml/src/tests/assets.rs diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rs new file mode 100644 index 000000000..73f7ac98d --- /dev/null +++ b/crates/doc/src/assets.rs @@ -0,0 +1,342 @@ +//! Placed files (assets): one asset per distinct file, the assets a document uses, and the +//! compaction a save applies (duplicates merged, unused assets left out). +//! +//! An asset holds what is shared by every placement of a file (bytes, link, type, the PDF page +//! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`]. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; + +use crate::story::Story; +use crate::{Asset, AssetId, Content, Document, Item}; + +/// Nesting limit for groups, anchored objects, table cells and notes. A document nested deeper +/// is treated as using every asset, so nothing it references is merged away or pruned. +const MAX_DEPTH: usize = 64; + +impl Asset { + /// Whether `other` holds the same file: the same type, page and bytes. Assets without bytes + /// (a link that couldn't be read) are the same only when they link to the same path. + pub fn same_file(&self, other: &Asset) -> bool { + if self.mime != other.mime || self.page != other.page { + return false; + } + match (self.data.is_empty(), other.data.is_empty()) { + (false, false) => Arc::ptr_eq(&self.data, &other.data) || (self.data.len() == other.data.len() && *self.data == *other.data), + (true, true) => self.link.is_some() && self.link == other.link, + _ => false, + } + } +} + +/// The asset in `assets` holding the same file as `asset`. +pub fn find_same_asset(assets: &BTreeMap>, asset: &Asset) -> Option { + assets.values().find(|a| a.same_file(asset)).map(|a| a.id) +} + +/// Add `asset` to `assets` unless they already hold the same file; returns the id placements +/// should reference. A reused asset without a link takes the new asset's link. +pub fn insert_asset(assets: &mut BTreeMap>, asset: Asset) -> AssetId { + if let Some(id) = find_same_asset(assets, &asset) { + if let Some(a) = assets.get_mut(&id) + && a.link.is_none() + && asset.link.is_some() + { + Arc::make_mut(a).link = asset.link; + } + return id; + } + let id = asset.id; + assets.insert(id, Arc::new(asset)); + id +} + +/// Calls `f` with every asset reference in `it` and its children; false when nesting exceeded +/// [`MAX_DEPTH`] (some references weren't visited). +fn item_refs(it: &Item, f: &mut impl FnMut(AssetId), depth: usize) -> bool { + if depth > MAX_DEPTH { + return false; + } + if let Content::Graphic(g) = &it.content { + f(g.asset); + } + if let Some(p) = it.media.as_ref().and_then(|m| m.poster) { + f(p); + } + it.children().iter().all(|c| item_refs(c, f, depth + 1)) +} + +/// Asset references in a story's anchored objects, graphic cells and notes (recursively). +fn story_refs(st: &Story, f: &mut impl FnMut(AssetId), depth: usize) -> bool { + if depth > MAX_DEPTH { + return false; + } + let mut complete = st.objects.iter().all(|o| item_refs(&o.item, f, depth + 1)); + for t in st.tables.values() { + for c in &t.cells { + if let Some(g) = &c.graphic { + f(g.asset); + } + complete &= story_refs(&c.text, f, depth + 1); + } + } + for n in st.notes.iter().chain(&st.endnotes) { + complete &= story_refs(&n.text, f, depth + 1); + } + complete +} + +/// Every asset reference in a story (its anchored objects, graphic cells and notes). +pub fn story_asset_refs(st: &Story) -> Vec { + let mut out = Vec::new(); + story_refs(st, &mut |a| out.push(a), 0); + out +} + +/// Point a story's references to a key of `map` at its value. +pub fn remap_story_assets(st: &mut Story, map: &HashMap) { + if !map.is_empty() { + remap_story(st, map, 0); + } +} + +fn item_uses(it: &Item, map: &HashMap) -> bool { + let mut hit = false; + item_refs(it, &mut |a| hit |= map.contains_key(&a), 0); + hit +} + +fn story_uses(st: &Story, map: &HashMap) -> bool { + let mut hit = false; + story_refs(st, &mut |a| hit |= map.contains_key(&a), 0); + hit +} + +fn remap_id(id: &mut AssetId, map: &HashMap) { + if let Some(n) = map.get(id) { + *id = *n; + } +} + +/// Rewrites the references in `it`, copying only the shared items that change. +fn remap_item(it: &mut Arc, map: &HashMap, depth: usize) { + if depth > MAX_DEPTH || !item_uses(it, map) { + return; + } + remap_item_in(Arc::make_mut(it), map, depth); +} + +fn remap_item_in(it: &mut Item, map: &HashMap, depth: usize) { + if let Content::Graphic(g) = &mut it.content { + remap_id(&mut g.asset, map); + } + if let Some(p) = it.media.as_mut().and_then(|m| m.poster.as_mut()) { + remap_id(p, map); + } + if let Some(kids) = it.children_mut() { + for k in kids { + remap_item(k, map, depth + 1); + } + } +} + +fn remap_story(st: &mut Story, map: &HashMap, depth: usize) { + if depth > MAX_DEPTH { + return; + } + for o in &mut st.objects { + if item_uses(&o.item, map) { + remap_item_in(&mut Arc::make_mut(o).item, map, depth + 1); + } + } + for t in st.tables.values_mut() { + if !t.cells.iter().any(|c| c.graphic.as_ref().is_some_and(|g| map.contains_key(&g.asset)) || story_uses(&c.text, map)) { + continue; + } + for c in &mut Arc::make_mut(t).cells { + if let Some(g) = &mut c.graphic { + remap_id(&mut g.asset, map); + } + remap_story(&mut c.text, map, depth + 1); + } + } + for n in st.notes.iter_mut().chain(st.endnotes.iter_mut()) { + if story_uses(&n.text, map) { + remap_story(&mut Arc::make_mut(n).text, map, depth + 1); + } + } +} + +impl Document { + /// Add a placed file, reusing the asset of an identical file already in the document; + /// returns the id to reference. + pub fn add_asset(&mut self, asset: Asset) -> AssetId { + insert_asset(&mut self.assets, asset) + } + + /// Copy asset `id` of `src` into this document (or reuse the identical asset it already + /// has); returns the id to reference here, `None` when `src` has no such asset. + pub fn adopt_asset(&mut self, src: &Document, id: AssetId) -> Option { + let a = src.assets.get(&id)?; + if let Some(same) = find_same_asset(&self.assets, a) { + return Some(same); + } + let fresh = AssetId(self.alloc()); + Some(self.add_asset(Asset { id: fresh, ..(**a).clone() })) + } + + /// Calls `f` with every asset reference: graphics on document and parent spreads (in + /// groups and multi-state objects too), media posters, anchored and inline objects, graphic + /// table cells and notes. False when nesting exceeded the depth limit. + fn asset_refs(&self, f: &mut impl FnMut(AssetId)) -> bool { + let mut complete = true; + for sp in self.spreads.iter().chain(&self.parents) { + for it in &sp.items { + complete &= item_refs(it, f, 0); + } + } + for st in self.stories.values() { + complete &= story_refs(st, f, 0); + } + complete + } + + /// The assets something in the document references. When the document nests deeper than + /// the traversal limit, every asset counts as used. + pub fn used_assets(&self) -> BTreeSet { + let mut used = BTreeSet::new(); + if !self.asset_refs(&mut |a| { + used.insert(a); + }) { + return self.assets.keys().copied().collect(); + } + used + } + + /// Point every reference to a key of `map` at its value. + pub fn remap_assets(&mut self, map: &HashMap) { + if map.is_empty() { + return; + } + for sp in self.spreads.iter_mut().chain(self.parents.iter_mut()) { + if sp.items.iter().any(|it| item_uses(it, map)) { + for it in &mut Arc::make_mut(sp).items { + remap_item(it, map, 0); + } + } + } + for st in self.stories.values_mut() { + if story_uses(st, map) { + remap_story(Arc::make_mut(st), map, 0); + } + } + } + + /// Merge assets holding the same file into the first of them (lowest id), rewriting the + /// references; returns how many assets were removed. + pub fn merge_duplicate_assets(&mut self) -> usize { + if !self.asset_refs(&mut |_| {}) { + return 0; + } + let mut reps: HashMap<(usize, &str, u32), Vec<&Arc>> = HashMap::new(); + let mut map = HashMap::new(); + let mut links = Vec::new(); + for a in self.assets.values() { + let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default(); + match bucket.iter().find(|r| r.same_file(a)) { + Some(r) => { + map.insert(a.id, r.id); + if r.link.is_none() + && let Some(l) = &a.link + { + links.push((r.id, l.clone())); + } + } + None => bucket.push(a), + } + } + for (id, link) in links { + if let Some(a) = self.assets.get_mut(&id) + && a.link.is_none() + { + Arc::make_mut(a).link = Some(link); + } + } + self.remap_assets(&map); + self.assets.retain(|id, _| !map.contains_key(id)); + map.len() + } + + /// Merge duplicate assets and drop the ones nothing references (what a saved file holds). + pub fn compact_assets(&mut self) { + self.merge_duplicate_assets(); + let used = self.used_assets(); + self.assets.retain(|id, _| used.contains(id)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Graphic, ItemId, LayerId, Shape, StoryId}; + use designcraft_geom::{Affine, Rect, shapes}; + + fn asset(id: u64, data: &[u8]) -> Asset { + Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), link: None, data: Arc::new(data.to_vec()), pixels: None, page: 0 } + } + + fn graphic_item(id: u64, asset: u64, crop: f64) -> Item { + let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0))); + it.content = Content::Graphic(Graphic { + asset: AssetId(asset), + size: (10.0, 10.0), + xf: Affine::translate((crop, 0.0)), + auto_fit: Default::default(), + fit_align: 4, + crop: [crop; 4], + }); + it + } + + #[test] + fn identical_files_share_one_asset() { + let mut d = Document::new(&Default::default()); + let a = d.add_asset(asset(1, b"same")); + let b = d.add_asset(Asset { link: Some("/x/p.png".into()), ..asset(2, b"same") }); + let c = d.add_asset(asset(3, b"diff")); + let p = d.add_asset(Asset { page: 1, ..asset(4, b"same") }); + assert_eq!(a, b); + assert_ne!(a, c); + assert_ne!(a, p); + assert_eq!(d.assets.len(), 3); + assert_eq!(d.assets[&a].link.as_deref(), Some("/x/p.png")); + // Unreadable links match by path only. + let m1 = d.add_asset(Asset { link: Some("/m/1.png".into()), ..asset(5, b"") }); + let m2 = d.add_asset(Asset { link: Some("/m/2.png".into()), ..asset(6, b"") }); + let m3 = d.add_asset(asset(7, b"")); + let m4 = d.add_asset(asset(8, b"")); + assert_ne!(m1, m2); + assert_ne!(m3, m4); + } + + #[test] + fn compaction_merges_duplicates_and_drops_unused_assets() { + let mut d = Document::new(&Default::default()); + for (id, data) in [(1, b"one"), (2, b"one"), (3, b"two"), (4, b"old")] { + d.assets.insert(AssetId(id), Arc::new(asset(id, data))); + } + let sp = Arc::make_mut(d.spreads.first_mut().unwrap()); + sp.items.push(Arc::new(graphic_item(10, 1, 1.0))); + sp.items.push(Arc::new(graphic_item(11, 2, 2.0))); + // A graphic cell and an anchored object in a story. + let mut st = Story::new(StoryId(20)); + st.objects.push(Arc::new(crate::AnchoredObject::new(graphic_item(12, 3, 3.0), Default::default()))); + d.stories.insert(StoryId(20), Arc::new(st)); + assert_eq!(d.used_assets(), [1, 2, 3].map(AssetId).into_iter().collect()); + + d.compact_assets(); + assert_eq!(d.assets.keys().copied().collect::>(), vec![AssetId(1), AssetId(3)]); + let crops: Vec<(AssetId, f64)> = d.spreads[0].items.iter().filter_map(|i| i.graphic().map(|g| (g.asset, g.crop[0]))).collect(); + assert_eq!(crops, vec![(AssetId(1), 1.0), (AssetId(1), 2.0)]); + } +} diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index c4b69e86d..2c90d8b95 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -12,6 +12,7 @@ pub mod anchored; pub mod arabic; pub mod arrow; +pub mod assets; pub mod attrs; pub mod build; pub mod cjk; @@ -36,6 +37,7 @@ use std::collections::BTreeMap; use std::sync::Arc; pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK}; +pub use assets::{find_same_asset, insert_asset, remap_story_assets, story_asset_refs}; pub use attrs::*; pub use datamerge::{ DataField, DataFieldKind, DataMerge, DataSource, Delimiter, Fingerprint, MergeOptions, Placeholder, PlaceholderAnchor, PlaceholderRole, diff --git a/crates/engine/src/cmd/datamerge/fill.rs b/crates/engine/src/cmd/datamerge/fill.rs index deb0948bc..89dd0c74e 100644 --- a/crates/engine/src/cmd/datamerge/fill.rs +++ b/crates/engine/src/cmd/datamerge/fill.rs @@ -75,6 +75,11 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document let new_id = ItemId(dst.alloc()); maps.items.insert(old, new_id); it.id = new_id; + if let Some(m) = it.media.as_mut() + && let Some(p) = m.poster + { + m.poster = dst.adopt_asset(src, p).or(Some(p)); + } match &mut it.content { Content::Text(tf) => { let new_sid = if let Some(n) = maps.stories.get(&tf.story) { @@ -83,6 +88,15 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document let n = StoryId(dst.alloc()); let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n)); st.id = n; + let mut assets = HashMap::new(); + for a in designcraft_doc::story_asset_refs(&st) { + if let Some(na) = dst.adopt_asset(src, a) + && na != a + { + assets.insert(a, na); + } + } + designcraft_doc::remap_story_assets(&mut st, &assets); st.frames.clear(); dst.stories.insert(n, Arc::new(st)); maps.stories.insert(tf.story, n); @@ -94,8 +108,8 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document } } Content::Graphic(g) => { - if let Some(a) = src.assets.get(&g.asset) { - dst.assets.entry(g.asset).or_insert_with(|| a.clone()); + if let Some(a) = dst.adopt_asset(src, g.asset) { + g.asset = a; } } Content::Group { items } => { @@ -355,21 +369,18 @@ fn place_image( }; let (nw, nh) = (f64::from(pw), f64::from(ph)); let mode = fitting_of(&options.fitting); - let aid = AssetId(doc.alloc()); + let fresh = AssetId(doc.alloc()); let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| field.to_string()); let link = options.link_images.then(|| path.to_string_lossy().into_owned()); - doc.assets.insert( - aid, - Arc::new(Asset { - id: aid, - name, - mime: designcraft_render::image_mime(&bytes).to_string(), - link, - data: Arc::new(bytes), - pixels: Some((pw, ph)), - page: 0, - }), - ); + let aid = doc.add_asset(Asset { + id: fresh, + name, + mime: designcraft_render::image_mime(&bytes).to_string(), + link, + data: Arc::new(bytes), + pixels: Some((pw, ph)), + page: 0, + }); let mut g = Graphic { asset: aid, size: (nw, nh), xf: Affine::translate((inner.x0, inner.y0)), auto_fit: mode, fit_align: 0, crop: [0.0; 4] }; if let Some(xf) = g.fitted(inner, mode) { g.xf = xf; diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index f871a43e8..e7bc7e4ec 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -336,9 +336,9 @@ fn file_place(s: &mut Session, p: &Value) -> Result { let py = p.get("y").and_then(Value::as_f64); let want_w = p.get("width").and_then(Value::as_f64); s.edit(|d, sel| { - let aid = AssetId(d.alloc()); + let fresh = AssetId(d.alloc()); let mime = designcraft_render::image_mime(&bytes).to_string(); - d.assets.insert(aid, Arc::new(Asset { page: pdf_page, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) })); + let aid = d.add_asset(Asset { page: pdf_page, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) }); // The width the whole page (or image) gets; the height follows the proportions. let w = match want_w { Some(w) => w, @@ -441,13 +441,9 @@ fn place_load(s: &mut Session, p: &Value) -> Result { let (bytes, name, link) = read_source(p)?; let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad("place.load", "unsupported or corrupt image"))?; let aid = s.edit(|d, _| { - let aid = AssetId(d.alloc()); + let fresh = AssetId(d.alloc()); let mime = designcraft_render::image_mime(&bytes).to_string(); - d.assets.insert( - aid, - Arc::new(Asset { page: 0, id: aid, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) }), - ); - Ok(aid) + Ok(d.add_asset(Asset { page: 0, id: fresh, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) })) })?; s.loaded = Some((aid, (pw as f64, ph as f64))); s.set_tool("placeGun"); diff --git a/crates/engine/src/cmd/links.rs b/crates/engine/src/cmd/links.rs index c05c6dad9..687cd7b1a 100644 --- a/crates/engine/src/cmd/links.rs +++ b/crates/engine/src/cmd/links.rs @@ -112,8 +112,10 @@ pub fn status(a: &designcraft_doc::Asset) -> &'static str { } fn list(d: &Document) -> Vec { + let used = d.used_assets(); d.assets .values() + .filter(|a| used.contains(&a.id)) .map(|a| { let uses: Vec = uses(d, a.id) .into_iter() @@ -311,3 +313,86 @@ mod relink_folder_tests { let _ = std::fs::remove_dir_all(&dir); } } + +#[cfg(test)] +mod shared_asset_tests { + use super::*; + + fn png(w: u32, h: u32) -> Vec { + designcraft_render::Rendered { width: w, height: h, pixels: vec![200u8; (w * h * 4) as usize] }.to_png() + } + + #[test] + fn placements_of_one_file_share_an_asset_and_saves_drop_replaced_ones() { + let mut s = Session::new(); + s.execute("file.new", &json!({})).unwrap(); + let a = super::super::file::base64_encode(&png(10, 10)); + s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 72, "y": 72})).unwrap(); + s.execute("edit.deselectAll", &json!({})).unwrap(); + s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 200, "y": 72})).unwrap(); + let l = s.execute("links.list", &json!({})).unwrap(); + assert_eq!(l.as_array().unwrap().len(), 1); + assert_eq!(l[0]["uses"].as_array().unwrap().len(), 2); + assert_eq!(s.doc().unwrap().doc.assets.len(), 1); + + // A new image into a frame: the old asset stays in memory (undo) but not in the file. + let f = s.execute("frame.create", &json!({"rect": [100, 300, 200, 400]})).unwrap()["id"].as_u64().unwrap(); + let b = super::super::file::base64_encode(&png(20, 10)); + let c = super::super::file::base64_encode(&png(30, 10)); + s.execute("file.place", &json!({"base64": b, "name": "b.png", "frame": f})).unwrap(); + s.execute("file.place", &json!({"base64": c, "name": "c.png", "frame": f})).unwrap(); + let d = &s.doc().unwrap().doc; + assert_eq!(d.assets.len(), 3); + assert_eq!(s.execute("links.list", &json!({})).unwrap().as_array().unwrap().len(), 2); + let back = designcraft_format::load(&designcraft_format::save(&s.doc().unwrap().doc).unwrap()).unwrap(); + let mut names: Vec<&str> = back.assets.values().map(|a| a.name.as_str()).collect(); + names.sort(); + assert_eq!(names, ["a.png", "c.png"]); + } + + #[test] + fn pasted_graphics_reuse_the_identical_asset_of_the_target() { + use designcraft_doc::build::NewDocument; + use designcraft_doc::{Asset, Graphic, Item, ItemId, Shape, SpreadRef}; + let asset = |id: u64, data: Vec| Asset { + id: AssetId(id), + name: "p.png".into(), + mime: "image/png".into(), + link: None, + data: Arc::new(data), + pixels: None, + page: 0, + }; + let mut src = Document::new(&NewDocument::default()); + let mut dst = Document::new(&NewDocument::default()); + let (pa, pb) = (png(4, 4), png(8, 4)); + // The source's asset id is taken in the target by a different file. + let sid = AssetId(src.alloc()); + src.assets.insert(sid, Arc::new(asset(sid.0, pa.clone()))); + dst.assets.insert(sid, Arc::new(asset(sid.0, pb))); + dst.alloc(); + let same = AssetId(dst.alloc()); + assert_ne!(same, sid); + dst.assets.insert(same, Arc::new(asset(same.0, pa))); + let iid = ItemId(src.alloc()); + let mut it = Item::new( + iid, + src.default_layer(), + Shape::Rectangle, + designcraft_geom::shapes::rectangle(designcraft_geom::Rect::new(0.0, 0.0, 9.0, 9.0)), + ); + it.content = Content::Graphic(Graphic { + asset: sid, + size: (4.0, 4.0), + xf: designcraft_geom::Affine::IDENTITY, + auto_fit: Default::default(), + fit_align: 4, + crop: [1.0; 4], + }); + src.insert_item(SpreadRef::Doc(0), it, None).unwrap(); + let ids = super::super::object::duplicate_from(&mut dst, &src, &[iid], SpreadRef::Doc(0), designcraft_geom::Vec2::ZERO).unwrap(); + let g = dst.item(ids[0]).and_then(|i| i.graphic()).unwrap(); + assert_eq!((g.asset, g.crop), (same, [1.0; 4])); + assert_eq!(dst.assets.len(), 2); + } +} diff --git a/crates/engine/src/cmd/mathexpr.rs b/crates/engine/src/cmd/mathexpr.rs index 791a0d704..ec90d52e3 100644 --- a/crates/engine/src/cmd/mathexpr.rs +++ b/crates/engine/src/cmd/mathexpr.rs @@ -13,11 +13,16 @@ use crate::{Result, Session}; fn math_item(d: &mut designcraft_doc::Document, latex: &str, size: f64, lid: designcraft_doc::LayerId, at: (f64, f64)) -> Item { let (svg, (w, h)) = crate::math::to_svg(latex, size); - let aid = AssetId(d.alloc()); - d.assets.insert( - aid, - Arc::new(Asset { page: 0, id: aid, name: "math.svg".into(), mime: "image/svg+xml".into(), link: None, data: Arc::new(svg), pixels: None }), - ); + let fresh = AssetId(d.alloc()); + let aid = d.add_asset(Asset { + page: 0, + id: fresh, + name: "math.svg".into(), + mime: "image/svg+xml".into(), + link: None, + data: Arc::new(svg), + pixels: None, + }); let id = ItemId(d.alloc()); let r = Rect::new(at.0, at.1, at.0 + w, at.1 + h); let mut it = Item::new(id, lid, Shape::Rectangle, shapes::rectangle(r)); diff --git a/crates/engine/src/cmd/media.rs b/crates/engine/src/cmd/media.rs index b80089585..b6e4cd4b8 100644 --- a/crates/engine/src/cmd/media.rs +++ b/crates/engine/src/cmd/media.rs @@ -25,8 +25,8 @@ pub(crate) fn place_media(s: &mut Session, p: &Value, name: String, mime: &str, let (px, py) = (p.get("x").and_then(Value::as_f64), p.get("y").and_then(Value::as_f64)); let mime = mime.to_string(); s.edit(|d, sel| { - let aid = AssetId(d.alloc()); - d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: None })); + let fresh = AssetId(d.alloc()); + let aid = d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: None }); let id = match target { Some(fid) => { let it = d.item_mut(fid).ok_or(designcraft_doc::DocError::NoItem(fid))?; @@ -90,10 +90,9 @@ pub fn specs() -> Vec { s.edit(|d, _| { let poster_id = match poster { Some(Some((bytes, name, link, px))) => { - let aid = AssetId(d.alloc()); + let fresh = AssetId(d.alloc()); let mime = designcraft_render::image_mime(&bytes).to_string(); - d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some(px) })); - Some(Some(aid)) + Some(Some(d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some(px) }))) } Some(None) => Some(None), None => None, diff --git a/crates/engine/src/cmd/object.rs b/crates/engine/src/cmd/object.rs index 17c44c27a..466c15914 100644 --- a/crates/engine/src/cmd/object.rs +++ b/crates/engine/src/cmd/object.rs @@ -900,6 +900,11 @@ pub fn duplicate_from(dst: &mut Document, src: &Document, ids: &[ItemId], to: Sp fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut HashMap) { it.id = ItemId(dst.alloc()); let my_id = it.id; + if let Some(m) = it.media.as_mut() + && let Some(p) = m.poster + { + m.poster = dst.adopt_asset(src, p).or(Some(p)); + } match &mut it.content { Content::Text(tf) => { let new_sid = match stories.get(&tf.story) { @@ -908,6 +913,15 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Has let n = StoryId(dst.alloc()); let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n)); st.id = n; + let mut assets = HashMap::new(); + for a in designcraft_doc::story_asset_refs(&st) { + if let Some(na) = dst.adopt_asset(src, a) + && na != a + { + assets.insert(a, na); + } + } + designcraft_doc::remap_story_assets(&mut st, &assets); st.frames.clear(); dst.stories.insert(n, Arc::new(st)); stories.insert(tf.story, n); @@ -920,10 +934,8 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Has } } Content::Graphic(g) => { - if let Some(a) = src.assets.get(&g.asset) - && !dst.assets.contains_key(&g.asset) - { - dst.assets.insert(g.asset, a.clone()); + if let Some(a) = dst.adopt_asset(src, g.asset) { + g.asset = a; } } Content::Group { items } => { diff --git a/crates/engine/src/cmd/package.rs b/crates/engine/src/cmd/package.rs index 468667510..25f09537c 100644 --- a/crates/engine/src/cmd/package.rs +++ b/crates/engine/src/cmd/package.rs @@ -285,13 +285,13 @@ fn write_links(d: &Document, root: &OutputRoot, dir: &Path, only: Option<&[Asset let mut out = d.clone(); let mut taken = Vec::new(); let mut written = Vec::new(); - let mut ids: Vec = d.assets.keys().copied().collect(); - ids.sort_by_key(|i| i.0); + // Only files something places; the saved document leaves the others out. + let ids = d.used_assets(); for id in ids { if only.is_some_and(|o| !o.contains(&id)) { continue; } - let a = &d.assets[&id]; + let Some(a) = d.assets.get(&id) else { continue }; let name = link_name(a.link.as_deref().unwrap_or(&a.name), id, &mut taken); let path = dir.join(&name); root.write(&path, a.data.as_slice())?; diff --git a/crates/engine/src/cmd/table.rs b/crates/engine/src/cmd/table.rs index 028971e97..fafece0df 100644 --- a/crates/engine/src/cmd/table.rs +++ b/crates/engine/src/cmd/table.rs @@ -1402,20 +1402,10 @@ fn place_graphic(s: &mut Session, p: &Value) -> Result { let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad(ID, "unsupported or corrupt image"))?; let fill = p.get("fit").and_then(Value::as_str) == Some("fill"); s.edit(|d, sel| { - let aid = designcraft_doc::AssetId(d.alloc()); + let fresh = designcraft_doc::AssetId(d.alloc()); let mime = designcraft_render::image_mime(&bytes).to_string(); - d.assets.insert( - aid, - std::sync::Arc::new(designcraft_doc::Asset { - page: 0, - id: aid, - name, - mime, - link, - data: std::sync::Arc::new(bytes), - pixels: Some((pw, ph)), - }), - ); + let aid = + d.add_asset(designcraft_doc::Asset { page: 0, id: fresh, name, mime, link, data: std::sync::Arc::new(bytes), pixels: Some((pw, ph)) }); let st = d.story_mut(g.story).ok_or_else(|| bad(ID, "no story"))?; let t = st.table_mut(g.table).ok_or_else(|| bad(ID, "no table"))?; let (r, c) = t.owner(g.range.r0, g.range.c0); diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index f80ad0a65..2de45402d 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -2,14 +2,15 @@ //! //! - `mimetype` (stored first, uncompressed): `application/vnd.designcraft+zip` //! - `document.json`: the [`Document`] (pretty JSON, documented by the serde model in `designcraft-doc`) -//! - `assets/.`: embedded placed files, byte-for-byte +//! - `assets/.`: embedded placed files, byte-for-byte: one per distinct file, and only +//! the ones the document references (see [`Document::compact_assets`]) //! - `meta.json`: format version and generator //! //! Older single-file JSON documents (assets as base64 in `assetData`) still open. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::io::{Cursor, Read, Write}; use std::sync::Arc; @@ -42,8 +43,12 @@ fn ext_for(mime: &str) -> &'static str { } } -/// Serialize a document to `.designcraft` bytes. +/// Serialize a document to `.designcraft` bytes. The file holds each distinct placed file once +/// and leaves out assets nothing references; `doc` itself keeps them (undo can bring them back). pub fn save(doc: &Document) -> Result, FormatError> { + let mut compact = doc.clone(); + compact.compact_assets(); + let doc = &compact; let io = |e: zip::result::ZipError| FormatError::Io(e.to_string()); let mut buf = Cursor::new(Vec::new()); { @@ -100,13 +105,26 @@ fn load_zip(bytes: &[u8]) -> Result { upgrade(&mut v); let mut doc: Document = serde_json::from_value(v).map_err(|e| FormatError::NotOurs(e.to_string()))?; let names: Vec = z.file_names().map(str::to_string).collect(); + // Files saved with one copy per placement: identical copies share one buffer as they are + // read, so memory holds each distinct file once. + let mut loaded: HashMap>>> = HashMap::new(); for name in names { let Some(stem) = name.strip_prefix("assets/") else { continue }; let Some(id) = stem.split('.').next().and_then(|s| s.parse::().ok()) else { continue }; if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) { - Arc::make_mut(a).data = Arc::new(data); + let same = loaded.entry(data.len()).or_default(); + let data = match same.iter().find(|d| ***d == data) { + Some(d) => d.clone(), + None => { + let d = Arc::new(data); + same.push(d.clone()); + d + } + }; + Arc::make_mut(a).data = data; } } + doc.merge_duplicate_assets(); doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?; Ok(doc) } @@ -124,6 +142,7 @@ fn load_legacy_json(bytes: &[u8]) -> Result { } } } + doc.merge_duplicate_assets(); doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?; Ok(doc) } @@ -205,24 +224,67 @@ mod tests { use designcraft_doc::geom::Rect; use designcraft_doc::{Asset, ParaFormat, SpreadRef}; + fn png_asset(d: &mut Document, data: &[u8]) -> AssetId { + let aid = AssetId(d.alloc()); + let a = Asset { + page: 0, + id: aid, + name: "x.png".into(), + mime: "image/png".into(), + link: None, + data: Arc::new(data.to_vec()), + pixels: Some((1, 1)), + }; + d.assets.insert(aid, Arc::new(a)); + aid + } + + fn place(d: &mut Document, asset: AssetId, crop: f64) -> designcraft_doc::ItemId { + use designcraft_doc::geom::{Affine, shapes}; + use designcraft_doc::{Content, Graphic, Item, ItemId, Shape}; + let id = ItemId(d.alloc()); + let mut it = Item::new(id, d.default_layer(), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 50.0, 50.0))); + it.content = Content::Graphic(Graphic { + asset, + size: (1.0, 1.0), + xf: Affine::translate((crop, crop)), + auto_fit: Default::default(), + fit_align: 4, + crop: [crop; 4], + }); + d.insert_item(SpreadRef::Doc(0), it, None).unwrap(); + id + } + + fn asset_entries(bytes: &[u8]) -> usize { + let z = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + z.file_names().filter(|n| n.starts_with("assets/")).count() + } + + /// Zip `d` the way files were saved with one asset per placement (nothing merged or pruned). + fn save_uncompacted(d: &Document) -> Vec { + let mut buf = Cursor::new(Vec::new()); + let mut z = zip::ZipWriter::new(&mut buf); + let opts = zip::write::SimpleFileOptions::default(); + z.start_file("mimetype", opts).unwrap(); + z.write_all(MIME.as_bytes()).unwrap(); + z.start_file("document.json", opts).unwrap(); + z.write_all(&serde_json::to_vec(d).unwrap()).unwrap(); + for (id, a) in &d.assets { + z.start_file(format!("assets/{}.png", id.0), opts).unwrap(); + z.write_all(&a.data).unwrap(); + } + z.finish().unwrap(); + buf.into_inner() + } + #[test] fn roundtrip_with_assets() { let mut d = Document::new(&NewDocument { pages: 3, ..Default::default() }); let lid = d.default_layer(); d.add_text_frame(SpreadRef::Doc(0), Rect::new(10.0, 10.0, 200.0, 200.0), lid, "héllo\nworld", ParaFormat::default()).unwrap(); - let aid = AssetId(d.alloc()); - d.assets.insert( - aid, - Arc::new(Asset { - page: 0, - id: aid, - name: "x.png".into(), - mime: "image/png".into(), - link: None, - data: Arc::new(vec![1, 2, 3, 4]), - pixels: Some((1, 1)), - }), - ); + let aid = png_asset(&mut d, &[1, 2, 3, 4]); + place(&mut d, aid, 0.0); let bytes = save(&d).unwrap(); assert!(bytes.starts_with(b"PK")); let back = load(&bytes).unwrap(); @@ -231,6 +293,42 @@ mod tests { assert_eq!(back.stories, d.stories); } + #[test] + fn saves_hold_each_used_file_once() { + let mut d = Document::new(&NewDocument::default()); + let first = png_asset(&mut d, &[7; 64]); + let copy = png_asset(&mut d, &[7; 64]); + let replaced = png_asset(&mut d, &[9; 64]); + let a = place(&mut d, first, 1.0); + let b = place(&mut d, copy, 2.0); + assert_eq!(d.assets.len(), 3); + + // Duplicates merge into the first asset, the unreferenced one is left out, and every + // placement keeps its own crop and transform. + let bytes = save(&d).unwrap(); + assert_eq!(asset_entries(&bytes), 1); + assert_eq!(d.assets.len(), 3, "the open document keeps its assets"); + let back = load(&bytes).unwrap(); + assert_eq!(back.assets.keys().copied().collect::>(), vec![first]); + assert!(!back.assets.contains_key(&replaced)); + for (id, crop) in [(a, 1.0), (b, 2.0)] { + let g = back.item(id).and_then(|i| i.graphic()).unwrap(); + assert_eq!((g.asset, g.crop, g.xf), (first, [crop; 4], d.item(id).unwrap().graphic().unwrap().xf)); + } + } + + #[test] + fn older_files_with_one_asset_per_placement_open_with_one() { + let mut d = Document::new(&NewDocument::default()); + let ids: Vec = (0..3).map(|_| png_asset(&mut d, &[5; 32])).collect(); + for (k, id) in ids.iter().enumerate() { + place(&mut d, *id, k as f64); + } + let back = load(&save_uncompacted(&d)).unwrap(); + assert_eq!(back.assets.len(), 1); + assert_eq!(asset_entries(&save(&back).unwrap()), 1); + } + /// The object of the item `id` in a document's JSON. fn item_json(v: &mut Value, id: u64) -> Option<&mut Value> { if v.get("id").and_then(Value::as_u64) == Some(id) && v.get("content").is_some() { diff --git a/crates/idml/src/import.rs b/crates/idml/src/import.rs index d85d9ee02..138775268 100644 --- a/crates/idml/src/import.rs +++ b/crates/idml/src/import.rs @@ -132,6 +132,8 @@ struct Importer<'r> { page_index: HashMap, item_ids: HashMap, assets: BTreeMap>, + /// Assets read from a link path, so later placements of that path reuse them unread. + linked_assets: HashMap, ctx: ItemCtx, sections: Vec
, footnote_options: designcraft_doc::FootnoteOptions, @@ -217,6 +219,7 @@ impl<'r> Importer<'r> { page_index: HashMap::new(), item_ids: HashMap::new(), assets: BTreeMap::new(), + linked_assets: HashMap::new(), ctx: ItemCtx { threads: Vec::new() }, sections: Vec::new(), footnote_options: Default::default(), @@ -2205,12 +2208,29 @@ impl<'r> Importer<'r> { let size = ((r - l).abs(), (b - t).abs()); let link = g.find("Link"); let uri = link.and_then(|k| k.get("LinkResourceURI")).map(uri_to_path); + let graphic = |asset| { + Content::Graphic(designcraft_doc::Graphic { + asset, + size, + xf: gxf * Affine::translate((l, t)), + auto_fit: Default::default(), + fit_align: 4, + crop: [0.0; 4], + }) + }; let mut data = g.prop_el("Contents").map(|c| base64_decode(&c.text_content())).unwrap_or_default(); + let mut read_from = None; if data.is_empty() && let Some(p) = &uri - && let Some(d) = (self.read_link)(p) { - data = d; + // Every placement of a linked file shares one asset; read the file once. + if let Some(id) = self.linked_assets.get(p) { + return graphic(*id); + } + if let Some(d) = (self.read_link)(p) { + data = d; + read_from = Some(p.clone()); + } } let (mime, px) = sniff_image(&data); let mime = mime.map(str::to_string).unwrap_or_else(|| match link.and_then(|k| k.get("LinkResourceFormat")).unwrap_or("") { @@ -2226,17 +2246,14 @@ impl<'r> Importer<'r> { }); let name = uri.as_deref().and_then(|p| p.rsplit(['/', '\\']).next()).filter(|n| !n.is_empty()).map(str::to_string).unwrap_or_else(|| "image".into()); - let id = AssetId(self.alloc()); + let fresh = AssetId(self.alloc()); let link_path = uri.filter(|p| p.contains('/') || p.contains('\\')); - self.assets.insert(id, Arc::new(Asset { page: 0, id, name, mime, link: link_path, data: Arc::new(data), pixels })); - Content::Graphic(designcraft_doc::Graphic { - asset: id, - size, - xf: gxf * Affine::translate((l, t)), - auto_fit: Default::default(), - fit_align: 4, - crop: [0.0; 4], - }) + let id = + designcraft_doc::insert_asset(&mut self.assets, Asset { page: 0, id: fresh, name, mime, link: link_path, data: Arc::new(data), pixels }); + if let Some(p) = read_from { + self.linked_assets.insert(p, id); + } + graphic(id) } fn link_threads(&mut self) { diff --git a/crates/idml/src/tests.rs b/crates/idml/src/tests.rs index 7b747587a..4c7817009 100644 --- a/crates/idml/src/tests.rs +++ b/crates/idml/src/tests.rs @@ -7,6 +7,7 @@ use designcraft_geom::{Rect, shapes}; use super::*; +mod assets; mod decorations; fn zip_files(files: &[(&str, &str)]) -> Vec { diff --git a/crates/idml/src/tests/assets.rs b/crates/idml/src/tests/assets.rs new file mode 100644 index 000000000..a5cd90fb0 --- /dev/null +++ b/crates/idml/src/tests/assets.rs @@ -0,0 +1,107 @@ +//! Placed files: every placement of one file shares an asset. + +use std::cell::Cell; + +use designcraft_doc::{AssetId, Content}; + +use super::*; + +const PHOTO: &str = "file:/photos/wedding%20day/couple.jpg"; + +/// A graphic frame at `x` showing `image` (an `` element) with `crop` on every side. +fn frame(n: usize, x: f64, crop: f64, image: &str) -> String { + format!( + r#" + + + + + {image} + "# + ) +} + +fn image(n: usize, uri: &str, contents: Option<&[u8]>) -> String { + let contents = contents.map(|c| format!("", base64_encode(c))).unwrap_or_default(); + format!( + r#" + {contents} + + "# + ) +} + +fn package(frames: &[String]) -> Vec { + let spread = format!( + r#" + + + + {} + +"#, + frames.join("\n") + ); + zip_files(&[ + ("designmap.xml", DESIGNMAP), + ("Resources/Graphic.xml", GRAPHIC), + ("Resources/Styles.xml", STYLES), + ("Resources/Preferences.xml", PREFS), + ("MasterSpreads/MasterSpread_m1.xml", MASTER), + ("Spreads/Spread_sp1.xml", &spread), + ("Stories/Story_s1.xml", STORY), + ]) +} + +/// (asset, crop, transform) of each graphic, left to right. +fn placements(d: &Document) -> Vec<(AssetId, [f64; 4], designcraft_geom::Affine)> { + let mut out: Vec<(f64, AssetId, [f64; 4], designcraft_geom::Affine)> = d.spreads[0] + .items + .iter() + .filter_map(|i| match &i.content { + Content::Graphic(g) => Some((i.xf.translation().x, g.asset, g.crop, g.xf)), + _ => None, + }) + .collect(); + out.sort_by(|a, b| a.0.total_cmp(&b.0)); + out.into_iter().map(|(_, a, c, x)| (a, c, x)).collect() +} + +#[test] +fn one_file_placed_several_times_imports_as_one_asset() { + let bytes = vec![42u8; 300]; + let reads = Cell::new(0); + let reader = |p: &str| { + reads.set(reads.get() + 1); + p.ends_with("couple.jpg").then(|| bytes.clone()) + }; + // Three placements linking the file, and one embedding the same bytes under another path. + let frames = vec![ + frame(1, 0.0, 1.0, &image(1, PHOTO, None)), + frame(2, 100.0, 2.0, &image(2, PHOTO, None)), + frame(3, 200.0, 3.0, &image(3, PHOTO, None)), + frame(4, 300.0, 4.0, &image(4, "file:/elsewhere/copy.jpg", Some(&bytes))), + ]; + let d = import_idml_with(&package(&frames), &reader).unwrap(); + assert_eq!(reads.get(), 1, "the linked file is read once"); + assert_eq!(d.assets.len(), 1); + let a = d.assets.values().next().unwrap(); + assert_eq!(a.link.as_deref(), Some("/photos/wedding day/couple.jpg")); + let p = placements(&d); + assert_eq!(p.iter().map(|(_, c, _)| c[0]).collect::>(), vec![1.0, 2.0, 3.0, 4.0]); + assert!(p.iter().all(|(id, _, _)| *id == a.id)); + + // Exported linked or embedded, the round trip keeps one asset and each placement's crop and + // transform. + for embed_images in [false, true] { + let out = export_idml_with(&d, &ExportOptions { embed_images }); + let back = import_idml_with(&out, &reader).unwrap(); + assert_eq!(back.assets.len(), 1, "embed_images: {embed_images}"); + let q = placements(&back); + assert_eq!(q.len(), 4); + for ((_, c0, x0), (_, c1, x1)) in p.iter().zip(&q) { + assert_eq!(c0, c1); + assert!(x0.as_coeffs().iter().zip(x1.as_coeffs()).all(|(a, b)| (a - b).abs() < 1e-6), "{x0:?} {x1:?}"); + } + } +}