From 330f80fcf5422d08e3fda41a30bec971d811c623 Mon Sep 17 00:00:00 2001 From: Andrii Volkov <60794004+volkovskey@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:46:08 +0300 Subject: [PATCH] fix(format): open large .fig files A real 705 MB .fig file (401 644 nodes, 894 004 layers once instances are expanded) didn't open: "message: too many values". Lifting that cap alone made it import only its first 200 000 layers, and lifting that one took 41 GB of memory. The global caps stay as they are. The .fig import gets budgets from the size of its compressed canvas instead (counted up to the file's own size), not from the message it unpacks to, since chunks inflate up to 512 MB: - the kiwi value budget is one value per 2 bytes of the canvas, at least the old 20 million (kiwi is only read for .fig files); - the layer cap is one layer per 32 bytes of the canvas, at least the REST import's 200 000 and at most 2 000 000. A small or crafted file can't expand its instances into millions of layers, however far it unpacks; the REST JSON import keeps 200 000. Memory: - The mapper no longer passes the document through `json!`, which serialises its arguments: the whole tree was deep-copied twice. - Vector networks are built once per distinct network and handed to the REST importer beside the JSON (`vectorNetworkRef`), not written into it as JSON objects for every instance that shows them. - Decoded kiwi objects are trimmed, and each import stage's input is dropped as soon as the next stage is built. On that file: peak footprint 41 GB -> 11.3 GB, 67 s -> 25 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YQxjkNSUJhTkTfY74aNeki --- crates/format/src/fig/kiwi.rs | 24 ++- crates/format/src/fig/map.rs | 165 ++++++++++++------ .../map_connector_snapshot_budget_tests.rs | 8 +- crates/format/src/fig/mod.rs | 30 +++- crates/format/src/fig/tail_tests.rs | 4 +- crates/format/src/fig/tests.rs | 61 ++++++- crates/format/src/rest/mod.rs | 66 +++++-- crates/format/src/rest/tests.rs | 15 ++ 8 files changed, 287 insertions(+), 86 deletions(-) diff --git a/crates/format/src/fig/kiwi.rs b/crates/format/src/fig/kiwi.rs index 528dbbf0..46a9daac 100644 --- a/crates/format/src/fig/kiwi.rs +++ b/crates/format/src/fig/kiwi.rs @@ -10,8 +10,12 @@ use std::fmt; /// Deepest struct/message nesting decoded. const MAX_DEPTH: usize = 64; -/// Most values decoded from one message (every scalar, array element and field counts). -const MAX_VALUES: usize = 20_000_000; +/// Most values decoded from one message (every scalar, array element and field counts): at least +/// [`MIN_VALUES`], and one per [`PACKED_BYTES_PER_VALUE`] bytes of the file's compressed canvas, +/// so a large real file decodes while a small one can't expand into a huge tree, however far its +/// chunks inflate. +const MIN_VALUES: usize = 20_000_000; +const PACKED_BYTES_PER_VALUE: usize = 2; /// Most definitions in a schema, and fields in one definition. const MAX_DEFS: usize = 10_000; const MAX_FIELDS: usize = 10_000; @@ -284,13 +288,23 @@ impl Schema { self.defs.iter().position(|d| d.name == name) } - /// Decode a message of definition `root`. + /// Decode a message of definition `root`, within [`MIN_VALUES`] values. pub fn decode_message<'s>(&'s self, root: usize, data: &[u8]) -> Result, KiwiError> { - let mut d = Decoder { schema: self, r: Reader { data, pos: 0 }, budget: MAX_VALUES }; + self.decode_message_within(root, data, MIN_VALUES) + } + + /// [`Self::decode_message`] within `budget` values (see [`value_budget`]). + pub fn decode_message_within<'s>(&'s self, root: usize, data: &[u8], budget: usize) -> Result, KiwiError> { + let mut d = Decoder { schema: self, r: Reader { data, pos: 0 }, budget }; d.def(root, 0) } } +/// Most values decoded from a file whose compressed canvas is `packed` bytes. +pub(crate) fn value_budget(packed: usize) -> usize { + MIN_VALUES.max(packed / PACKED_BYTES_PER_VALUE) +} + struct Decoder<'s, 'b> { schema: &'s Schema, r: Reader<'b>, @@ -327,6 +341,8 @@ impl<'s> Decoder<'s, '_> { loop { let id = self.r.var_uint()?; if id == 0 { + // Grown by doubling: trimmed, a large file's tree takes a good part less. + out.shrink_to_fit(); return Ok(Kv::Object(out)); } let f = d.fields.iter().find(|f| f.value == id).ok_or_else(|| KiwiError(format!("message: {} has no field {id}", d.name)))?; diff --git a/crates/format/src/fig/map.rs b/crates/format/src/fig/map.rs index 7a3dab5c..a8af2ab9 100644 --- a/crates/format/src/fig/map.rs +++ b/crates/format/src/fig/map.rs @@ -6,9 +6,10 @@ //! are expanded here from the main component (overrides and derived layout matched by id path, //! component property values applied) so the import looks as saved. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::rc::Rc; +use figcraft_doc::VectorNetwork; use figcraft_doc::geom::{Affine, Point}; use figcraft_doc::proto::{MAX_ACTION_DEPTH, MAX_EXPR_DEPTH}; use serde_json::{Map, Value, json}; @@ -18,8 +19,7 @@ use super::network; type Obj = Map; -/// Most layers produced (instance expansion included) and deepest nesting followed. -const MAX_NODES: usize = crate::rest::MAX_NODES; +/// Deepest nesting followed. const MAX_DEPTH: usize = 256; /// Most path commands read from one geometry blob. const MAX_COMMANDS: usize = 200_000; @@ -30,11 +30,41 @@ const NO_STYLE: &str = "4294967295:4294967295"; /// Longest asset `key` or `version` kept for resolving references given as assets. const MAX_ASSET_PART: usize = 256; -/// The REST API file JSON for message `msg`, with the warnings about what was skipped. +/// The REST API file JSON for message `msg`, with the warnings about what was skipped and the +/// vector networks its layers name by `vectorNetworkRef` (for [`crate::rest::import_fig_parts`]). +/// [`to_rest_parts`] within the REST import's layer cap, with each vector's network written back +/// into its JSON as `vectorNetwork`: what the tests read. +#[cfg(test)] pub(super) fn to_rest(msg: &Kv, name: &str) -> (Value, Vec) { - let mut m = Mapper::new(msg); + fn inline(v: &mut Value, networks: &[Option]) { + match v { + Value::Object(o) => { + let net = o.remove("vectorNetworkRef").and_then(|r| networks.get(usize::try_from(r.as_u64()?).ok()?).cloned().flatten()); + if let Some(net) = net.and_then(|n| serde_json::to_value(n).ok()) { + o.insert("vectorNetwork".into(), net); + } + o.values_mut().for_each(|c| inline(c, networks)); + } + Value::Array(a) => a.iter_mut().for_each(|c| inline(c, networks)), + _ => {} + } + } + let (mut json, warnings, networks) = to_rest_parts(msg, name, MAX_NODES); + inline(&mut json, &networks); + (json, warnings) +} + +/// The REST import's layer cap, for the tests. +#[cfg(test)] +pub(super) const MAX_NODES: usize = crate::rest::MAX_NODES; + +pub(super) fn to_rest_parts(msg: &Kv, name: &str, max_nodes: usize) -> (Value, Vec, Vec>) { + let mut m = Mapper::new(msg, max_nodes); let doc = m.document(); - let mut out = json!({"name": name, "document": doc}); + // Large values are moved in, never passed through `json!`, which serialises its arguments: + // a deep copy of the whole tree, briefly holding it twice. + let mut out = json!({"name": name}); + out["document"] = doc; if !m.library_contexts.is_empty() { out[crate::rest::FIG_LIBRARY_CONTEXTS_FIELD] = Value::Array(std::mem::take(&mut m.library_contexts)); } @@ -54,7 +84,7 @@ pub(super) fn to_rest(msg: &Kv, name: &str) -> (Value, Vec) { if !styles.is_empty() { out["styleDefinitions"] = Value::Array(styles); } - (out, m.warnings) + (out, m.warnings, m.networks) } #[cfg(test)] @@ -374,20 +404,27 @@ struct Mapper<'a> { sets: Obj, /// Layers and keys of each main, built when an instance of it first needs them. override_scopes: BTreeMap>, - /// Entries held by `override_scopes` (bounded by [`MAX_NODES`] together). + /// Entries held by `override_scopes` (bounded by `max_nodes` together). override_scope_entries: usize, /// Component-bearing branches of the file's internal canvases (cached library definitions). library_contexts: Vec, /// Layers visited while scanning internal canvases, across all of them. library_scan_count: usize, - /// Most layers produced or scanned; the one place the cap comes from. + /// Most layers produced (instance expansion included) or scanned; the one place the cap comes + /// from (see [`super::layer_budget`]). max_nodes: usize, warnings: Vec, count: usize, + /// Vector networks, each built once: the layers that show one (every instance of a main + /// holding it) name it by `vectorNetworkRef`. Kept out of the JSON, where a large file's + /// networks took gigabytes. + networks: Vec>, + /// Index in `networks` by what a network is made from: its blob, its scale and its style table. + network_ids: HashMap<(usize, u64, u64, usize), usize>, } impl<'a> Mapper<'a> { - fn new(msg: &'a Kv<'a>) -> Self { + fn new(msg: &'a Kv<'a>, max_nodes: usize) -> Self { let mut nodes = BTreeMap::new(); let mut kids: BTreeMap> = BTreeMap::new(); let mut prop_defs = BTreeMap::new(); @@ -447,9 +484,11 @@ impl<'a> Mapper<'a> { override_scope_entries: 0, library_contexts: Vec::new(), library_scan_count: 0, - max_nodes: MAX_NODES, + max_nodes, warnings: Vec::new(), count: 0, + networks: Vec::new(), + network_ids: HashMap::new(), } } @@ -498,7 +537,7 @@ impl<'a> Mapper<'a> { let mut pending: Vec<(String, usize)> = self.children(main).into_iter().map(|id| (id, 1)).collect(); let mut complete = true; while let Some((id, depth)) = pending.pop() { - if depth > MAX_DEPTH || visited.len() >= MAX_NODES { + if depth > MAX_DEPTH || visited.len() >= self.max_nodes { complete = false; break; } @@ -523,7 +562,7 @@ impl<'a> Mapper<'a> { } let scope = Rc::new(OverrideScope { current, aliases }); let cost = scope.current.len() + scope.aliases.len(); - if cost <= MAX_NODES.saturating_sub(self.override_scope_entries) { + if cost <= self.max_nodes.saturating_sub(self.override_scope_entries) { self.override_scope_entries += cost; self.override_scopes.insert(main.to_string(), Rc::clone(&scope)); } @@ -563,7 +602,9 @@ impl<'a> Mapper<'a> { if out.is_empty() { self.warn("the file has no pages"); } - json!({"id": doc_id.unwrap_or_else(|| "0:0".into()), "type": "DOCUMENT", "children": out}) + let mut doc = json!({"id": doc_id.unwrap_or_else(|| "0:0".into()), "type": "DOCUMENT"}); + doc["children"] = Value::Array(out); + doc } /// The ids of the components the visible pages use, directly or through the components they @@ -777,8 +818,9 @@ impl<'a> Mapper<'a> { self.warn(format!("layers nested deeper than {MAX_DEPTH} levels were skipped")); return None; } - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; the rest were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; the rest were skipped")); return None; } let base = *self.nodes.get(id)?; @@ -813,7 +855,7 @@ impl<'a> Mapper<'a> { let connector_snapshot = own == "CONNECTOR" && self.connector_snapshot_supported(&view); // Only a connector with a saved label needs the frame wrapper; an unlabelled one stays a single vector. let connector_terminal = - connector_snapshot && (!connector_has_saved_label(&view) || depth >= MAX_DEPTH || self.count.saturating_add(1) >= MAX_NODES); + connector_snapshot && (!connector_has_saved_label(&view) || depth >= MAX_DEPTH || self.count.saturating_add(1) >= self.max_nodes); // A supported connector snapshot has its own note below instead of the generic stand-in one. let stand = stand_in(own).filter(|_| !connector_snapshot); let ty = match own { @@ -947,7 +989,7 @@ impl<'a> Mapper<'a> { fn shape_with_text(&mut self, o: &mut Obj, v: &View<'a>, own: &str, rest_id: &str) -> Option { let shape = v.str("shapeWithTextType").unwrap_or("SQUARE"); let outlined = !v.list("fillGeometry").is_empty() || !v.list("strokeGeometry").is_empty(); - if own != "SHAPE_WITH_TEXT" || matches!(shape, "SQUARE" | "ROUNDED_RECTANGLE") || !outlined || self.count >= MAX_NODES { + if own != "SHAPE_WITH_TEXT" || matches!(shape, "SQUARE" | "ROUNDED_RECTANGLE") || !outlined || self.count >= self.max_nodes { return None; } let mut s = Obj::new(); @@ -979,7 +1021,7 @@ impl<'a> Mapper<'a> { /// layer filling it, inset a little; centred unless it's a note. fn inner_text(&mut self, o: &Obj, v: &View<'a>, rest_id: &str, note: bool) -> Option { let chars = own_text(v)?; - if self.count >= MAX_NODES { + if self.count >= self.max_nodes { return None; } self.count += 1; @@ -1013,7 +1055,7 @@ impl<'a> Mapper<'a> { if connector_has_saved_label(v) { self.warn("connector labels were skipped"); } - if ["vectorNetwork", "fillGeometry", "strokeGeometry"].iter().any(|k| o.contains_key(*k)) { + if ["vectorNetwork", "vectorNetworkRef", "fillGeometry", "strokeGeometry"].iter().any(|k| o.contains_key(*k)) { return; } let parent = v.get("parentIndex").and_then(|p| p.get("guid")).and_then(guid); @@ -1114,7 +1156,7 @@ impl<'a> Mapper<'a> { out.insert("opacity".into(), json!(root_opacity * line_opacity)); } if connector_has_saved_label(v) { - let reason = if self.count >= MAX_NODES { "node budget" } else { "depth limit" }; + let reason = if self.count >= self.max_nodes { "node budget" } else { "depth limit" }; self.warn(format!("connector label was skipped: {reason} prevents nested snapshot children")); } } @@ -1144,8 +1186,9 @@ impl<'a> Mapper<'a> { /// Snapshot the straight, manual connector contract carried by its generated children. fn connector_snapshot_children(&mut self, v: &View<'a>, rest_id: &str) -> Vec { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector snapshot children were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector snapshot children were skipped")); return Vec::new(); } let Some(vector) = self.connector_snapshot_vector(v, rest_id, Affine::IDENTITY, true) else { @@ -1207,8 +1250,9 @@ impl<'a> Mapper<'a> { }; let background_view = View::of(background).with([background_style]); if !background_view.list("fillPaints").is_empty() { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector label background was skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector label background was skipped")); } else { let mut rect = Obj::new(); rect.insert("id".into(), json!(format!("{rest_id};label-background"))); @@ -1235,8 +1279,9 @@ impl<'a> Mapper<'a> { transform: Affine, view: &View<'a>, ) { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector label was skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector label was skipped")); return; } let mut text = Obj::new(); @@ -1648,7 +1693,8 @@ impl<'a> Mapper<'a> { /// False when there's none or it's damaged (the outlines stand in). fn network(&mut self, o: &mut Obj, v: &View<'a>) -> bool { let Some(vd) = v.get("vectorData") else { return false }; - let Some(blob) = num(vd, "vectorNetworkBlob").and_then(|i| self.blobs.get(i as usize)).copied() else { return false }; + let Some(blob_id) = num(vd, "vectorNetworkBlob").map(|i| i as usize) else { return false }; + let Some(blob) = self.blobs.get(blob_id).copied() else { return false }; let Some(net) = network::decode(blob) else { self.warn("damaged vector networks import as outlines"); return false; @@ -1686,28 +1732,6 @@ impl<'a> Mapper<'a> { size.zip(norm).filter(|(s, n)| s.is_finite() && *n > 1e-9).map_or(1.0, |(s, n)| s / n) }; let (sx, sy) = (scale("x"), scale("y")); - let vertices: Vec = net - .vertices - .iter() - .map(|p| { - let mut out = json!({"x": p.x * sx, "y": p.y * sy}); - let st = style(p.style); - if let Some(r) = st.and_then(|s| num(s, "cornerRadius")).filter(|r| *r > 0.0) { - out["radius"] = json!(r); - } - match st.and_then(|s| s.get("handleMirroring")).and_then(Kv::str) { - Some("ANGLE") => out["mirror"] = json!("angle"), - Some("ANGLE_AND_LENGTH") => out["mirror"] = json!("angleAndLength"), - Some("NONE") => out["mirror"] = json!("none"), - _ => {} - } - out - }) - .collect(); - let segments: Vec = - net.segments.iter().map(|s| json!({"a": s.a, "b": s.b, "ta": [s.ta.0 * sx, s.ta.1 * sy], "tb": [s.tb.0 * sx, s.tb.1 * sy]})).collect(); - let regions: Vec = - net.regions.iter().map(|r| json!({"loops": r.loops, "winding": if r.non_zero { "nonZero" } else { "evenOdd" }})).collect(); if net.regions.iter().any(|r| style(r.style).is_some_and(|s| s.get("fillPaints").is_some())) { self.warn("filled areas of a vector with their own fill use the layer's fill"); } @@ -1733,7 +1757,46 @@ impl<'a> Mapper<'a> { o.insert("strokeCapStart".into(), json!(cap_name(first.unwrap_or(node)))); o.insert("strokeCapEnd".into(), json!(cap_name(last.unwrap_or(node)))); } - o.insert("vectorNetwork".into(), json!({"vertices": vertices, "segments": segments, "regions": regions})); + // Every instance of a main shows the same network: it is written once and shared. + let key = (blob_id, sx.to_bits(), sy.to_bits(), std::ptr::from_ref(vd) as usize); + let id = match self.network_ids.get(&key) { + Some(id) => *id, + None => { + let vertices: Vec = net + .vertices + .iter() + .map(|p| { + let mut out = json!({"x": p.x * sx, "y": p.y * sy}); + let st = style(p.style); + if let Some(r) = st.and_then(|s| num(s, "cornerRadius")).filter(|r| *r > 0.0) { + out["radius"] = json!(r); + } + match st.and_then(|s| s.get("handleMirroring")).and_then(Kv::str) { + Some("ANGLE") => out["mirror"] = json!("angle"), + Some("ANGLE_AND_LENGTH") => out["mirror"] = json!("angleAndLength"), + Some("NONE") => out["mirror"] = json!("none"), + _ => {} + } + out + }) + .collect(); + let segments: Vec = net + .segments + .iter() + .map(|s| json!({"a": s.a, "b": s.b, "ta": [s.ta.0 * sx, s.ta.1 * sy], "tb": [s.tb.0 * sx, s.tb.1 * sy]})) + .collect(); + let regions: Vec = + net.regions.iter().map(|r| json!({"loops": r.loops, "winding": if r.non_zero { "nonZero" } else { "evenOdd" }})).collect(); + let mut entry = json!({}); + entry["vertices"] = Value::Array(vertices); + entry["segments"] = Value::Array(segments); + entry["regions"] = Value::Array(regions); + self.networks.push(crate::rest::network(&entry)); + self.network_ids.insert(key, self.networks.len() - 1); + self.networks.len() - 1 + } + }; + o.insert("vectorNetworkRef".into(), json!(id)); true } diff --git a/crates/format/src/fig/map_connector_snapshot_budget_tests.rs b/crates/format/src/fig/map_connector_snapshot_budget_tests.rs index a14e671c..4d8ae9be 100644 --- a/crates/format/src/fig/map_connector_snapshot_budget_tests.rs +++ b/crates/format/src/fig/map_connector_snapshot_budget_tests.rs @@ -106,7 +106,7 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { let message = message(); let node = message.get("nodeChanges").unwrap().list().first().unwrap(); let view = View::of(node); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES - 1; let children = mapper.connector_snapshot_children(&view, "1:123"); assert_eq!(children.len(), 1, "the final available slot preserves the saved vector"); @@ -116,7 +116,7 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { assert_eq!(mapper.count, MAX_NODES); assert!(mapper.warnings.iter().any(|warning| warning == &format!("the file has more than {MAX_NODES} layers; connector label was skipped"))); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES; let children = mapper.connector_snapshot_children(&view, "1:123"); assert!(children.is_empty()); @@ -127,14 +127,14 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { #[test] fn connector_snapshot_flattens_vector_at_depth_and_node_limits() { let message = message(); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); let depth_limited = mapper.layer("1:123", None, Affine::IDENTITY, None, None, MAX_DEPTH).expect("terminal connector"); assert_eq!(depth_limited["type"], "VECTOR"); assert!(!depth_limited["fillGeometry"].as_array().unwrap().is_empty()); assert!(depth_limited.get("children").is_none()); assert!(mapper.warnings.iter().any(|warning| warning == "connector label was skipped: depth limit prevents nested snapshot children")); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES - 1; let budget_limited = mapper.layer("1:123", None, Affine::IDENTITY, None, None, 0).expect("last allowed connector"); assert_eq!(budget_limited["type"], "VECTOR"); diff --git a/crates/format/src/fig/mod.rs b/crates/format/src/fig/mod.rs index 0d341307..08a3409e 100644 --- a/crates/format/src/fig/mod.rs +++ b/crates/format/src/fig/mod.rs @@ -54,6 +54,18 @@ const MAX_CHUNK: u64 = 512 * 1024 * 1024; const MAX_IMAGE: u64 = 256 * 1024 * 1024; /// Most chunks read from `canvas.fig`. const MAX_CHUNKS: usize = 16; +/// Most layers imported from a `.fig` file: one per [`PACKED_BYTES_PER_LAYER`] bytes of its +/// compressed canvas, at least the REST import's cap and at most [`MAX_FIG_NODES`]. Instances +/// expand into a layer per layer of their component, and chunks inflate up to [`MAX_CHUNK`], so the +/// budget follows what the file takes, not what it unpacks to: a small file can't claim a large +/// count, however it's built; a large one gets room. +const PACKED_BYTES_PER_LAYER: usize = 32; +const MAX_FIG_NODES: usize = 2_000_000; + +/// Most layers imported from a `.fig` file whose compressed canvas is `packed` bytes. +pub(crate) fn layer_budget(packed: usize) -> usize { + (packed / PACKED_BYTES_PER_LAYER).clamp(crate::rest::MAX_NODES, MAX_FIG_NODES) +} /// Whether `bytes` look like a `.fig` file: a zip holding `canvas.fig`, or a bare canvas. pub fn is_fig(bytes: &[u8]) -> bool { @@ -72,14 +84,22 @@ pub fn try_import(bytes: &[u8], name: &str) -> Option Result<(Document, Vec), FormatError> { - let archive = Archive::open(bytes)?; - let (schema, message) = decode_canvas(&archive.canvas)?; + let mut archive = Archive::open(bytes)?; + // The budgets follow the compressed canvas, which the file itself bounds (its zip member may + // inflate too). + let packed = archive.canvas.len().min(bytes.len()); + // Each stage's input is dropped as soon as the next one is built: in a large file every one + // of them takes gigabytes, and holding them all at once runs out of memory. + let (schema, message) = decode_canvas(&std::mem::take(&mut archive.canvas))?; let root = schema.find("Message").ok_or_else(|| corrupt("the schema has no Message definition"))?; - let msg = schema.decode_message(root, &message).map_err(|e| corrupt(e.to_string()))?; + let msg = schema.decode_message_within(root, &message, kiwi::value_budget(packed)).map_err(|e| corrupt(e.to_string()))?; + let max_nodes = layer_budget(packed); + drop(message); let file_name = archive.file_name.as_deref().filter(|n| !n.trim().is_empty()).unwrap_or(name); - let (json, mut warnings) = map::to_rest(&msg, file_name); + let (json, mut warnings, networks) = map::to_rest_parts(&msg, file_name, max_nodes); + drop(msg); let images = |r: &str| archive.image(r); - let (doc, more) = crate::rest::import_fig(&json, &images)?; + let (doc, more) = crate::rest::import_fig_parts(&json, &images, networks, max_nodes)?; warnings.extend(more.into_iter().filter_map(map::adapt_warning)); Ok((doc, warnings)) } diff --git a/crates/format/src/fig/tail_tests.rs b/crates/format/src/fig/tail_tests.rs index a15334a5..43e997a5 100644 --- a/crates/format/src/fig/tail_tests.rs +++ b/crates/format/src/fig/tail_tests.rs @@ -446,8 +446,8 @@ fn mangled_tail_files_never_panic() { m[i] = r.next() as u8; } if let Ok(kv) = sc.decode_message(root, &m) { - let (json, _) = map::to_rest(&kv, "x"); - let _ = crate::rest::import(&json, &crate::rest::no_images); + let (json, _, networks) = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); + let _ = crate::rest::import_fig_parts(&json, &crate::rest::no_images, networks, crate::rest::MAX_NODES); } } } diff --git a/crates/format/src/fig/tests.rs b/crates/format/src/fig/tests.rs index 24baef30..5f20f5ac 100644 --- a/crates/format/src/fig/tests.rs +++ b/crates/format/src/fig/tests.rs @@ -757,7 +757,7 @@ fn mangled_kiwi_never_panics() { } for cut in (0..msg.len()).step_by(3) { if let Ok(kv) = sc.decode_message(root, &msg[..cut]) { - let _ = map::to_rest(&kv, "x"); + let _ = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); } } for _ in 0..2000 { @@ -767,8 +767,8 @@ fn mangled_kiwi_never_panics() { m[i] = r.next() as u8; } if let Ok(kv) = sc.decode_message(root, &m) { - let (json, _) = map::to_rest(&kv, "x"); - let _ = crate::rest::import(&json, &crate::rest::no_images); + let (json, _, networks) = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); + let _ = crate::rest::import_fig_parts(&json, &crate::rest::no_images, networks, crate::rest::MAX_NODES); } let mut sbytes = schema_bytes.clone(); let i = (r.next() as usize) % sbytes.len(); @@ -790,6 +790,59 @@ fn mangled_kiwi_never_panics() { } } +#[test] +fn large_files_get_a_larger_value_budget() { + // Small files keep the fixed floor: a few bytes can't expand into a huge tree. + assert_eq!(kiwi::value_budget(0), 20_000_000); + assert_eq!(kiwi::value_budget(1 << 20), 20_000_000); + // A real 400 000-layer file: an 83.5 MB canvas whose message unpacks to 334 MB and holds + // 30.5 million values was refused as "too many values". + assert!(kiwi::value_budget(83_491_232) > 30_514_899); + // Still bounded by the largest canvas read. + assert!(kiwi::value_budget(MAX_CANVAS as usize) <= 128 << 20); +} + +#[test] +fn the_layer_cap_follows_the_file_size() { + // A small (or broken) file keeps the REST import's cap: instances expand into a layer per + // layer of their component, and a few bytes mustn't claim millions of layers. + assert_eq!(layer_budget(0), crate::rest::MAX_NODES); + assert_eq!(layer_budget(1 << 20), crate::rest::MAX_NODES); + // A real 894 004-layer file (an 83.5 MB canvas) stopped at 200 000 and came in with 27 of + // its 30 pages empty. + assert!(layer_budget(83_491_232) > 894_004); + // No file gets more than the ceiling. + assert_eq!(layer_budget(MAX_CANVAS as usize), MAX_FIG_NODES); +} + +#[test] +fn a_small_file_that_unpacks_large_gets_no_larger_budgets() { + // 40 MB of zeros in a blob pack into a few KB. Keyed to the unpacked message, they lifted the + // layer cap above the 231 231 layers its instances expand into; keyed to the file, the cap + // stays at the REST import's. + let mut nodes = vec![node(0, None, "DOCUMENT", "D", vec![]), node(1, Some((0, "!")), "CANVAS", "P", vec![])]; + nodes.push(node(5, Some((1, "!")), "SYMBOL", "Main", vec![("size", xy(10.0, 10.0))])); + nodes.extend((10..1010).map(|i| node(i, Some((5, "!")), "RECTANGLE", "R", vec![("size", xy(1.0, 1.0))]))); + nodes.extend((2000..2230).map(|i| node(i, Some((1, "!")), "INSTANCE", "I", vec![("symbolData", o(vec![("symbolID", g(5))]))]))); + let blob = o(vec![("bytes", Kv::Bytes(vec![0; 40 << 20]))]); + let file = canvas(&schema(), &o(vec![("nodeChanges", arr(nodes)), ("blobs", arr(vec![blob]))])); + assert!(file.len() < 1 << 20, "{} bytes", file.len()); + let (doc, warnings) = import(&file, "x").unwrap(); + assert!(warnings.iter().any(|w| w.contains("more than 200000 layers")), "{warnings:?}"); + assert!(doc.node_count() <= crate::rest::MAX_NODES + 10); +} + +#[test] +fn the_mapper_stops_at_the_layer_cap_it_is_given() { + let mut nodes = vec![node(0, None, "DOCUMENT", "D", vec![]), node(1, Some((0, "!")), "CANVAS", "P", vec![])]; + nodes.extend((2..12).map(|i| node(i, Some((1, "!")), "RECTANGLE", "R", vec![("size", xy(1.0, 1.0))]))); + let msg = o(vec![("nodeChanges", arr(nodes))]); + let (_, warnings, _) = map::to_rest_parts(&msg, "x", 5); + assert!(warnings.iter().any(|w| w.contains("more than 5 layers")), "{warnings:?}"); + let (_, warnings, _) = map::to_rest_parts(&msg, "x", crate::rest::MAX_NODES); + assert!(!warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); +} + #[test] fn hostile_input_is_refused() { // A message that contains itself, nested past the depth cap. @@ -1088,7 +1141,7 @@ fn dump_samples() { std::fs::write(format!("{out}/{stem}.schema.txt"), txt).unwrap(); let msg = schema.decode_message(schema.find("Message").unwrap(), &message).unwrap(); std::fs::write(format!("{out}/{stem}.json"), serde_json::to_string_pretty(&debug_json(&msg)).unwrap()).unwrap(); - let (rest, _) = map::to_rest(&msg, &stem); + let (rest, ..) = map::to_rest_parts(&msg, &stem, super::layer_budget(message.len())); std::fs::write(format!("{out}/{stem}.rest.json"), serde_json::to_string_pretty(&rest).unwrap()).unwrap(); } } diff --git a/crates/format/src/rest/mod.rs b/crates/format/src/rest/mod.rs index e460c8b8..0fa53a7c 100644 --- a/crates/format/src/rest/mod.rs +++ b/crates/format/src/rest/mod.rs @@ -53,7 +53,7 @@ use crate::FormatError; use props::{MAX_COORD, MAX_SIZE, clip_str, flag, list, num, text}; pub use variables::export as export_variables; -/// Most layers imported from one file. +/// Most layers imported from one file. The `.fig` import sets its own, from the file's size. pub const MAX_NODES: usize = 200_000; /// Deepest layer nesting followed. const MAX_DEPTH: usize = 256; @@ -146,22 +146,43 @@ pub fn import(v: &Value, images: ImageResolver) -> Result<(Document, Vec /// in the file JSON (a `meta` with `variableCollections`, or `variableCollections` and /// `variables` at the top level) are read too; the first definition of an id wins. pub fn import_with(v: &Value, variables: Option<&Value>, images: ImageResolver) -> Result<(Document, Vec), FormatError> { - import_impl(v, variables, images, false) + import_impl(v, variables, images, false, Vec::new(), MAX_NODES) } -/// Internal .fig JSON path: GUID provenance is not part of the public REST import contract. +/// Internal .fig JSON path: GUID provenance is not part of the public REST import contract. Its +/// vectors name their networks by `vectorNetworkRef`, an index in `networks` (built once each, and +/// far smaller than as JSON); `max_nodes` is the layer cap the `.fig` import sets in place of +/// [`MAX_NODES`]. +pub(crate) fn import_fig_parts( + v: &Value, + images: ImageResolver, + networks: Vec>, + max_nodes: usize, +) -> Result<(Document, Vec), FormatError> { + import_impl(v, None, images, true, networks, max_nodes) +} + +/// [`import_fig_parts`] of JSON that carries its vectors' networks, within [`MAX_NODES`]. +#[cfg(test)] pub(crate) fn import_fig(v: &Value, images: ImageResolver) -> Result<(Document, Vec), FormatError> { - import_impl(v, None, images, true) + import_fig_parts(v, images, Vec::new(), MAX_NODES) } -fn import_impl(v: &Value, variables: Option<&Value>, images: ImageResolver, fig_sources: bool) -> Result<(Document, Vec), FormatError> { +fn import_impl( + v: &Value, + variables: Option<&Value>, + images: ImageResolver, + fig_sources: bool, + networks: Vec>, + max_nodes: usize, +) -> Result<(Document, Vec), FormatError> { let entries: Vec<&Value> = if is_document(v) { vec![v] } else { nodes_entries(v).collect() }; if entries.is_empty() { return Err(FormatError::Corrupt("not a REST API file JSON (no `document`)".into())); } let name = text(v, "name").filter(|n| !n.trim().is_empty()).map_or("Imported file", clip_str); let mut cx = Ctx::new(Document::new(name), images); - cx.fig_sources = fig_sources.then(|| fig_sources::Sources::new(MAX_NODES)); + cx.fig_sources = fig_sources.then(|| fig_sources::Sources::new(max_nodes)); let blank = cx.doc.first_page(); let mut loose: Option = None; for e in &entries { @@ -175,6 +196,8 @@ fn import_impl(v: &Value, variables: Option<&Value>, images: ImageResolver, fig_ cx.warn("the variables JSON has no `variableCollections` (expected the local variables response) and was ignored"); } cx.style_definitions(v); + cx.networks = networks; + cx.max_nodes = max_nodes; for e in entries { let Some(root) = e.get("document") else { continue }; match text(root, "type") { @@ -356,6 +379,10 @@ pub(crate) struct Ctx<'a> { /// Imported variables by id, and the mode ids of imported collections (default first). var_kinds: BTreeMap, var_modes: BTreeMap>, + /// Vector networks built by the `.fig` import, named by layers' `vectorNetworkRef`. + networks: Vec>, + /// Most layers imported: [`MAX_NODES`], or the `.fig` import's own cap. + max_nodes: usize, } impl<'a> Ctx<'a> { @@ -380,6 +407,8 @@ impl<'a> Ctx<'a> { overlay_meta: BTreeMap::new(), var_kinds: BTreeMap::new(), var_modes: BTreeMap::new(), + networks: Vec::new(), + max_nodes: MAX_NODES, } } @@ -435,8 +464,9 @@ impl<'a> Ctx<'a> { self.warn(format!("layers nested deeper than {MAX_DEPTH} levels were skipped")); return; } - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; the rest were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; the rest were skipped")); return; } let Some(ty) = text(j, "type") else { @@ -642,14 +672,18 @@ impl<'a> Ctx<'a> { } } - /// A vector's network: the editable one when the JSON carries it (`vectorNetwork`, our - /// extension, written by the `.fig` import), else from its geometry. A stroke-only vector - /// uses the stroke outline, painted with the stroke paints (exact look, editable as a shape): - /// returns whether it does, and the caller moves the paints with [`outline_stroke`] once the - /// styles and variables of the stroke are read. + /// A vector's network: the editable one when the JSON carries it (`vectorNetwork`, or + /// `vectorNetworkRef` naming one built by the `.fig` import: our extensions), else from its + /// geometry. A stroke-only vector uses the stroke outline, painted with the stroke paints + /// (exact look, editable as a shape): returns whether it does, and the caller moves the paints + /// with [`outline_stroke`] once the styles and variables of the stroke are read. fn vector(&mut self, n: &mut Node, j: &Value) -> bool { - if let Some(net) = j.get("vectorNetwork") { - match network(net) { + let net = match j.get("vectorNetworkRef") { + Some(r) => Some(r.as_u64().and_then(|i| self.networks.get(usize::try_from(i).ok()?).cloned().flatten())), + None => j.get("vectorNetwork").map(network), + }; + if let Some(net) = net { + match net { Some(network) => { n.kind = NodeKind::Vector { network }; return false; @@ -1173,7 +1207,7 @@ fn local(parent: Affine, world: Affine) -> Affine { /// Finite path data within the coordinate cap. /// A vector network in our own JSON shape, when it's within bounds and refers only to what it has. -fn network(v: &Value) -> Option { +pub(crate) fn network(v: &Value) -> Option { use serde::Deserialize; let mut net = VectorNetwork::deserialize(v).ok()?; let entries: usize = net.regions.iter().flat_map(|r| &r.loops).map(Vec::len).sum(); diff --git a/crates/format/src/rest/tests.rs b/crates/format/src/rest/tests.rs index 7db22db1..cdf27905 100644 --- a/crates/format/src/rest/tests.rs +++ b/crates/format/src/rest/tests.rs @@ -729,3 +729,18 @@ fn layer_count_is_capped() { assert_eq!(doc.node_count(), MAX_NODES + 1); assert!(warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); } + +#[test] +fn the_fig_import_sets_its_own_layer_cap() { + // The `.fig` import passes a cap from the file's size: above `MAX_NODES` every layer comes in. + let kids: Vec = (0..MAX_NODES + 5).map(|i| json!({"id": i.to_string(), "type": "RECTANGLE", "name": "R"})).collect(); + let json = file(json!([page("Page", Value::Array(kids))])); + let (doc, warnings) = import_fig_parts(&json, &no_images, Vec::new(), MAX_NODES + 10).unwrap(); + assert_eq!(doc.node_count(), MAX_NODES + 6); + assert!(!warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); + // Below it, the import stops at the cap it was given. + let few: Vec = (0..10).map(|i| json!({"id": i.to_string(), "type": "RECTANGLE", "name": "R"})).collect(); + let (doc, warnings) = import_fig_parts(&file(json!([page("Page", Value::Array(few))])), &no_images, Vec::new(), 4).unwrap(); + assert_eq!(doc.node_count(), 5); + assert!(warnings.iter().any(|w| w.contains("more than 4 layers")), "{warnings:?}"); +}