diff --git a/crates/format/src/fig/kiwi.rs b/crates/format/src/fig/kiwi.rs index 528dbbf0..46a9daac 100644 --- a/crates/format/src/fig/kiwi.rs +++ b/crates/format/src/fig/kiwi.rs @@ -10,8 +10,12 @@ use std::fmt; /// Deepest struct/message nesting decoded. const MAX_DEPTH: usize = 64; -/// Most values decoded from one message (every scalar, array element and field counts). -const MAX_VALUES: usize = 20_000_000; +/// Most values decoded from one message (every scalar, array element and field counts): at least +/// [`MIN_VALUES`], and one per [`PACKED_BYTES_PER_VALUE`] bytes of the file's compressed canvas, +/// so a large real file decodes while a small one can't expand into a huge tree, however far its +/// chunks inflate. +const MIN_VALUES: usize = 20_000_000; +const PACKED_BYTES_PER_VALUE: usize = 2; /// Most definitions in a schema, and fields in one definition. const MAX_DEFS: usize = 10_000; const MAX_FIELDS: usize = 10_000; @@ -284,13 +288,23 @@ impl Schema { self.defs.iter().position(|d| d.name == name) } - /// Decode a message of definition `root`. + /// Decode a message of definition `root`, within [`MIN_VALUES`] values. pub fn decode_message<'s>(&'s self, root: usize, data: &[u8]) -> Result, KiwiError> { - let mut d = Decoder { schema: self, r: Reader { data, pos: 0 }, budget: MAX_VALUES }; + self.decode_message_within(root, data, MIN_VALUES) + } + + /// [`Self::decode_message`] within `budget` values (see [`value_budget`]). + pub fn decode_message_within<'s>(&'s self, root: usize, data: &[u8], budget: usize) -> Result, KiwiError> { + let mut d = Decoder { schema: self, r: Reader { data, pos: 0 }, budget }; d.def(root, 0) } } +/// Most values decoded from a file whose compressed canvas is `packed` bytes. +pub(crate) fn value_budget(packed: usize) -> usize { + MIN_VALUES.max(packed / PACKED_BYTES_PER_VALUE) +} + struct Decoder<'s, 'b> { schema: &'s Schema, r: Reader<'b>, @@ -327,6 +341,8 @@ impl<'s> Decoder<'s, '_> { loop { let id = self.r.var_uint()?; if id == 0 { + // Grown by doubling: trimmed, a large file's tree takes a good part less. + out.shrink_to_fit(); return Ok(Kv::Object(out)); } let f = d.fields.iter().find(|f| f.value == id).ok_or_else(|| KiwiError(format!("message: {} has no field {id}", d.name)))?; diff --git a/crates/format/src/fig/map.rs b/crates/format/src/fig/map.rs index 7a3dab5c..a8af2ab9 100644 --- a/crates/format/src/fig/map.rs +++ b/crates/format/src/fig/map.rs @@ -6,9 +6,10 @@ //! are expanded here from the main component (overrides and derived layout matched by id path, //! component property values applied) so the import looks as saved. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::rc::Rc; +use figcraft_doc::VectorNetwork; use figcraft_doc::geom::{Affine, Point}; use figcraft_doc::proto::{MAX_ACTION_DEPTH, MAX_EXPR_DEPTH}; use serde_json::{Map, Value, json}; @@ -18,8 +19,7 @@ use super::network; type Obj = Map; -/// Most layers produced (instance expansion included) and deepest nesting followed. -const MAX_NODES: usize = crate::rest::MAX_NODES; +/// Deepest nesting followed. const MAX_DEPTH: usize = 256; /// Most path commands read from one geometry blob. const MAX_COMMANDS: usize = 200_000; @@ -30,11 +30,41 @@ const NO_STYLE: &str = "4294967295:4294967295"; /// Longest asset `key` or `version` kept for resolving references given as assets. const MAX_ASSET_PART: usize = 256; -/// The REST API file JSON for message `msg`, with the warnings about what was skipped. +/// The REST API file JSON for message `msg`, with the warnings about what was skipped and the +/// vector networks its layers name by `vectorNetworkRef` (for [`crate::rest::import_fig_parts`]). +/// [`to_rest_parts`] within the REST import's layer cap, with each vector's network written back +/// into its JSON as `vectorNetwork`: what the tests read. +#[cfg(test)] pub(super) fn to_rest(msg: &Kv, name: &str) -> (Value, Vec) { - let mut m = Mapper::new(msg); + fn inline(v: &mut Value, networks: &[Option]) { + match v { + Value::Object(o) => { + let net = o.remove("vectorNetworkRef").and_then(|r| networks.get(usize::try_from(r.as_u64()?).ok()?).cloned().flatten()); + if let Some(net) = net.and_then(|n| serde_json::to_value(n).ok()) { + o.insert("vectorNetwork".into(), net); + } + o.values_mut().for_each(|c| inline(c, networks)); + } + Value::Array(a) => a.iter_mut().for_each(|c| inline(c, networks)), + _ => {} + } + } + let (mut json, warnings, networks) = to_rest_parts(msg, name, MAX_NODES); + inline(&mut json, &networks); + (json, warnings) +} + +/// The REST import's layer cap, for the tests. +#[cfg(test)] +pub(super) const MAX_NODES: usize = crate::rest::MAX_NODES; + +pub(super) fn to_rest_parts(msg: &Kv, name: &str, max_nodes: usize) -> (Value, Vec, Vec>) { + let mut m = Mapper::new(msg, max_nodes); let doc = m.document(); - let mut out = json!({"name": name, "document": doc}); + // Large values are moved in, never passed through `json!`, which serialises its arguments: + // a deep copy of the whole tree, briefly holding it twice. + let mut out = json!({"name": name}); + out["document"] = doc; if !m.library_contexts.is_empty() { out[crate::rest::FIG_LIBRARY_CONTEXTS_FIELD] = Value::Array(std::mem::take(&mut m.library_contexts)); } @@ -54,7 +84,7 @@ pub(super) fn to_rest(msg: &Kv, name: &str) -> (Value, Vec) { if !styles.is_empty() { out["styleDefinitions"] = Value::Array(styles); } - (out, m.warnings) + (out, m.warnings, m.networks) } #[cfg(test)] @@ -374,20 +404,27 @@ struct Mapper<'a> { sets: Obj, /// Layers and keys of each main, built when an instance of it first needs them. override_scopes: BTreeMap>, - /// Entries held by `override_scopes` (bounded by [`MAX_NODES`] together). + /// Entries held by `override_scopes` (bounded by `max_nodes` together). override_scope_entries: usize, /// Component-bearing branches of the file's internal canvases (cached library definitions). library_contexts: Vec, /// Layers visited while scanning internal canvases, across all of them. library_scan_count: usize, - /// Most layers produced or scanned; the one place the cap comes from. + /// Most layers produced (instance expansion included) or scanned; the one place the cap comes + /// from (see [`super::layer_budget`]). max_nodes: usize, warnings: Vec, count: usize, + /// Vector networks, each built once: the layers that show one (every instance of a main + /// holding it) name it by `vectorNetworkRef`. Kept out of the JSON, where a large file's + /// networks took gigabytes. + networks: Vec>, + /// Index in `networks` by what a network is made from: its blob, its scale and its style table. + network_ids: HashMap<(usize, u64, u64, usize), usize>, } impl<'a> Mapper<'a> { - fn new(msg: &'a Kv<'a>) -> Self { + fn new(msg: &'a Kv<'a>, max_nodes: usize) -> Self { let mut nodes = BTreeMap::new(); let mut kids: BTreeMap> = BTreeMap::new(); let mut prop_defs = BTreeMap::new(); @@ -447,9 +484,11 @@ impl<'a> Mapper<'a> { override_scope_entries: 0, library_contexts: Vec::new(), library_scan_count: 0, - max_nodes: MAX_NODES, + max_nodes, warnings: Vec::new(), count: 0, + networks: Vec::new(), + network_ids: HashMap::new(), } } @@ -498,7 +537,7 @@ impl<'a> Mapper<'a> { let mut pending: Vec<(String, usize)> = self.children(main).into_iter().map(|id| (id, 1)).collect(); let mut complete = true; while let Some((id, depth)) = pending.pop() { - if depth > MAX_DEPTH || visited.len() >= MAX_NODES { + if depth > MAX_DEPTH || visited.len() >= self.max_nodes { complete = false; break; } @@ -523,7 +562,7 @@ impl<'a> Mapper<'a> { } let scope = Rc::new(OverrideScope { current, aliases }); let cost = scope.current.len() + scope.aliases.len(); - if cost <= MAX_NODES.saturating_sub(self.override_scope_entries) { + if cost <= self.max_nodes.saturating_sub(self.override_scope_entries) { self.override_scope_entries += cost; self.override_scopes.insert(main.to_string(), Rc::clone(&scope)); } @@ -563,7 +602,9 @@ impl<'a> Mapper<'a> { if out.is_empty() { self.warn("the file has no pages"); } - json!({"id": doc_id.unwrap_or_else(|| "0:0".into()), "type": "DOCUMENT", "children": out}) + let mut doc = json!({"id": doc_id.unwrap_or_else(|| "0:0".into()), "type": "DOCUMENT"}); + doc["children"] = Value::Array(out); + doc } /// The ids of the components the visible pages use, directly or through the components they @@ -777,8 +818,9 @@ impl<'a> Mapper<'a> { self.warn(format!("layers nested deeper than {MAX_DEPTH} levels were skipped")); return None; } - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; the rest were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; the rest were skipped")); return None; } let base = *self.nodes.get(id)?; @@ -813,7 +855,7 @@ impl<'a> Mapper<'a> { let connector_snapshot = own == "CONNECTOR" && self.connector_snapshot_supported(&view); // Only a connector with a saved label needs the frame wrapper; an unlabelled one stays a single vector. let connector_terminal = - connector_snapshot && (!connector_has_saved_label(&view) || depth >= MAX_DEPTH || self.count.saturating_add(1) >= MAX_NODES); + connector_snapshot && (!connector_has_saved_label(&view) || depth >= MAX_DEPTH || self.count.saturating_add(1) >= self.max_nodes); // A supported connector snapshot has its own note below instead of the generic stand-in one. let stand = stand_in(own).filter(|_| !connector_snapshot); let ty = match own { @@ -947,7 +989,7 @@ impl<'a> Mapper<'a> { fn shape_with_text(&mut self, o: &mut Obj, v: &View<'a>, own: &str, rest_id: &str) -> Option { let shape = v.str("shapeWithTextType").unwrap_or("SQUARE"); let outlined = !v.list("fillGeometry").is_empty() || !v.list("strokeGeometry").is_empty(); - if own != "SHAPE_WITH_TEXT" || matches!(shape, "SQUARE" | "ROUNDED_RECTANGLE") || !outlined || self.count >= MAX_NODES { + if own != "SHAPE_WITH_TEXT" || matches!(shape, "SQUARE" | "ROUNDED_RECTANGLE") || !outlined || self.count >= self.max_nodes { return None; } let mut s = Obj::new(); @@ -979,7 +1021,7 @@ impl<'a> Mapper<'a> { /// layer filling it, inset a little; centred unless it's a note. fn inner_text(&mut self, o: &Obj, v: &View<'a>, rest_id: &str, note: bool) -> Option { let chars = own_text(v)?; - if self.count >= MAX_NODES { + if self.count >= self.max_nodes { return None; } self.count += 1; @@ -1013,7 +1055,7 @@ impl<'a> Mapper<'a> { if connector_has_saved_label(v) { self.warn("connector labels were skipped"); } - if ["vectorNetwork", "fillGeometry", "strokeGeometry"].iter().any(|k| o.contains_key(*k)) { + if ["vectorNetwork", "vectorNetworkRef", "fillGeometry", "strokeGeometry"].iter().any(|k| o.contains_key(*k)) { return; } let parent = v.get("parentIndex").and_then(|p| p.get("guid")).and_then(guid); @@ -1114,7 +1156,7 @@ impl<'a> Mapper<'a> { out.insert("opacity".into(), json!(root_opacity * line_opacity)); } if connector_has_saved_label(v) { - let reason = if self.count >= MAX_NODES { "node budget" } else { "depth limit" }; + let reason = if self.count >= self.max_nodes { "node budget" } else { "depth limit" }; self.warn(format!("connector label was skipped: {reason} prevents nested snapshot children")); } } @@ -1144,8 +1186,9 @@ impl<'a> Mapper<'a> { /// Snapshot the straight, manual connector contract carried by its generated children. fn connector_snapshot_children(&mut self, v: &View<'a>, rest_id: &str) -> Vec { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector snapshot children were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector snapshot children were skipped")); return Vec::new(); } let Some(vector) = self.connector_snapshot_vector(v, rest_id, Affine::IDENTITY, true) else { @@ -1207,8 +1250,9 @@ impl<'a> Mapper<'a> { }; let background_view = View::of(background).with([background_style]); if !background_view.list("fillPaints").is_empty() { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector label background was skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector label background was skipped")); } else { let mut rect = Obj::new(); rect.insert("id".into(), json!(format!("{rest_id};label-background"))); @@ -1235,8 +1279,9 @@ impl<'a> Mapper<'a> { transform: Affine, view: &View<'a>, ) { - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; connector label was skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; connector label was skipped")); return; } let mut text = Obj::new(); @@ -1648,7 +1693,8 @@ impl<'a> Mapper<'a> { /// False when there's none or it's damaged (the outlines stand in). fn network(&mut self, o: &mut Obj, v: &View<'a>) -> bool { let Some(vd) = v.get("vectorData") else { return false }; - let Some(blob) = num(vd, "vectorNetworkBlob").and_then(|i| self.blobs.get(i as usize)).copied() else { return false }; + let Some(blob_id) = num(vd, "vectorNetworkBlob").map(|i| i as usize) else { return false }; + let Some(blob) = self.blobs.get(blob_id).copied() else { return false }; let Some(net) = network::decode(blob) else { self.warn("damaged vector networks import as outlines"); return false; @@ -1686,28 +1732,6 @@ impl<'a> Mapper<'a> { size.zip(norm).filter(|(s, n)| s.is_finite() && *n > 1e-9).map_or(1.0, |(s, n)| s / n) }; let (sx, sy) = (scale("x"), scale("y")); - let vertices: Vec = net - .vertices - .iter() - .map(|p| { - let mut out = json!({"x": p.x * sx, "y": p.y * sy}); - let st = style(p.style); - if let Some(r) = st.and_then(|s| num(s, "cornerRadius")).filter(|r| *r > 0.0) { - out["radius"] = json!(r); - } - match st.and_then(|s| s.get("handleMirroring")).and_then(Kv::str) { - Some("ANGLE") => out["mirror"] = json!("angle"), - Some("ANGLE_AND_LENGTH") => out["mirror"] = json!("angleAndLength"), - Some("NONE") => out["mirror"] = json!("none"), - _ => {} - } - out - }) - .collect(); - let segments: Vec = - net.segments.iter().map(|s| json!({"a": s.a, "b": s.b, "ta": [s.ta.0 * sx, s.ta.1 * sy], "tb": [s.tb.0 * sx, s.tb.1 * sy]})).collect(); - let regions: Vec = - net.regions.iter().map(|r| json!({"loops": r.loops, "winding": if r.non_zero { "nonZero" } else { "evenOdd" }})).collect(); if net.regions.iter().any(|r| style(r.style).is_some_and(|s| s.get("fillPaints").is_some())) { self.warn("filled areas of a vector with their own fill use the layer's fill"); } @@ -1733,7 +1757,46 @@ impl<'a> Mapper<'a> { o.insert("strokeCapStart".into(), json!(cap_name(first.unwrap_or(node)))); o.insert("strokeCapEnd".into(), json!(cap_name(last.unwrap_or(node)))); } - o.insert("vectorNetwork".into(), json!({"vertices": vertices, "segments": segments, "regions": regions})); + // Every instance of a main shows the same network: it is written once and shared. + let key = (blob_id, sx.to_bits(), sy.to_bits(), std::ptr::from_ref(vd) as usize); + let id = match self.network_ids.get(&key) { + Some(id) => *id, + None => { + let vertices: Vec = net + .vertices + .iter() + .map(|p| { + let mut out = json!({"x": p.x * sx, "y": p.y * sy}); + let st = style(p.style); + if let Some(r) = st.and_then(|s| num(s, "cornerRadius")).filter(|r| *r > 0.0) { + out["radius"] = json!(r); + } + match st.and_then(|s| s.get("handleMirroring")).and_then(Kv::str) { + Some("ANGLE") => out["mirror"] = json!("angle"), + Some("ANGLE_AND_LENGTH") => out["mirror"] = json!("angleAndLength"), + Some("NONE") => out["mirror"] = json!("none"), + _ => {} + } + out + }) + .collect(); + let segments: Vec = net + .segments + .iter() + .map(|s| json!({"a": s.a, "b": s.b, "ta": [s.ta.0 * sx, s.ta.1 * sy], "tb": [s.tb.0 * sx, s.tb.1 * sy]})) + .collect(); + let regions: Vec = + net.regions.iter().map(|r| json!({"loops": r.loops, "winding": if r.non_zero { "nonZero" } else { "evenOdd" }})).collect(); + let mut entry = json!({}); + entry["vertices"] = Value::Array(vertices); + entry["segments"] = Value::Array(segments); + entry["regions"] = Value::Array(regions); + self.networks.push(crate::rest::network(&entry)); + self.network_ids.insert(key, self.networks.len() - 1); + self.networks.len() - 1 + } + }; + o.insert("vectorNetworkRef".into(), json!(id)); true } diff --git a/crates/format/src/fig/map_connector_snapshot_budget_tests.rs b/crates/format/src/fig/map_connector_snapshot_budget_tests.rs index a14e671c..4d8ae9be 100644 --- a/crates/format/src/fig/map_connector_snapshot_budget_tests.rs +++ b/crates/format/src/fig/map_connector_snapshot_budget_tests.rs @@ -106,7 +106,7 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { let message = message(); let node = message.get("nodeChanges").unwrap().list().first().unwrap(); let view = View::of(node); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES - 1; let children = mapper.connector_snapshot_children(&view, "1:123"); assert_eq!(children.len(), 1, "the final available slot preserves the saved vector"); @@ -116,7 +116,7 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { assert_eq!(mapper.count, MAX_NODES); assert!(mapper.warnings.iter().any(|warning| warning == &format!("the file has more than {MAX_NODES} layers; connector label was skipped"))); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES; let children = mapper.connector_snapshot_children(&view, "1:123"); assert!(children.is_empty()); @@ -127,14 +127,14 @@ fn connector_snapshot_respects_last_and_zero_child_slots() { #[test] fn connector_snapshot_flattens_vector_at_depth_and_node_limits() { let message = message(); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); let depth_limited = mapper.layer("1:123", None, Affine::IDENTITY, None, None, MAX_DEPTH).expect("terminal connector"); assert_eq!(depth_limited["type"], "VECTOR"); assert!(!depth_limited["fillGeometry"].as_array().unwrap().is_empty()); assert!(depth_limited.get("children").is_none()); assert!(mapper.warnings.iter().any(|warning| warning == "connector label was skipped: depth limit prevents nested snapshot children")); - let mut mapper = Mapper::new(&message); + let mut mapper = Mapper::new(&message, MAX_NODES); mapper.count = MAX_NODES - 1; let budget_limited = mapper.layer("1:123", None, Affine::IDENTITY, None, None, 0).expect("last allowed connector"); assert_eq!(budget_limited["type"], "VECTOR"); diff --git a/crates/format/src/fig/mod.rs b/crates/format/src/fig/mod.rs index 0d341307..08a3409e 100644 --- a/crates/format/src/fig/mod.rs +++ b/crates/format/src/fig/mod.rs @@ -54,6 +54,18 @@ const MAX_CHUNK: u64 = 512 * 1024 * 1024; const MAX_IMAGE: u64 = 256 * 1024 * 1024; /// Most chunks read from `canvas.fig`. const MAX_CHUNKS: usize = 16; +/// Most layers imported from a `.fig` file: one per [`PACKED_BYTES_PER_LAYER`] bytes of its +/// compressed canvas, at least the REST import's cap and at most [`MAX_FIG_NODES`]. Instances +/// expand into a layer per layer of their component, and chunks inflate up to [`MAX_CHUNK`], so the +/// budget follows what the file takes, not what it unpacks to: a small file can't claim a large +/// count, however it's built; a large one gets room. +const PACKED_BYTES_PER_LAYER: usize = 32; +const MAX_FIG_NODES: usize = 2_000_000; + +/// Most layers imported from a `.fig` file whose compressed canvas is `packed` bytes. +pub(crate) fn layer_budget(packed: usize) -> usize { + (packed / PACKED_BYTES_PER_LAYER).clamp(crate::rest::MAX_NODES, MAX_FIG_NODES) +} /// Whether `bytes` look like a `.fig` file: a zip holding `canvas.fig`, or a bare canvas. pub fn is_fig(bytes: &[u8]) -> bool { @@ -72,14 +84,22 @@ pub fn try_import(bytes: &[u8], name: &str) -> Option Result<(Document, Vec), FormatError> { - let archive = Archive::open(bytes)?; - let (schema, message) = decode_canvas(&archive.canvas)?; + let mut archive = Archive::open(bytes)?; + // The budgets follow the compressed canvas, which the file itself bounds (its zip member may + // inflate too). + let packed = archive.canvas.len().min(bytes.len()); + // Each stage's input is dropped as soon as the next one is built: in a large file every one + // of them takes gigabytes, and holding them all at once runs out of memory. + let (schema, message) = decode_canvas(&std::mem::take(&mut archive.canvas))?; let root = schema.find("Message").ok_or_else(|| corrupt("the schema has no Message definition"))?; - let msg = schema.decode_message(root, &message).map_err(|e| corrupt(e.to_string()))?; + let msg = schema.decode_message_within(root, &message, kiwi::value_budget(packed)).map_err(|e| corrupt(e.to_string()))?; + let max_nodes = layer_budget(packed); + drop(message); let file_name = archive.file_name.as_deref().filter(|n| !n.trim().is_empty()).unwrap_or(name); - let (json, mut warnings) = map::to_rest(&msg, file_name); + let (json, mut warnings, networks) = map::to_rest_parts(&msg, file_name, max_nodes); + drop(msg); let images = |r: &str| archive.image(r); - let (doc, more) = crate::rest::import_fig(&json, &images)?; + let (doc, more) = crate::rest::import_fig_parts(&json, &images, networks, max_nodes)?; warnings.extend(more.into_iter().filter_map(map::adapt_warning)); Ok((doc, warnings)) } diff --git a/crates/format/src/fig/tail_tests.rs b/crates/format/src/fig/tail_tests.rs index a15334a5..43e997a5 100644 --- a/crates/format/src/fig/tail_tests.rs +++ b/crates/format/src/fig/tail_tests.rs @@ -446,8 +446,8 @@ fn mangled_tail_files_never_panic() { m[i] = r.next() as u8; } if let Ok(kv) = sc.decode_message(root, &m) { - let (json, _) = map::to_rest(&kv, "x"); - let _ = crate::rest::import(&json, &crate::rest::no_images); + let (json, _, networks) = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); + let _ = crate::rest::import_fig_parts(&json, &crate::rest::no_images, networks, crate::rest::MAX_NODES); } } } diff --git a/crates/format/src/fig/tests.rs b/crates/format/src/fig/tests.rs index 24baef30..5f20f5ac 100644 --- a/crates/format/src/fig/tests.rs +++ b/crates/format/src/fig/tests.rs @@ -757,7 +757,7 @@ fn mangled_kiwi_never_panics() { } for cut in (0..msg.len()).step_by(3) { if let Ok(kv) = sc.decode_message(root, &msg[..cut]) { - let _ = map::to_rest(&kv, "x"); + let _ = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); } } for _ in 0..2000 { @@ -767,8 +767,8 @@ fn mangled_kiwi_never_panics() { m[i] = r.next() as u8; } if let Ok(kv) = sc.decode_message(root, &m) { - let (json, _) = map::to_rest(&kv, "x"); - let _ = crate::rest::import(&json, &crate::rest::no_images); + let (json, _, networks) = map::to_rest_parts(&kv, "x", crate::rest::MAX_NODES); + let _ = crate::rest::import_fig_parts(&json, &crate::rest::no_images, networks, crate::rest::MAX_NODES); } let mut sbytes = schema_bytes.clone(); let i = (r.next() as usize) % sbytes.len(); @@ -790,6 +790,59 @@ fn mangled_kiwi_never_panics() { } } +#[test] +fn large_files_get_a_larger_value_budget() { + // Small files keep the fixed floor: a few bytes can't expand into a huge tree. + assert_eq!(kiwi::value_budget(0), 20_000_000); + assert_eq!(kiwi::value_budget(1 << 20), 20_000_000); + // A real 400 000-layer file: an 83.5 MB canvas whose message unpacks to 334 MB and holds + // 30.5 million values was refused as "too many values". + assert!(kiwi::value_budget(83_491_232) > 30_514_899); + // Still bounded by the largest canvas read. + assert!(kiwi::value_budget(MAX_CANVAS as usize) <= 128 << 20); +} + +#[test] +fn the_layer_cap_follows_the_file_size() { + // A small (or broken) file keeps the REST import's cap: instances expand into a layer per + // layer of their component, and a few bytes mustn't claim millions of layers. + assert_eq!(layer_budget(0), crate::rest::MAX_NODES); + assert_eq!(layer_budget(1 << 20), crate::rest::MAX_NODES); + // A real 894 004-layer file (an 83.5 MB canvas) stopped at 200 000 and came in with 27 of + // its 30 pages empty. + assert!(layer_budget(83_491_232) > 894_004); + // No file gets more than the ceiling. + assert_eq!(layer_budget(MAX_CANVAS as usize), MAX_FIG_NODES); +} + +#[test] +fn a_small_file_that_unpacks_large_gets_no_larger_budgets() { + // 40 MB of zeros in a blob pack into a few KB. Keyed to the unpacked message, they lifted the + // layer cap above the 231 231 layers its instances expand into; keyed to the file, the cap + // stays at the REST import's. + let mut nodes = vec![node(0, None, "DOCUMENT", "D", vec![]), node(1, Some((0, "!")), "CANVAS", "P", vec![])]; + nodes.push(node(5, Some((1, "!")), "SYMBOL", "Main", vec![("size", xy(10.0, 10.0))])); + nodes.extend((10..1010).map(|i| node(i, Some((5, "!")), "RECTANGLE", "R", vec![("size", xy(1.0, 1.0))]))); + nodes.extend((2000..2230).map(|i| node(i, Some((1, "!")), "INSTANCE", "I", vec![("symbolData", o(vec![("symbolID", g(5))]))]))); + let blob = o(vec![("bytes", Kv::Bytes(vec![0; 40 << 20]))]); + let file = canvas(&schema(), &o(vec![("nodeChanges", arr(nodes)), ("blobs", arr(vec![blob]))])); + assert!(file.len() < 1 << 20, "{} bytes", file.len()); + let (doc, warnings) = import(&file, "x").unwrap(); + assert!(warnings.iter().any(|w| w.contains("more than 200000 layers")), "{warnings:?}"); + assert!(doc.node_count() <= crate::rest::MAX_NODES + 10); +} + +#[test] +fn the_mapper_stops_at_the_layer_cap_it_is_given() { + let mut nodes = vec![node(0, None, "DOCUMENT", "D", vec![]), node(1, Some((0, "!")), "CANVAS", "P", vec![])]; + nodes.extend((2..12).map(|i| node(i, Some((1, "!")), "RECTANGLE", "R", vec![("size", xy(1.0, 1.0))]))); + let msg = o(vec![("nodeChanges", arr(nodes))]); + let (_, warnings, _) = map::to_rest_parts(&msg, "x", 5); + assert!(warnings.iter().any(|w| w.contains("more than 5 layers")), "{warnings:?}"); + let (_, warnings, _) = map::to_rest_parts(&msg, "x", crate::rest::MAX_NODES); + assert!(!warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); +} + #[test] fn hostile_input_is_refused() { // A message that contains itself, nested past the depth cap. @@ -1088,7 +1141,7 @@ fn dump_samples() { std::fs::write(format!("{out}/{stem}.schema.txt"), txt).unwrap(); let msg = schema.decode_message(schema.find("Message").unwrap(), &message).unwrap(); std::fs::write(format!("{out}/{stem}.json"), serde_json::to_string_pretty(&debug_json(&msg)).unwrap()).unwrap(); - let (rest, _) = map::to_rest(&msg, &stem); + let (rest, ..) = map::to_rest_parts(&msg, &stem, super::layer_budget(message.len())); std::fs::write(format!("{out}/{stem}.rest.json"), serde_json::to_string_pretty(&rest).unwrap()).unwrap(); } } diff --git a/crates/format/src/rest/mod.rs b/crates/format/src/rest/mod.rs index e460c8b8..0fa53a7c 100644 --- a/crates/format/src/rest/mod.rs +++ b/crates/format/src/rest/mod.rs @@ -53,7 +53,7 @@ use crate::FormatError; use props::{MAX_COORD, MAX_SIZE, clip_str, flag, list, num, text}; pub use variables::export as export_variables; -/// Most layers imported from one file. +/// Most layers imported from one file. The `.fig` import sets its own, from the file's size. pub const MAX_NODES: usize = 200_000; /// Deepest layer nesting followed. const MAX_DEPTH: usize = 256; @@ -146,22 +146,43 @@ pub fn import(v: &Value, images: ImageResolver) -> Result<(Document, Vec /// in the file JSON (a `meta` with `variableCollections`, or `variableCollections` and /// `variables` at the top level) are read too; the first definition of an id wins. pub fn import_with(v: &Value, variables: Option<&Value>, images: ImageResolver) -> Result<(Document, Vec), FormatError> { - import_impl(v, variables, images, false) + import_impl(v, variables, images, false, Vec::new(), MAX_NODES) } -/// Internal .fig JSON path: GUID provenance is not part of the public REST import contract. +/// Internal .fig JSON path: GUID provenance is not part of the public REST import contract. Its +/// vectors name their networks by `vectorNetworkRef`, an index in `networks` (built once each, and +/// far smaller than as JSON); `max_nodes` is the layer cap the `.fig` import sets in place of +/// [`MAX_NODES`]. +pub(crate) fn import_fig_parts( + v: &Value, + images: ImageResolver, + networks: Vec>, + max_nodes: usize, +) -> Result<(Document, Vec), FormatError> { + import_impl(v, None, images, true, networks, max_nodes) +} + +/// [`import_fig_parts`] of JSON that carries its vectors' networks, within [`MAX_NODES`]. +#[cfg(test)] pub(crate) fn import_fig(v: &Value, images: ImageResolver) -> Result<(Document, Vec), FormatError> { - import_impl(v, None, images, true) + import_fig_parts(v, images, Vec::new(), MAX_NODES) } -fn import_impl(v: &Value, variables: Option<&Value>, images: ImageResolver, fig_sources: bool) -> Result<(Document, Vec), FormatError> { +fn import_impl( + v: &Value, + variables: Option<&Value>, + images: ImageResolver, + fig_sources: bool, + networks: Vec>, + max_nodes: usize, +) -> Result<(Document, Vec), FormatError> { let entries: Vec<&Value> = if is_document(v) { vec![v] } else { nodes_entries(v).collect() }; if entries.is_empty() { return Err(FormatError::Corrupt("not a REST API file JSON (no `document`)".into())); } let name = text(v, "name").filter(|n| !n.trim().is_empty()).map_or("Imported file", clip_str); let mut cx = Ctx::new(Document::new(name), images); - cx.fig_sources = fig_sources.then(|| fig_sources::Sources::new(MAX_NODES)); + cx.fig_sources = fig_sources.then(|| fig_sources::Sources::new(max_nodes)); let blank = cx.doc.first_page(); let mut loose: Option = None; for e in &entries { @@ -175,6 +196,8 @@ fn import_impl(v: &Value, variables: Option<&Value>, images: ImageResolver, fig_ cx.warn("the variables JSON has no `variableCollections` (expected the local variables response) and was ignored"); } cx.style_definitions(v); + cx.networks = networks; + cx.max_nodes = max_nodes; for e in entries { let Some(root) = e.get("document") else { continue }; match text(root, "type") { @@ -356,6 +379,10 @@ pub(crate) struct Ctx<'a> { /// Imported variables by id, and the mode ids of imported collections (default first). var_kinds: BTreeMap, var_modes: BTreeMap>, + /// Vector networks built by the `.fig` import, named by layers' `vectorNetworkRef`. + networks: Vec>, + /// Most layers imported: [`MAX_NODES`], or the `.fig` import's own cap. + max_nodes: usize, } impl<'a> Ctx<'a> { @@ -380,6 +407,8 @@ impl<'a> Ctx<'a> { overlay_meta: BTreeMap::new(), var_kinds: BTreeMap::new(), var_modes: BTreeMap::new(), + networks: Vec::new(), + max_nodes: MAX_NODES, } } @@ -435,8 +464,9 @@ impl<'a> Ctx<'a> { self.warn(format!("layers nested deeper than {MAX_DEPTH} levels were skipped")); return; } - if self.count >= MAX_NODES { - self.warn(format!("the file has more than {MAX_NODES} layers; the rest were skipped")); + if self.count >= self.max_nodes { + let max = self.max_nodes; + self.warn(format!("the file has more than {max} layers; the rest were skipped")); return; } let Some(ty) = text(j, "type") else { @@ -642,14 +672,18 @@ impl<'a> Ctx<'a> { } } - /// A vector's network: the editable one when the JSON carries it (`vectorNetwork`, our - /// extension, written by the `.fig` import), else from its geometry. A stroke-only vector - /// uses the stroke outline, painted with the stroke paints (exact look, editable as a shape): - /// returns whether it does, and the caller moves the paints with [`outline_stroke`] once the - /// styles and variables of the stroke are read. + /// A vector's network: the editable one when the JSON carries it (`vectorNetwork`, or + /// `vectorNetworkRef` naming one built by the `.fig` import: our extensions), else from its + /// geometry. A stroke-only vector uses the stroke outline, painted with the stroke paints + /// (exact look, editable as a shape): returns whether it does, and the caller moves the paints + /// with [`outline_stroke`] once the styles and variables of the stroke are read. fn vector(&mut self, n: &mut Node, j: &Value) -> bool { - if let Some(net) = j.get("vectorNetwork") { - match network(net) { + let net = match j.get("vectorNetworkRef") { + Some(r) => Some(r.as_u64().and_then(|i| self.networks.get(usize::try_from(i).ok()?).cloned().flatten())), + None => j.get("vectorNetwork").map(network), + }; + if let Some(net) = net { + match net { Some(network) => { n.kind = NodeKind::Vector { network }; return false; @@ -1173,7 +1207,7 @@ fn local(parent: Affine, world: Affine) -> Affine { /// Finite path data within the coordinate cap. /// A vector network in our own JSON shape, when it's within bounds and refers only to what it has. -fn network(v: &Value) -> Option { +pub(crate) fn network(v: &Value) -> Option { use serde::Deserialize; let mut net = VectorNetwork::deserialize(v).ok()?; let entries: usize = net.regions.iter().flat_map(|r| &r.loops).map(Vec::len).sum(); diff --git a/crates/format/src/rest/tests.rs b/crates/format/src/rest/tests.rs index 7db22db1..cdf27905 100644 --- a/crates/format/src/rest/tests.rs +++ b/crates/format/src/rest/tests.rs @@ -729,3 +729,18 @@ fn layer_count_is_capped() { assert_eq!(doc.node_count(), MAX_NODES + 1); assert!(warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); } + +#[test] +fn the_fig_import_sets_its_own_layer_cap() { + // The `.fig` import passes a cap from the file's size: above `MAX_NODES` every layer comes in. + let kids: Vec = (0..MAX_NODES + 5).map(|i| json!({"id": i.to_string(), "type": "RECTANGLE", "name": "R"})).collect(); + let json = file(json!([page("Page", Value::Array(kids))])); + let (doc, warnings) = import_fig_parts(&json, &no_images, Vec::new(), MAX_NODES + 10).unwrap(); + assert_eq!(doc.node_count(), MAX_NODES + 6); + assert!(!warnings.iter().any(|w| w.contains("more than")), "{warnings:?}"); + // Below it, the import stops at the cap it was given. + let few: Vec = (0..10).map(|i| json!({"id": i.to_string(), "type": "RECTANGLE", "name": "R"})).collect(); + let (doc, warnings) = import_fig_parts(&file(json!([page("Page", Value::Array(few))])), &no_images, Vec::new(), 4).unwrap(); + assert_eq!(doc.node_count(), 5); + assert!(warnings.iter().any(|w| w.contains("more than 4 layers")), "{warnings:?}"); +}