Merge #989: Large .fig files keep their layers and their library links (#981) (@Ridzzz0Alam)

crates/format/src/fig/layer_cap_tests.rs

@@ -0,0 +1,138 @@
+//! Large `.fig` files at the layer cap (#981): the budget a design-system file gets, the library
+//! copies its instances need kept ahead of the pages, honest notes for what the cap kept out, and
+//! one note for the cap itself.
+
+use figcraft_doc::{Document, NodeKind};
+
+use super::kiwi::{Kv, Schema};
+use super::*;
+
+type Fields = Vec<(&'static str, Kv<'static>)>;
+
+fn o(fields: Fields) -> Kv<'static> {
+    Kv::Object(fields)
+}
+
+fn g(id: u64) -> Kv<'static> {
+    o(vec![("sessionID", Kv::Uint(1)), ("localID", Kv::Uint(id))])
+}
+
+fn xy(x: f64, y: f64) -> Kv<'static> {
+    o(vec![("x", Kv::Float(x as f32)), ("y", Kv::Float(y as f32))])
+}
+
+/// A layer under `parent`; siblings come in id order.
+fn node(id: u64, parent: Option<u64>, ty: &'static str, name: &str, more: Fields) -> Kv<'static> {
+    let mut fields = vec![("guid", g(id)), ("phase", Kv::Enum("CREATED")), ("type", Kv::Enum(ty)), ("name", Kv::Str(name.into()))];
+    if let Some(parent) = parent {
+        fields.push(("parentIndex", o(vec![("guid", g(parent)), ("position", Kv::Str(format!("{id:08}")))])));
+    }
+    fields.extend(more);
+    o(fields)
+}
+
+fn instance(id: u64, page: u64, main: u64) -> Kv<'static> {
+    node(id, Some(page), "INSTANCE", "Instance", vec![("symbolData", o(vec![("symbolID", g(main))]))])
+}
+
+/// A main `id` of `layers` rectangles under `parent`.
+fn main(id: u64, parent: u64, name: &str, layers: u64) -> Vec<Kv<'static>> {
+    let mut out = vec![node(id, Some(parent), "SYMBOL", name, vec![("size", xy(10.0, 10.0))])];
+    out.extend((1..=layers).map(|i| node(id * 10_000 + i, Some(id), "RECTANGLE", "R", vec![("size", xy(1.0, 1.0))])));
+    out
+}
+
+/// A page (2) of `count` instances of a main kept on the file's hidden library canvas (3), as a
+/// design system's file carries the library components it uses.
+fn library_message(count: u64, main_layers: u64) -> Kv<'static> {
+    let mut nodes = vec![
+        node(1, None, "DOCUMENT", "Document", vec![]),
+        node(2, Some(1), "CANVAS", "Screens", vec![]),
+        node(3, Some(1), "CANVAS", "Internal", vec![("internalOnly", Kv::Bool(true))]),
+    ];
+    nodes.extend(main(20, 3, "Library main", main_layers));
+    nodes.extend((0..count).map(|i| instance(1_000 + i, 2, 20)));
+    o(vec![("nodeChanges", Kv::Array(nodes))])
+}
+
+/// The `.fig` import's two stages at layer cap `cap`.
+fn import_at(msg: &Kv, cap: usize) -> (Document, Vec<String>) {
+    let (json, mut warnings, networks) = map::to_rest(msg, "Capped", cap);
+    let (doc, more) = crate::rest::import_with_networks(&json, &|_: &str| None, networks, cap).unwrap();
+    warnings.extend(more);
+    (doc, warnings)
+}
+
+fn linked_to(doc: &Document, main_name: &str) -> usize {
+    doc.nodes().filter(|n| matches!(n.kind, NodeKind::Instance { component, .. } if doc.node(component).is_some_and(|c| c.name == main_name))).count()
+}
+
+#[test]
+fn a_design_system_sized_canvas_gets_room() {
+    // The file in #981 stopped at 244 248 layers: a 7.8 MB canvas at 32 bytes a layer.
+    let reported = 244_248 * 32;
+    assert!(layer_budget(reported) >= 4 * 244_248, "{}", layer_budget(reported));
+    // Small files still can't claim more than the REST import's cap, nor any file the ceiling.
+    assert_eq!(layer_budget(1 << 20), crate::rest::MAX_NODES);
+    assert_eq!(layer_budget(usize::MAX), MAX_FIG_NODES);
+}
+
+#[test]
+fn library_mains_come_before_the_pages_at_the_cap() {
+    // 40 instances of an 11-layer library main on a page, with room for 100 layers: mapped after
+    // the pages, the main found no room and every instance became a frame of "a component that
+    // isn't in the file".
+    let (doc, warnings) = import_at(&library_message(40, 10), 100);
+    assert!(warnings.iter().any(|w| w.contains("more than 100 layers")), "{warnings:?}");
+    assert!(!warnings.iter().any(|w| w.contains("aren't in the file")), "{warnings:?}");
+    let linked = linked_to(&doc, "Library main");
+    assert!(linked >= 5, "{linked} instances linked to the library main: {warnings:?}");
+    assert!(doc.nodes().any(|n| n.name == "Library main" && doc.is_remote(n.id)));
+}
+
+#[test]
+fn instances_of_a_main_the_cap_kept_out_say_so() {
+    // The page's instances come before their main: at the cap, the main is skipped though the
+    // file has it.
+    let mut nodes = vec![node(1, None, "DOCUMENT", "Document", vec![]), node(2, Some(1), "CANVAS", "Page", vec![])];
+    nodes.extend((0..20).map(|i| instance(100 + i, 2, 900)));
+    nodes.extend(main(900, 2, "Local main", 10));
+    let (doc, warnings) = import_at(&o(vec![("nodeChanges", Kv::Array(nodes))]), 60);
+    assert!(warnings.iter().any(|w| w.starts_with("instances whose component was past the layer limit were imported as frames")), "{warnings:?}");
+    assert!(!warnings.iter().any(|w| w.contains("aren't in the file")), "{warnings:?}");
+    assert!(!doc.nodes().any(|n| n.name == "Local main"));
+}
+
+#[test]
+fn swap_defaults_the_cap_kept_out_say_so() {
+    // A kept main's instance swap default names a main with no instances, which comes after
+    // enough layers to be skipped at the cap.
+    let swap = o(vec![
+        ("id", g(70)),
+        ("name", Kv::Str("Icon".into())),
+        ("type", Kv::Enum("INSTANCE_SWAP")),
+        ("initialValue", o(vec![("guidValue", g(900))])),
+    ]);
+    let mut nodes = vec![node(1, None, "DOCUMENT", "Document", vec![]), node(2, Some(1), "CANVAS", "Page", vec![])];
+    nodes.push(node(50, Some(2), "SYMBOL", "Card", vec![("size", xy(10.0, 10.0)), ("componentPropDefs", Kv::Array(vec![swap]))]));
+    nodes.extend(main(60, 2, "Filler", 40));
+    nodes.extend(main(900, 2, "Icon main", 10));
+    let (doc, warnings) = import_at(&o(vec![("nodeChanges", Kv::Array(nodes))]), 30);
+    assert!(doc.nodes().any(|n| n.name == "Card"));
+    assert!(!doc.nodes().any(|n| n.name == "Icon main"));
+    assert!(warnings.iter().any(|w| w.starts_with("instance swap properties whose default component was past the layer limit")), "{warnings:?}");
+    assert!(!warnings.iter().any(|w| w.contains("isn't in the file")), "{warnings:?}");
+}
+
+#[test]
+fn a_whole_file_past_the_cap_keeps_its_library_links_and_says_so_once() {
+    // A 1 001-layer library main used 210 times: about 210 000 layers in a file far too small to
+    // lift the cap above the REST import's 200 000.
+    let file = super::tests::canvas(&Schema::parse(export::SCHEMA).unwrap(), &library_message(210, 1_000));
+    assert_eq!(layer_budget(file.len()), crate::rest::MAX_NODES);
+    let (doc, warnings) = import(&file, "Design system.fig").unwrap();
+    let cap: Vec<_> = warnings.iter().filter(|w| w.contains("layers; the rest were skipped")).collect();
+    assert_eq!(cap.len(), 1, "{warnings:?}");
+    assert!(!warnings.iter().any(|w| w.contains("aren't in the file")), "{warnings:?}");
+    assert!(linked_to(&doc, "Library main") >= 190, "{warnings:?}");
+}

crates/format/src/fig/map.rs

@@ -42,6 +42,14 @@ pub(super) fn to_rest(msg: &Kv, name: &str, max_nodes: usize) -> (Value, Vec<Str
     if !m.library_contexts.is_empty() {
         out[crate::rest::FIG_LIBRARY_CONTEXTS_FIELD] = Value::Array(std::mem::take(&mut m.library_contexts));
     }
+    // Mains in the file that the layer cap kept out: their instances are told apart from those of
+    // components the file doesn't have (#981).
+    if m.count >= m.max_nodes {
+        let capped: Vec<Value> = m.referenced_mains.difference(&m.mapped_mains).map(|id| json!(id)).collect();
+        if !capped.is_empty() {
+            out[crate::rest::FIG_CAPPED_MAINS_FIELD] = Value::Array(capped);
+        }
+    }
     if !m.styles.is_empty() {
         out["styles"] = Value::Object(std::mem::take(&mut m.styles));
     }
@@ -389,6 +397,10 @@ struct Mapper<'a> {
     library_scan_count: usize,
     warnings: crate::rest::Warnings,
     count: usize,
+    /// The mains in the file that instances or instance swap defaults name, and the mains mapped
+    /// as layers: the difference is what the layer cap kept out.
+    referenced_mains: BTreeSet<String>,
+    mapped_mains: BTreeSet<String>,
     /// Most layers produced or scanned, instance expansion included; the one place the cap comes
     /// from (see [`super::layer_budget`]).
     max_nodes: usize,
@@ -466,6 +478,8 @@ impl<'a> Mapper<'a> {
             library_scan_count: 0,
             warnings: crate::rest::Warnings::default(),
             count: 0,
+            referenced_mains: BTreeSet::new(),
+            mapped_mains: BTreeSet::new(),
             max_nodes,
             networks: Vec::new(),
             network_ids: HashMap::new(),
@@ -558,9 +572,8 @@ impl<'a> Mapper<'a> {
             Some(d) => self.children(d),
             None => self.nodes.iter().filter(|(_, n)| n.get("type").and_then(Kv::str) == Some("CANVAS")).map(|(id, _)| id.clone()).collect(),
         };
-        let mut out = Vec::new();
         let mut internal = Vec::new();
-        let mut visible_pages = Vec::new();
+        let mut visible = Vec::new();
         for p in pages {
             let Some(&n) = self.nodes.get(&p) else { continue };
             if n.get("type").and_then(Kv::str) != Some("CANVAS") {
@@ -570,17 +583,20 @@ impl<'a> Mapper<'a> {
             // The hidden page that holds styles, variables and library copies.
             if n.get("internalOnly").and_then(Kv::bool) == Some(true) {
                 internal.push((p, n));
-                continue;
+            } else {
+                visible.push((p, n));
             }
-            visible_pages.push(p.clone());
-            out.push(self.page(&p, n));
         }
+        // The library copies the pages use come first: in a file past the layer cap, the pages'
+        // last layers are skipped rather than the mains every instance of them needs (#981).
+        let visible_pages: Vec<String> = visible.iter().map(|(p, _)| p.clone()).collect();
         let required_mains = self.referenced_component_mains(&visible_pages);
         for (id, canvas) in internal {
             if let Some(context) = self.library_context(&id, canvas, &required_mains) {
                 self.library_contexts.push(context);
             }
         }
+        let out: Vec<Value> = visible.into_iter().map(|(p, n)| self.page(&p, n)).collect();
         if out.is_empty() {
             self.warn("the file has no pages");
         }
@@ -881,6 +897,9 @@ impl<'a> Mapper<'a> {
             self.warn("connector layers import as static frame/vector snapshots with labels when present; live endpoint bindings are not retained");
         }
         self.count += 1;
+        if own == "SYMBOL" && inst.is_none() {
+            self.mapped_mains.insert(id.to_string());
+        }
         let mut o = Obj::new();
         o.insert("id".into(), json!(rest_id));
         // Private provenance for the .fig importer; ordinary REST import ignores it.
@@ -1852,7 +1871,12 @@ impl<'a> Mapper<'a> {
                 "BOOL" => json!({"type": "BOOLEAN", "defaultValue": init.and_then(|i| i.get("boolValue")).and_then(Kv::bool).unwrap_or(true)}),
                 "TEXT" => json!({"type": "TEXT", "defaultValue": init.and_then(prop_text).unwrap_or("")}),
                 "INSTANCE_SWAP" => match init.and_then(|i| i.get("guidValue")).and_then(guid) {
-                    Some(g) => json!({"type": "INSTANCE_SWAP", "defaultValue": g}),
+                    Some(g) => {
+                        if self.nodes.contains_key(&g) {
+                            self.referenced_mains.insert(g.clone());
+                        }
+                        json!({"type": "INSTANCE_SWAP", "defaultValue": g})
+                    }
                     None => continue,
                 },
                 "VARIANT" => json!({"type": "VARIANT"}),
@@ -1996,6 +2020,9 @@ impl<'a> Mapper<'a> {
             self.warn("instances of components that aren't in the file show no layers");
             return None;
         }
+        if !self.referenced_mains.contains(&sym) {
+            self.referenced_mains.insert(sym.clone());
+        }
         if outer.is_some_and(|i| i.symbols.contains(&sym)) {
             return None;
         }

crates/format/src/fig/mod.rs

@@ -23,6 +23,8 @@ mod export_tests;
 #[cfg(test)]
 mod interop_tests;
 pub mod kiwi;
+#[cfg(test)]
+mod layer_cap_tests;
 mod map;
 mod network;
 #[cfg(test)]
@@ -64,8 +66,11 @@ const MAX_CHUNKS: usize = 16;
 /// compressed canvas, at least the REST import's cap and at most [`MAX_FIG_NODES`]. Instances
 /// expand into a layer per layer of their component, and chunks inflate up to [`MAX_CHUNK`], so the
 /// budget follows what the file takes, not what it unpacks to: a small file can't claim a large
-/// count, however it's built; a large one gets room.
-const PACKED_BYTES_PER_LAYER: usize = 32;
+/// count, however it's built; a large one gets room. Design systems are mostly instances, which
+/// pack tightly: a 7.8 MB canvas stopped at 244 248 layers at 32 bytes a layer (#981). At 8, files
+/// under about 1.6 MB keep the floor and the ceiling comes at a 16 MB canvas (about 11 GB and 40 s
+/// to import 2 million synthetic layers; memory grows with the count, about 5.4 KB a layer).
+const PACKED_BYTES_PER_LAYER: usize = 8;
 const MAX_FIG_NODES: usize = 2_000_000;
 
 /// Most layers imported from a `.fig` file whose compressed canvas is `packed` bytes.
@@ -108,7 +113,13 @@ pub fn import(bytes: &[u8], name: &str) -> Result<(Document, Vec<String>), Forma
     drop(msg);
     let images = |r: &str| archive.image(r).or_else(|| embedded.get(r).cloned());
     let (doc, more) = crate::rest::import_with_networks(&json, &images, networks, max_nodes)?;
-    warnings.extend(more.into_iter().filter_map(map::adapt_warning));
+    // Both stages stop at the same cap and say so in the same words: one note (#981).
+    for w in more.into_iter().filter_map(map::adapt_warning) {
+        let base = w.rsplit_once(" (×").map_or(w.as_str(), |(b, _)| b);
+        if !warnings.iter().any(|seen| seen == base) {
+            warnings.push(w);
+        }
+    }
     Ok((doc, warnings))
 }
 

crates/format/src/rest/mod.rs

@@ -65,6 +65,8 @@ const MAX_DEPTH: usize = 256;
 pub(crate) const FIG_SOURCE_FIELD: &str = "_figSource";
 /// Hidden .fig library context; accepted only by the `.fig` import ([`import_with_networks`]).
 pub(crate) const FIG_LIBRARY_CONTEXTS_FIELD: &str = "_figLibraryContexts";
+/// Ids of mains in a `.fig` file that its layer cap kept out; read only by the `.fig` import.
+pub(crate) const FIG_CAPPED_MAINS_FIELD: &str = "_figCappedMains";
 /// Longest SVG path string read from one geometry entry.
 const MAX_PATH_BYTES: usize = 4 << 20;
 /// Most path elements in one vector.
@@ -215,6 +217,14 @@ fn import_parts(
     cx.style_definitions(v);
     cx.networks = networks;
     cx.max_nodes = max_nodes;
+    // The `.fig` mapper already kept the layers, library copies included, within the cap; each
+    // library copy's wrapper frame is the one layer here it doesn't count (#981).
+    let contexts = if fig_sources { v.get(FIG_LIBRARY_CONTEXTS_FIELD).and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default() } else { &[] };
+    if fig_sources {
+        cx.capped_mains =
+            v.get(FIG_CAPPED_MAINS_FIELD).and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str).map(str::to_owned).collect();
+        cx.max_nodes = max_nodes.saturating_add(contexts.len());
+    }
     for e in entries {
         let Some(root) = e.get("document") else { continue };
         match text(root, "type") {
@@ -236,11 +246,9 @@ fn import_parts(
             None => cx.warn("entries without a node type were skipped"),
         }
     }
-    if fig_sources {
-        for context in v.get(FIG_LIBRARY_CONTEXTS_FIELD).and_then(Value::as_array).into_iter().flatten() {
-            let store = cx.doc.ensure_library_store();
-            cx.build(context, store, Space::default(), None, 0, false);
-        }
+    for context in contexts {
+        let store = cx.doc.ensure_library_store();
+        cx.build(context, store, Space::default(), None, 0, false);
     }
     // Drop the empty page the new document started with once real pages exist.
     if let Some(p) = blank
@@ -415,6 +423,8 @@ pub(crate) struct Ctx<'a> {
     networks: Vec<Option<VectorNetwork>>,
     /// Most layers imported: [`MAX_NODES`], or the `.fig` import's own cap.
     max_nodes: usize,
+    /// Mains the `.fig` file has but its layer cap kept out ([`FIG_CAPPED_MAINS_FIELD`]).
+    capped_mains: BTreeSet<String>,
 }
 
 impl<'a> Ctx<'a> {
@@ -443,6 +453,7 @@ impl<'a> Ctx<'a> {
             var_modes: BTreeMap::new(),
             networks: Vec::new(),
             max_nodes: MAX_NODES,
+            capped_mains: BTreeSet::new(),
         }
     }
 
@@ -1009,7 +1020,11 @@ impl<'a> Ctx<'a> {
     fn resolve_swap_defaults(&mut self) {
         for (owner, name, rest_id) in std::mem::take(&mut self.swap_defaults) {
             let Some(c) = self.component(&rest_id) else {
-                self.warn("instance swap properties whose default component isn't in the file were skipped");
+                if self.capped_mains.contains(&rest_id) {
+                    self.warn("instance swap properties whose default component was past the layer limit were skipped");
+                } else {
+                    self.warn("instance swap properties whose default component isn't in the file were skipped");
+                }
                 continue;
             };
             let def = PropDef::InstanceSwap { default: c, preferred: Vec::new() };
@@ -1028,7 +1043,11 @@ impl<'a> Ctx<'a> {
         let frames: BTreeSet<NodeId> = self.instances.iter().map(|(i, _)| *i).collect();
         let mut links: Vec<(NodeId, NodeId, &'a Value)> = Vec::new();
         for (inst, j) in std::mem::take(&mut self.instances) {
-            match text(j, "componentId").and_then(|r| self.component(r)) {
+            let main = text(j, "componentId");
+            match main.and_then(|r| self.component(r)) {
+                None if main.is_some_and(|r| self.capped_mains.contains(r)) => {
+                    self.warn("instances whose component was past the layer limit were imported as frames");
+                }
                 None => self.warn("instances of components that aren't in the file were imported as frames"),
                 Some(c) if self.doc.is_ancestor(c, inst) => self.warn("instances inside their own main component were imported as frames"),
                 Some(c) => links.push((inst, c, j)),

crates/ui-egui/locales/en.toml

@@ -1629,10 +1629,12 @@ name = "English"
 "image filters are ambiguous because both filter containers are present; filter conversion was skipped" = "image filters are ambiguous because both filter containers are present; filter conversion was skipped"
 "image filters use FigCraft's approximate renderer" = "image filters use FigCraft's approximate renderer"
 "instance swap properties whose default component isn't in the file were skipped" = "instance swap properties whose default component isn't in the file were skipped"
+"instance swap properties whose default component was past the layer limit were skipped" = "instance swap properties whose default component was past the layer limit were skipped"
 "instances inside their own main component were imported as frames" = "instances inside their own main component were imported as frames"
 "instances of components that aren't in the file show no layers" = "instances of components that aren't in the file show no layers"
 "instances of components that aren't in the file were imported as frames" = "instances of components that aren't in the file were imported as frames"
 "instances that would nest a component inside itself were imported as frames" = "instances that would nest a component inside itself were imported as frames"
+"instances whose component was past the layer limit were imported as frames" = "instances whose component was past the layer limit were imported as frames"
 "instances without a main component were imported as empty frames" = "instances without a main component were imported as empty frames"
 "invalid internal .fig text path data was ignored" = "invalid internal .fig text path data was ignored"
 "invalid or oversized vector networks import as outlines" = "invalid or oversized vector networks import as outlines"

crates/ui-egui/locales/es.toml

@@ -1630,10 +1630,12 @@ status = "draft — needs native review"
 "image filters are ambiguous because both filter containers are present; filter conversion was skipped" = "los filtros de imagen son ambiguos porque están los dos contenedores de filtros; se omitió la conversión de filtros"
 "image filters use FigCraft's approximate renderer" = "los filtros de imagen usan el renderizado aproximado de FigCraft"
 "instance swap properties whose default component isn't in the file were skipped" = "se omitieron las propiedades de intercambio de instancia cuyo componente predeterminado no está en el archivo"
+"instance swap properties whose default component was past the layer limit were skipped" = "se omitieron las propiedades de intercambio de instancia cuyo componente predeterminado quedó fuera del límite de capas"
 "instances inside their own main component were imported as frames" = "las instancias dentro de su propio componente principal se importaron como marcos"
 "instances of components that aren't in the file show no layers" = "las instancias de componentes que no están en el archivo no muestran capas"
 "instances of components that aren't in the file were imported as frames" = "las instancias de componentes que no están en el archivo se importaron como marcos"
 "instances that would nest a component inside itself were imported as frames" = "las instancias que anidarían un componente dentro de sí mismo se importaron como marcos"
+"instances whose component was past the layer limit were imported as frames" = "las instancias cuyo componente quedó fuera del límite de capas se importaron como marcos"
 "instances without a main component were imported as empty frames" = "las instancias sin componente principal se importaron como marcos vacíos"
 "invalid internal .fig text path data was ignored" = "se ignoraron datos internos no válidos de texto en trazado del .fig"
 "invalid or oversized vector networks import as outlines" = "las redes vectoriales no válidas o demasiado grandes se importan como contornos"