Merge #330: M1: deeply nested layers don't run the layout out of stack (#210) (@Barzabel)

crates/layout/src/depth_tests.rs

@@ -0,0 +1,95 @@
+//! Deeply nested layers and a resize (#210): constraints, group scaling and auto layout fill
+//! follow a resize down to [`MAX_NESTING`] levels and stop there, instead of running out of stack.
+//! Each case runs on a 1 MB stack, the main thread's on Windows; thousands of levels overflowed it.
+
+use super::*;
+use figcraft_doc::{AutoLayout, Constraints, Document, HConstraint, NodeKind, VConstraint};
+
+/// Far deeper than the nesting the layout follows.
+const DEEP: usize = 5000;
+
+fn on_small_stack(f: impl FnOnce() + Send + 'static) {
+    std::thread::Builder::new().stack_size(1 << 20).spawn(f).unwrap().join().unwrap();
+}
+
+fn width(d: &Document, id: NodeId) -> f64 {
+    d.node(id).unwrap().width
+}
+
+/// A 100 × 100 frame holding `depth` frames, each inside the last and stretching with it; the
+/// root and the chain from the top.
+fn stretching_frames(depth: usize) -> (Document, NodeId, Vec<NodeId>) {
+    let mut d = Document::new("deep");
+    let page = d.first_page().unwrap();
+    let root = d.create(page, NodeKind::Frame, "F", 0.0, 0.0, 100.0, 100.0).unwrap();
+    let mut chain = Vec::new();
+    let mut parent = root;
+    for _ in 0..depth {
+        let f = d.create(parent, NodeKind::Frame, "F", 0.0, 0.0, 100.0, 100.0).unwrap();
+        d.get_mut(f).unwrap().constraints = Constraints { horizontal: HConstraint::LeftRight, vertical: VConstraint::TopBottom };
+        chain.push(f);
+        parent = f;
+    }
+    (d, root, chain)
+}
+
+#[test]
+fn deeply_nested_frames_follow_their_constraints_without_running_out_of_stack() {
+    on_small_stack(|| {
+        let (mut d, root, chain) = stretching_frames(DEEP);
+        set_size(&mut d, root, 200.0, 200.0);
+        apply_constraints(&mut d, root, 100.0, 100.0);
+        assert_eq!(width(&d, chain[0]), 200.0);
+        assert_eq!(width(&d, chain[MAX_NESTING - 2]), 200.0);
+        assert_eq!(width(&d, chain[DEEP - 1]), 100.0, "past the nesting the layout follows, layers stay");
+    });
+}
+
+#[test]
+fn a_shallow_chain_follows_all_the_way_down() {
+    let (mut d, root, chain) = stretching_frames(20);
+    set_size(&mut d, root, 200.0, 200.0);
+    apply_constraints(&mut d, root, 100.0, 100.0);
+    assert!(chain.iter().all(|f| width(&d, *f) == 200.0));
+}
+
+#[test]
+fn deeply_nested_groups_scale_without_running_out_of_stack() {
+    on_small_stack(|| {
+        let mut d = Document::new("deep");
+        let mut parent = d.first_page().unwrap();
+        let root = d.create(parent, NodeKind::Group, "G", 0.0, 0.0, 100.0, 100.0).unwrap();
+        parent = root;
+        let mut chain = Vec::new();
+        for _ in 0..DEEP {
+            parent = d.create(parent, NodeKind::Group, "G", 0.0, 0.0, 100.0, 100.0).unwrap();
+            chain.push(parent);
+        }
+        scale_group_children(&mut d, root, 2.0, 2.0);
+        assert_eq!(width(&d, chain[0]), 200.0);
+    });
+}
+
+#[test]
+fn deeply_nested_fill_frames_lay_out_without_running_out_of_stack() {
+    on_small_stack(|| {
+        let mut d = Document::new("deep");
+        let page = d.first_page().unwrap();
+        let root = d.create(page, NodeKind::Frame, "F", 0.0, 0.0, 100.0, 100.0).unwrap();
+        d.get_mut(root).unwrap().layout = Some(AutoLayout { padding: [0.0; 4], ..AutoLayout::default() });
+        let mut parent = root;
+        let mut chain = Vec::new();
+        for _ in 0..DEEP {
+            let f = d.create(parent, NodeKind::Frame, "F", 0.0, 0.0, 10.0, 10.0).unwrap();
+            let n = d.get_mut(f).unwrap();
+            n.layout = Some(AutoLayout { padding: [0.0; 4], ..AutoLayout::default() });
+            n.child_layout.sizing_h = Sizing::Fill;
+            chain.push(f);
+            parent = f;
+        }
+        d.get_mut(root).unwrap().width = 300.0;
+        layout_frame(&mut d, root, &NoMeasure);
+        assert_eq!(width(&d, chain[0]), 300.0);
+        assert_eq!(width(&d, chain[DEEP - 1]), 10.0, "past the nesting the layout follows, layers stay");
+    });
+}

crates/layout/src/lib.rs

@@ -20,6 +20,30 @@ use figcraft_doc::{
 };
 use figcraft_geom::{Affine, Point, Rect};
 
+/// Most layout calls nested in one another (about one per level of layers): deeper layers don't
+/// follow a resize, so a deeply nested document can't run the stack out (#210).
+pub const MAX_NESTING: usize = 256;
+
+thread_local! {
+    static NESTING: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+}
+
+/// One level of nested layout calls, released when dropped.
+struct Nested;
+
+impl Nested {
+    /// Enter one more level; `None` when [`MAX_NESTING`] are in progress.
+    fn enter() -> Option<Self> {
+        NESTING.with(|n| (n.get() < MAX_NESTING).then(|| n.set(n.get() + 1)).map(|()| Self))
+    }
+}
+
+impl Drop for Nested {
+    fn drop(&mut self) {
+        NESTING.with(|n| n.set(n.get().saturating_sub(1)));
+    }
+}
+
 /// Measures text for a given width (auto-height text inside a Fill container).
 pub trait Measure {
     /// The height a text node needs at `width`.
@@ -311,6 +335,7 @@ struct Item {
 
 /// Lay out one auto layout frame's children and size the frame if it hugs.
 pub fn layout_frame(doc: &mut Document, frame: NodeId, m: &dyn Measure) {
+    let Some(_nested) = Nested::enter() else { return };
     let Some(f) = doc.node(frame) else { return };
     let Some(al) = f.layout.clone() else { return };
     let horizontal = matches!(al.mode, LayoutMode::Horizontal);
@@ -893,6 +918,7 @@ fn layout_grid(doc: &mut Document, frame: NodeId, al: &AutoLayout, m: &dyn Measu
 /// (recursively for children frames that resize). In an auto layout frame only the children that
 /// ignore auto layout (absolute position) follow constraints; the layout places the others.
 pub fn apply_constraints(doc: &mut Document, id: NodeId, ow: f64, oh: f64) {
+    let Some(_nested) = Nested::enter() else { return };
     let Some(f) = doc.node(id) else { return };
     let absolute_only = f.layout.is_some();
     let (nw, nh) = (f.width, f.height);
@@ -977,6 +1003,7 @@ pub fn apply_constraints(doc: &mut Document, id: NodeId, ow: f64, oh: f64) {
 /// Each child keeps its rotation and flip and scales along its own axes ([`axis_factors`]), its
 /// centre following the group's scale.
 pub fn scale_group_children(doc: &mut Document, id: NodeId, kx: f64, ky: f64) {
+    let Some(_nested) = Nested::enter() else { return };
     let Some(n) = doc.node(id) else { return };
     for c in n.children.clone() {
         let Some(cn) = doc.node(c) else { continue };
@@ -1101,6 +1128,8 @@ mod behaviour_tests;
 #[cfg(test)]
 mod content_align_tests;
 #[cfg(test)]
+mod depth_tests;
+#[cfg(test)]
 mod nested_tests;
 #[cfg(test)]
 mod tests;