fix(render): keep one outline depth budget through nested booleans (#193) (#231)

Boolean operands restarted subtree_outline at depth 0, so booleans nested
in booleans bypassed the 64-level limit and could overflow the stack.

Co-authored-by: Justin John <babyjjbabyjj549@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

crates/render/src/boolean_cache_tests.rs

@@ -133,3 +133,23 @@ fn one_sided_widths_paint_one_side() {
     let b = d.world_render_bounds(l).unwrap();
     assert!(b.y1 >= 120.0 - 1e-6, "{b:?}");
 }
+
+#[test]
+fn deep_boolean_nesting_stops_at_the_outline_depth_limit() {
+    // Booleans directly inside booleans used to restart the depth count at every level.
+    let mut d = Document::new("t");
+    let mut parent = d.first_page().unwrap();
+    let top = d.create(parent, NodeKind::BooleanOp { op: BoolOp::Union }, "B", 0.0, 0.0, 10.0, 10.0).unwrap();
+    parent = top;
+    for _ in 0..20_000 {
+        parent = d.create(parent, NodeKind::BooleanOp { op: BoolOp::Union }, "B", 0.0, 0.0, 10.0, 10.0).unwrap();
+    }
+    d.create(parent, NodeKind::Rectangle, "R", 0.0, 0.0, 10.0, 10.0).unwrap();
+    // Too deep: no stack overflow, and the unreachable square contributes nothing.
+    assert!(outline(&d, top).area().abs() < 1e-9);
+    // Within the limit, nested booleans still combine.
+    let (mut d, _, b, _) = row_of_squares(BoolOp::Union, 1);
+    let inner = d.create(b, NodeKind::BooleanOp { op: BoolOp::Union }, "B", 0.0, 0.0, 0.0, 0.0).unwrap();
+    d.create(inner, NodeKind::Rectangle, "R", 40.0, 10.0, 20.0, 20.0).unwrap();
+    assert!((outline(&d, b).area().abs() - 2.0 * 20.0 * 20.0).abs() < 0.5);
+}

crates/render/src/draw.rs

@@ -203,6 +203,11 @@ pub(crate) fn stroke_rule(n: &Node, closed: bool) -> FillRule {
 /// The geometry a node fills, in local space: shapes, booleans (computed from children) and text
 /// (glyph outlines). Groups and pages have none.
 pub fn node_outline(doc: &Document, n: &Node) -> Option<(BezPath, FillRule)> {
+    node_outline_at(doc, n, 0)
+}
+
+/// [`node_outline`] `depth` levels into a boolean or group nesting (one budget for both).
+fn node_outline_at(doc: &Document, n: &Node, depth: usize) -> Option<(BezPath, FillRule)> {
     match &n.kind {
         NodeKind::BooleanOp { op } => {
             let operands: Vec<(BezPath, figcraft_doc::BoolOp)> = n
@@ -211,7 +216,7 @@ pub fn node_outline(doc: &Document, n: &Node) -> Option<(BezPath, FillRule)> {
                 .filter_map(|c| doc.node(*c))
                 .filter(|c| c.visible)
                 .filter_map(|c| {
-                    let (mut p, _) = subtree_outline(doc, c, 0)?;
+                    let (mut p, _) = subtree_outline(doc, c, depth + 1)?;
                     p.apply_affine(c.transform);
                     Some((p, c.bool_op.unwrap_or(*op)))
                 })
@@ -258,7 +263,7 @@ pub fn subtree_outline(doc: &Document, n: &Node, depth: usize) -> Option<(BezPat
             }
             Some((out, FillRule::Winding))
         }
-        _ => node_outline(doc, n),
+        _ => node_outline_at(doc, n, depth),
     }
 }