Calc: a formula calculated inside another one shares its stack budget

A cell that another cell's formula calculates on demand got its own
evaluator, which started a new 896 KiB stack budget wherever it was
called. Two cells that each count down from 131 with a LAMBDA, one read
at the deepest call of the other, overflowed a 1 MiB thread.

The budget of a formula calculated on demand is now counted from where
the outermost formula started. When it runs out there, the formula is
calculated again from the top, as a formula needed at the nesting limit
is, so its value does not depend on the order cells are calculated in.

crates/calc/src/eval.rs

@@ -60,13 +60,13 @@ const MAX_LAMBDA_CALLS: usize = 140;
 
 /// Native stack, in bytes, that one formula's evaluation may use below its first `eval`. Past it,
 /// `eval` answers `#NUM!`. It guards the stack and leaves 128 KiB of a 1 MiB thread for the frames
-/// above the formula. A cell that another cell's formula computes on demand gets its own `Evaluator`,
-/// which starts its own budget.
+/// above the formula. A cell that another cell's formula computes on demand shares the budget of the
+/// outermost formula ([`Host::stack_top`]).
 const STACK_BUDGET: usize = 896 * 1024;
 
 /// The address of a local variable, which shows how far down the native stack this call is.
 #[inline(always)]
-fn stack_position() -> usize {
+pub(crate) fn stack_position() -> usize {
     let marker = 0u8;
     std::hint::black_box(&marker) as *const u8 as usize
 }
@@ -98,6 +98,13 @@ pub trait Host {
     fn visit_range(&mut self, _sheet: usize, _range: RangeRef, _f: &mut dyn FnMut(&Value) -> bool) -> bool {
         false
     }
+    /// While a formula is calculated on demand inside another one: `stack_position()` where the
+    /// outermost one started, which the stack budget is counted from.
+    fn stack_top(&self) -> Option<usize> {
+        None
+    }
+    /// Called when a formula's evaluation runs past the stack budget.
+    fn stack_exhausted(&mut self) {}
 }
 
 pub struct Evaluator<'h> {
@@ -265,8 +272,10 @@ impl<'h> Evaluator<'h> {
     pub fn eval(&mut self, e: &Expr) -> Ev {
         let here = stack_position();
         if self.depth == 0 {
-            self.stack_top = here;
-        } else if self.stack_top.abs_diff(here) > STACK_BUDGET {
+            self.stack_top = self.host.stack_top().unwrap_or(here);
+        }
+        if self.stack_top.abs_diff(here) > STACK_BUDGET {
+            self.host.stack_exhausted();
             return err(CellError::Num);
         }
         self.depth += 1;

crates/calc/src/recalc.rs

@@ -44,7 +44,7 @@ use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value};
 use gridcraft_formula::Expr;
 use gridcraft_model::{CalcMode, Workbook};
 
-use crate::eval::{Area, Evaluator, Host, precedents};
+use crate::eval::{Area, Evaluator, Host, precedents, stack_position};
 
 pub type Key = (usize, CellRef);
 
@@ -432,6 +432,10 @@ struct PassHost<'a> {
     tainted: HashSet<Key>,
     /// Whether the formula being evaluated has read a previous value.
     taint: bool,
+    /// `stack_position()` where the outermost formula being calculated started.
+    stack_top: usize,
+    /// Whether an evaluation ran past the stack budget since the innermost formula started.
+    out_of_stack: bool,
 }
 
 impl<'a> PassHost<'a> {
@@ -462,6 +466,8 @@ impl<'a> PassHost<'a> {
             members: HashSet::default(),
             tainted: HashSet::default(),
             taint: false,
+            stack_top: 0,
+            out_of_stack: false,
         }
     }
 }
@@ -580,6 +586,9 @@ impl PassHost<'_> {
             self.suspend = Some(k);
             return Value::Error(CellError::Calc);
         }
+        if self.depth == 0 {
+            self.stack_top = stack_position();
+        }
         self.in_progress.insert(k);
         self.depth += 1;
         let outer_taint = std::mem::replace(&mut self.taint, false);
@@ -587,10 +596,24 @@ impl PassHost<'_> {
             self.stack.push(k);
         }
         let expr = std::sync::Arc::clone(expr);
+        let outer_out_of_stack = std::mem::replace(&mut self.out_of_stack, false);
         let v = {
             let mut ev = Evaluator::new(self, k.0, k.1);
             ev.value(&expr)
         };
+        let out_of_stack = std::mem::replace(&mut self.out_of_stack, outer_out_of_stack);
+        if out_of_stack && self.depth > 1 && self.suspend.is_none() {
+            // It ran out of the stack left below the formulas that read it: it is calculated
+            // from the top instead, as a formula needed at `MAX_NESTED` levels deep is.
+            self.suspend = Some(k);
+            self.depth -= 1;
+            self.in_progress.remove(&k);
+            if self.iterate {
+                self.stack.pop();
+            }
+            self.taint = outer_taint;
+            return Value::Error(CellError::Calc);
+        }
         if self.suspend.is_some() {
             // Abandoned: `calculate` evaluates this formula again once the one it needs is done.
             self.depth -= 1;
@@ -745,6 +768,12 @@ impl Host for PassHost<'_> {
     fn workbook(&self) -> &Workbook {
         self.wb
     }
+    fn stack_top(&self) -> Option<usize> {
+        (self.depth > 1).then_some(self.stack_top)
+    }
+    fn stack_exhausted(&mut self) {
+        self.out_of_stack = true;
+    }
     fn spill_range(&mut self, sheet: usize, anchor: CellRef) -> Option<RangeRef> {
         let k = (sheet, anchor);
         if !(self.pending.contains(&k) || self.results.contains_key(&k)) {

crates/calc/src/tests.rs

@@ -834,6 +834,40 @@ fn a_recursive_lambda_stops_at_the_same_call_on_any_stack_size() {
     assert_eq!(run(64 << 20), small);
 }
 
+#[test]
+fn a_recursive_lambda_that_reads_another_cells_recursive_lambda_shares_the_stack() {
+    let run = |stack: usize| {
+        std::thread::Builder::new()
+            .stack_size(stack)
+            .spawn(|| {
+                // Row-major order calculates A1 first, so each cell below is calculated on demand
+                // at the deepest call of the one above it.
+                let countdown = |then: &str| format!("=LET(F,LAMBDA(me,n,IF(n<=0,{then},1+me(me,n-1))),F(F,131))");
+                let mut wb = Workbook::new();
+                let sheet = wb.sheet_mut(0).expect("sheet");
+                for (at, f) in [
+                    ("A1", countdown("A2")),
+                    ("A2", countdown("A3")),
+                    ("A3", countdown("0")),
+                    ("B1", countdown("B2")),
+                    ("B2", "=LET(f,LAMBDA(g,x,g(g,x)),f(f,1))".to_string()),
+                ] {
+                    sheet.cells.set(c(at), Cell::formula(Formula::new(&f)));
+                }
+                let mut calc = Calc::new();
+                calc.recalc_all(&mut wb);
+                ["A1", "A2", "A3", "B1", "B2"].map(|at| wb.sheet(0).map(|s| s.value(c(at))).unwrap_or_default())
+            })
+            .expect("spawn")
+            .join()
+            .expect("recursion must not overflow the stack")
+    };
+    let small = run(1 << 20);
+    let num = Value::Error(CellError::Num);
+    assert_eq!(small, [Value::Number(393.0), Value::Number(262.0), Value::Number(131.0), num.clone(), num]);
+    assert_eq!(run(64 << 20), small);
+}
+
 #[test]
 fn names_and_sheets() {
     let mut t = T::new();