Calc: a LAMBDA that calls itself without end answers #NUM! instead of overflowing the stack

The evaluator stopped at MAX_DEPTH (400) nested eval calls. A LAMBDA
call takes one of those levels but several stack frames (eval,
eval_inner, call, builtin, apply_lambda, call_lambda_values), so on the
1 MiB main thread of the Windows build
`=LET(f,LAMBDA(g,x,g(g,x)),f(f,1))` overflowed the stack before it
reached the limit, and the process ended.

eval now also stops once a formula has used 512 KiB of stack below its
first eval. Both limits answer #NUM!, the error Excel documents for too
many recursive calls, and a defined name that holds the recursion passes
that error on instead of answering #NAME?.

Fixes #1662

crates/calc/src/eval.rs

@@ -49,6 +49,18 @@ fn err(e: CellError) -> Ev {
     Ev::V(Value::Error(e))
 }
 
+/// Native stack, in bytes, that one formula's evaluation may use below its first `eval`. Past it,
+/// `eval` answers `#NUM!`. One level of `MAX_DEPTH` can take several stack frames. The Windows main
+/// thread and the web build have 1 MiB of stack.
+const STACK_BUDGET: usize = 512 * 1024;
+
+/// The address of a local variable, which shows how far down the native stack this call is.
+#[inline(always)]
+fn stack_position() -> usize {
+    let marker = 0u8;
+    std::hint::black_box(&marker) as *const u8 as usize
+}
+
 /// What the evaluator needs from its surroundings.
 pub trait Host {
     fn workbook(&self) -> &Workbook;
@@ -84,6 +96,8 @@ pub struct Evaluator<'h> {
     pub at: CellRef,
     env: Vec<(String, Ev)>,
     depth: usize,
+    /// `stack_position()` at the outermost `eval`.
+    stack_top: usize,
 }
 
 struct FnCtx<'a, 'h> {
@@ -121,14 +135,14 @@ impl Ctx for FnCtx<'_, '_> {
                 _ => None,
             })
             .collect();
-        let mut ev = Evaluator { host: &mut *self.ev.host, sheet: area.sheet, at, env, depth: self.ev.depth };
+        let mut ev = Evaluator { host: &mut *self.ev.host, sheet: area.sheet, at, env, depth: self.ev.depth, stack_top: self.ev.stack_top };
         Some(matches!(ev.value(&shifted).scalar().to_bool(), Ok(true)))
     }
 }
 
 impl<'h> Evaluator<'h> {
     pub fn new(host: &'h mut dyn Host, sheet: usize, at: CellRef) -> Self {
-        Evaluator { host, sheet, at, env: Vec::new(), depth: 0 }
+        Evaluator { host, sheet, at, env: Vec::new(), depth: 0, stack_top: 0 }
     }
 
     /// Evaluates to a value (references are dereferenced; multi-cell → array).
@@ -205,10 +219,16 @@ impl<'h> Evaluator<'h> {
     }
 
     pub fn eval(&mut self, e: &Expr) -> Ev {
+        let here = stack_position();
+        if self.depth == 0 {
+            self.stack_top = here;
+        } else if self.stack_top.abs_diff(here) > STACK_BUDGET {
+            return err(CellError::Num);
+        }
         self.depth += 1;
         if self.depth > MAX_DEPTH {
             self.depth -= 1;
-            return err(CellError::Calc);
+            return err(CellError::Num);
         }
         let r = self.eval_inner(e);
         self.depth -= 1;
@@ -492,7 +512,7 @@ impl<'h> Evaluator<'h> {
             }
             _ => {
                 // A long chain such as A1+A2+…+A500 nests to the left: fold it in a loop instead
-                // of one level of recursion per operator (past the depth limit it was #CALC!).
+                // of one level of recursion per operator (past the depth limit it was an error).
                 let mut rest = vec![(op, b)];
                 let mut left = a;
                 while let Expr::Binary(o, l, r) = left
@@ -801,10 +821,11 @@ impl<'h> Evaluator<'h> {
     fn builtin(&mut self, name: &str, args: &[Expr]) -> Ev {
         let Some(spec) = gridcraft_functions::lookup(name) else {
             // A defined name holding a LAMBDA, or a LET-bound lambda.
-            if let Ev::L(l) = self.name(name) {
-                return self.apply_lambda(&l, args);
-            }
-            return err(CellError::Name);
+            return match self.name(name) {
+                Ev::L(l) => self.apply_lambda(&l, args),
+                Ev::V(Value::Error(x)) => err(x),
+                _ => err(CellError::Name),
+            };
         };
         if args.len() < spec.min_args || args.len() > spec.max_args {
             return err(CellError::Value);

crates/calc/src/tests.rs

@@ -643,6 +643,38 @@ fn lambda_argument_count() {
     assert_eq!(t.get("A9"), Value::Bool(true));
 }
 
+#[test]
+fn a_lambda_that_calls_itself_without_end_is_num_on_a_small_stack() {
+    // 1 MiB is the main thread's stack on Windows, where commands, recalculation and file open run.
+    let handle = std::thread::Builder::new()
+        .stack_size(1 << 20)
+        .spawn(|| {
+            let mut t = T::new();
+            t.wb.names.push(gridcraft_model::DefinedName {
+                name: "LOOP".into(),
+                scope: None,
+                formula: "LAMBDA(n,LOOP(n))".into(),
+                comment: String::new(),
+                hidden: false,
+            });
+            t.calc.rebuild(&t.wb);
+            let endless = "=LET(f,LAMBDA(g,x,g(g,x)),f(f,1))";
+            t.set("A1", endless);
+            t.set("A2", "=LOOP(1)");
+            t.set("A3", "=LET(F,LAMBDA(me,n,IF(n<=0,0,1+SUM(MAP({1},LAMBDA(v,me(me,n-1)))))),F(F,100000))");
+            t.set("A4", "=LET(F,LAMBDA(me,n,IF(n<=0,0,1+me(me,n-1))),F(F,50))");
+            let evaluated = crate::evaluate(&t.wb, 0, c("B1"), endless);
+            [t.get("A1"), t.get("A2"), t.get("A3"), evaluated, t.get("A4")]
+        })
+        .expect("spawn");
+    let [endless, endless_name, through_map, evaluated, countdown] = handle.join().expect("recursion must not overflow the stack");
+    assert_eq!(endless, Value::Error(CellError::Num));
+    assert_eq!(endless_name, Value::Error(CellError::Num));
+    assert_eq!(through_map, Value::Error(CellError::Num));
+    assert_eq!(evaluated, Value::Error(CellError::Num));
+    assert_eq!(countdown, Value::Number(50.0));
+}
+
 #[test]
 fn names_and_sheets() {
     let mut t = T::new();