calc: cap the text one & over arrays may build in all

crates/calc/src/eval/array.rs

@@ -135,10 +135,41 @@ pub(super) fn single(v: Value) -> Option<Value> {
     }
 }
 
+/// Text bytes that one `&` over arrays may produce in all. Each piece is capped on its own, but a
+/// compact formula can broadcast a long text across millions of cells
+/// (`SEQUENCE(4000,4000)&REPT("x",30000)` is 480 GB): past this the whole result is `#NUM!`, the
+/// error for a result too large to hold, before the rest is built.
+pub(super) const MAX_CONCAT_ARRAY_BYTES: usize = 512 * 1024 * 1024;
+
 pub fn binary_values(op: BinOp, x: &Value, y: &Value, sys: gridcraft_core::DateSystem) -> Value {
+    if op == BinOp::Concat && (matches!(x, Value::Array(_)) || matches!(y, Value::Array(_))) {
+        return concat_arrays(x, y, sys, MAX_CONCAT_ARRAY_BYTES);
+    }
     zip2(x, y, |a, b| binary_scalar(op, a, b, sys))
 }
 
+/// `x & y` with at least one array operand, giving `#NUM!` once the texts built so far pass `budget`
+/// bytes.
+fn concat_arrays(x: &Value, y: &Value, sys: gridcraft_core::DateSystem, budget: usize) -> Value {
+    let used = std::cell::Cell::new(0usize);
+    let over = std::cell::Cell::new(false);
+    let r = zip2(x, y, |a, b| {
+        if over.get() {
+            return Value::Empty;
+        }
+        let v = binary_scalar(BinOp::Concat, a, b, sys);
+        if let Value::Text(t) = &v {
+            used.set(used.get().saturating_add(t.len()));
+            if used.get() > budget {
+                over.set(true);
+                return Value::Empty;
+            }
+        }
+        v
+    });
+    if over.get() { Value::Error(CellError::Num) } else { r }
+}
+
 fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSystem) -> Value {
     if let Value::Error(e) = a {
         return Value::Error(*e);
@@ -187,7 +218,22 @@ fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSyste
 
 #[cfg(test)]
 mod tests {
-    use super::{MAX_CELLS, broadcast_len};
+    use super::{Array, BinOp, CellError, MAX_CELLS, Value, broadcast_len, concat_arrays};
+
+    #[test]
+    fn concatenating_arrays_stops_at_the_byte_budget() {
+        let sys = gridcraft_core::DateSystem::default();
+        let col = Value::from(Array::column((0..100).map(|i| Value::number(i as f64)).collect()));
+        let long = Value::text("x".repeat(100));
+        // 100 cells of 101 or 102 bytes each: about 10 KB in all.
+        let r = concat_arrays(&col, &long, sys, 20_000);
+        assert!(matches!(&r, Value::Array(a) if a.data.len() == 100), "{r:?}");
+        assert_eq!(concat_arrays(&col, &long, sys, 5_000), Value::Error(CellError::Num));
+        assert_eq!(concat_arrays(&long, &col, sys, 5_000), Value::Error(CellError::Num));
+        // Through the operator, a small result is unaffected.
+        let r = super::binary_values(BinOp::Concat, &col, &Value::text("a"), sys);
+        assert!(matches!(&r, Value::Array(a) if a.data.first() == Some(&Value::text("0a"))), "{r:?}");
+    }
 
     #[test]
     fn broadcast_len_caps_at_max_cells() {

crates/engine/src/cmd/analysis.rs

@@ -272,25 +272,11 @@ struct Grid {
     cells: BTreeMap<(u32, u32), Value>,
     /// Cells shown as percentages.
     pct: Vec<(u32, u32)>,
-    /// Rows and columns spanned by `cells`.
-    rows: u32,
-    cols: u32,
-    /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused.
-    too_large: bool,
 }
 
 impl Grid {
-    /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept,
-    /// so a builder that would produce millions of entries stops allocating them, and
-    /// [`write_report`] refuses the report.
     fn put(&mut self, r: u32, c: u32, v: Value) {
         if !v.is_empty() {
-            let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1)));
-            if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
-                self.too_large = true;
-                return;
-            }
-            (self.rows, self.cols) = (rows, cols);
             self.cells.insert((r, c), v);
         }
     }
@@ -310,7 +296,9 @@ impl Grid {
     }
     /// (rows, cols) spanned.
     fn extent(&self) -> (u32, u32) {
-        (self.rows, self.cols)
+        let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0);
+        let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0);
+        (rows, cols)
     }
 }
 
@@ -1761,7 +1749,7 @@ fn sampling(s: &Session, p: &Json, seed: Option<u64>) -> Result<Grid> {
 fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result<Json> {
     let (rows, cols) = g.extent();
     let (rows, cols) = (rows.max(1), cols.max(1));
-    if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+    if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
         return Err(bad(CMD, "the report is too large"));
     }
     let d = s.doc()?;
@@ -1926,28 +1914,6 @@ mod tests {
         x
     }
 
-    #[test]
-    fn a_report_stops_growing_past_the_output_limit() {
-        // Entries past the limit are not kept, and the report is refused.
-        let mut g = super::Grid::default();
-        g.num(1999, 1999, 1.0);
-        assert!(!g.too_large);
-        g.num(0, 2000, 1.0);
-        assert!(g.too_large);
-        assert_eq!(g.cells.len(), 1);
-        assert_eq!(g.extent(), (2000, 2000));
-        // Row-grouped statistics of a tall, almost empty column: one variable per row would make
-        // millions of report entries; the report is refused instead of built.
-        let mut s = s();
-        s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap();
-        s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap();
-        let err = s
-            .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"}))
-            .expect_err("report too large");
-        assert!(err.to_string().contains("too large"), "{err}");
-        assert_eq!(v(&s, "C1"), Value::Empty);
-    }
-
     #[test]
     fn forecast_standard_errors_match_the_full_window_rescan() {
         // #1043: the running sum must give what re-adding every window gave, gaps included.