Merge branch 'main' into fix/sec-concat-bytes

crates/calc/src/eval/array.rs

@@ -86,6 +86,30 @@ pub fn zip2(a: &Value, b: &Value, f: impl Fn(&Value, &Value) -> Value) -> Value
     }
 }
 
+/// [`zip2`] that stops at the first cell `f` refuses (`None`), before the rest is built: the
+/// result grows as cells are made rather than being reserved for the whole broadcast up front.
+fn try_zip2(a: &Value, b: &Value, mut f: impl FnMut(&Value, &Value) -> Option<Value>) -> Option<Value> {
+    Some(match (a, b) {
+        (Value::Array(x), Value::Array(y)) => {
+            let rows = broadcast_dim(x.rows, y.rows);
+            let cols = broadcast_dim(x.cols, y.cols);
+            if broadcast_len(rows, cols).is_none() {
+                return Some(Value::Error(CellError::Num));
+            }
+            let mut data = Vec::new();
+            for r in 0..rows {
+                for c in 0..cols {
+                    data.push(f(&x.get_broadcast(r, c), &y.get_broadcast(r, c))?);
+                }
+            }
+            Value::from(Array { rows, cols, data })
+        }
+        (Value::Array(x), s) => Value::from(Array { rows: x.rows, cols: x.cols, data: x.data.iter().map(|v| f(v, s)).collect::<Option<_>>()? }),
+        (s, Value::Array(y)) => Value::from(Array { rows: y.rows, cols: y.cols, data: y.data.iter().map(|v| f(s, v)).collect::<Option<_>>()? }),
+        (x, y) => f(x, y)?,
+    })
+}
+
 /// Cell count of a `rows`×`cols` broadcast result, or `None` when it would exceed [`MAX_CELLS`].
 /// Each operand passes its own size cap, but their broadcast product can be far larger
 /// (`SEQUENCE(1048576)*SEQUENCE(1,16384)` is 17 billion cells), so it is checked before allocating.
@@ -135,10 +159,36 @@ pub(super) fn single(v: Value) -> Option<Value> {
     }
 }
 
+/// Text bytes that one `&` over arrays may produce in all. Each piece is capped on its own, but a
+/// compact formula can broadcast a long text across millions of cells
+/// (`SEQUENCE(4000,4000)&REPT("x",30000)` is 480 GB): past this the whole result is `#NUM!`, the
+/// error for a result too large to hold, before the rest is built.
+pub(super) const MAX_CONCAT_ARRAY_BYTES: usize = 512 * 1024 * 1024;
+
 pub fn binary_values(op: BinOp, x: &Value, y: &Value, sys: gridcraft_core::DateSystem) -> Value {
+    if op == BinOp::Concat && (matches!(x, Value::Array(_)) || matches!(y, Value::Array(_))) {
+        return concat_arrays(x, y, sys, MAX_CONCAT_ARRAY_BYTES);
+    }
     zip2(x, y, |a, b| binary_scalar(op, a, b, sys))
 }
 
+/// `x & y` with at least one array operand, giving `#NUM!` once the texts built so far pass `budget`
+/// bytes.
+fn concat_arrays(x: &Value, y: &Value, sys: gridcraft_core::DateSystem, budget: usize) -> Value {
+    let mut used = 0usize;
+    try_zip2(x, y, |a, b| {
+        let v = binary_scalar(BinOp::Concat, a, b, sys);
+        if let Value::Text(t) = &v {
+            used = used.saturating_add(t.len());
+            if used > budget {
+                return None;
+            }
+        }
+        Some(v)
+    })
+    .unwrap_or(Value::Error(CellError::Num))
+}
+
 fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSystem) -> Value {
     if let Value::Error(e) = a {
         return Value::Error(*e);
@@ -187,7 +237,35 @@ fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSyste
 
 #[cfg(test)]
 mod tests {
-    use super::{MAX_CELLS, broadcast_len};
+    use super::{Array, BinOp, CellError, MAX_CELLS, Value, broadcast_len, concat_arrays, try_zip2};
+
+    /// The broadcast stops at the first refused cell: nothing after it is visited.
+    #[test]
+    fn a_refused_cell_stops_the_broadcast() {
+        let col = Value::from(Array::column((0..4_000).map(|i| Value::number(i as f64)).collect()));
+        let row = Value::from(Array { rows: 1, cols: 4_000, data: (0..4_000).map(|i| Value::number(i as f64)).collect() });
+        let mut calls = 0;
+        let r = try_zip2(&col, &row, |a, _| {
+            calls += 1;
+            (calls <= 10).then(|| a.clone())
+        });
+        assert_eq!((r, calls), (None, 11));
+    }
+
+    #[test]
+    fn concatenating_arrays_stops_at_the_byte_budget() {
+        let sys = gridcraft_core::DateSystem::default();
+        let col = Value::from(Array::column((0..100).map(|i| Value::number(i as f64)).collect()));
+        let long = Value::text("x".repeat(100));
+        // 100 cells of 101 or 102 bytes each: about 10 KB in all.
+        let r = concat_arrays(&col, &long, sys, 20_000);
+        assert!(matches!(&r, Value::Array(a) if a.data.len() == 100), "{r:?}");
+        assert_eq!(concat_arrays(&col, &long, sys, 5_000), Value::Error(CellError::Num));
+        assert_eq!(concat_arrays(&long, &col, sys, 5_000), Value::Error(CellError::Num));
+        // Through the operator, a small result is unaffected.
+        let r = super::binary_values(BinOp::Concat, &col, &Value::text("a"), sys);
+        assert!(matches!(&r, Value::Array(a) if a.data.first() == Some(&Value::text("0a"))), "{r:?}");
+    }
 
     #[test]
     fn broadcast_len_caps_at_max_cells() {