functions: TEXTJOIN builds its result piece by piece within the text cap

TEXTJOIN converted every input and delimiter into an owned String before
joining, so many references to one long shared text copied gigabytes before
the joined length was ever checked. Errors are now found first without
building text (so they still win in argument order), then each piece and
delimiter is converted only as it is appended, stopping at the cap.

crates/engine/src/cmd/analysis.rs

@@ -272,25 +272,11 @@ struct Grid {
     cells: BTreeMap<(u32, u32), Value>,
     /// Cells shown as percentages.
     pct: Vec<(u32, u32)>,
-    /// Rows and columns spanned by `cells`.
-    rows: u32,
-    cols: u32,
-    /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused.
-    too_large: bool,
 }
 
 impl Grid {
-    /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept,
-    /// so a builder that would produce millions of entries stops allocating them, and
-    /// [`write_report`] refuses the report.
     fn put(&mut self, r: u32, c: u32, v: Value) {
         if !v.is_empty() {
-            let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1)));
-            if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
-                self.too_large = true;
-                return;
-            }
-            (self.rows, self.cols) = (rows, cols);
             self.cells.insert((r, c), v);
         }
     }
@@ -310,7 +296,9 @@ impl Grid {
     }
     /// (rows, cols) spanned.
     fn extent(&self) -> (u32, u32) {
-        (self.rows, self.cols)
+        let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0);
+        let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0);
+        (rows, cols)
     }
 }
 
@@ -332,10 +320,6 @@ fn descriptive(s: &Session, p: &Json) -> Result<Grid> {
     }
     let mut g = Grid::default();
     for (i, v) in vars.iter().enumerate() {
-        // The report is refused once the grid is too large: the rest would only build discarded labels.
-        if g.too_large {
-            break;
-        }
         let x = numbers(&v.vals, true)?;
         let c = idx(i).saturating_mul(2);
         g.txt(0, c, &v.name);
@@ -1765,7 +1749,7 @@ fn sampling(s: &Session, p: &Json, seed: Option<u64>) -> Result<Grid> {
 fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result<Json> {
     let (rows, cols) = g.extent();
     let (rows, cols) = (rows.max(1), cols.max(1));
-    if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+    if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
         return Err(bad(CMD, "the report is too large"));
     }
     let d = s.doc()?;
@@ -1930,28 +1914,6 @@ mod tests {
         x
     }
 
-    #[test]
-    fn a_report_stops_growing_past_the_output_limit() {
-        // Entries past the limit are not kept, and the report is refused.
-        let mut g = super::Grid::default();
-        g.num(1999, 1999, 1.0);
-        assert!(!g.too_large);
-        g.num(0, 2000, 1.0);
-        assert!(g.too_large);
-        assert_eq!(g.cells.len(), 1);
-        assert_eq!(g.extent(), (2000, 2000));
-        // Row-grouped statistics of a tall, almost empty column: one variable per row would make
-        // millions of report entries; the report is refused instead of built.
-        let mut s = s();
-        s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap();
-        s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap();
-        let err = s
-            .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"}))
-            .expect_err("report too large");
-        assert!(err.to_string().contains("too large"), "{err}");
-        assert_eq!(v(&s, "C1"), Value::Empty);
-    }
-
     #[test]
     fn forecast_standard_errors_match_the_full_window_rescan() {
         // #1043: the running sum must give what re-adding every window gave, gaps included.

crates/functions/src/text.rs

@@ -53,25 +53,41 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     check_len(out)
 }
 
+/// The error `v.to_text()` would give, found without building any text.
+fn text_error(v: &Value) -> Option<CellError> {
+    match v {
+        Value::Error(e) => Some(*e),
+        Value::Array(a) => a.data.first().and_then(text_error),
+        _ => None,
+    }
+}
+
 fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let delims: Vec<String> = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::<R<_>>()?;
+    // Errors are found first, in argument order, so they win as before; text is then built one
+    // piece at a time into the capped output, so many references to one long text (or a long
+    // delimiter list) stop at the cap instead of first copying every piece.
+    let delims = as_array(&arg(a, 0)?.value);
+    if let Some(e) = delims.iter().find_map(text_error) {
+        return Err(e);
+    }
     let ignore = boolean(a, 1)?;
-    let mut items = Vec::new();
-    for x in a.iter().skip(2) {
-        for v in as_array(&x.value).iter() {
-            let s = v.to_text()?;
-            if ignore && s.is_empty() {
-                continue;
-            }
-            items.push(s);
-        }
+    let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect();
+    if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) {
+        return Err(e);
     }
     let mut out = String::new();
-    for (i, s) in items.iter().enumerate() {
-        if i > 0 && !delims.is_empty() {
-            push_capped(&mut out, &delims[(i - 1) % delims.len()])?;
+    let mut joined = 0usize;
+    for v in items.iter().flat_map(|x| x.iter()) {
+        let s = v.to_text()?;
+        if ignore && s.is_empty() {
+            continue;
         }
-        push_capped(&mut out, s)?;
+        if joined > 0 && !delims.data.is_empty() {
+            let d = delims.data.get((joined - 1) % delims.data.len()).map(Value::to_text).transpose()?.unwrap_or_default();
+            push_capped(&mut out, &d)?;
+        }
+        push_capped(&mut out, &s)?;
+        joined += 1;
     }
     check_len(out)
 }
@@ -1131,6 +1147,22 @@ mod tests {
         assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]);
     }
 
+    /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather
+    /// than first copying every input (100,000 references to one 32,767-character text were
+    /// 3 GB of copies). Errors among the inputs still win, in argument order.
+    #[test]
+    fn textjoin_stops_at_the_text_cap() {
+        use gridcraft_core::Array;
+        let long = tv(&"x".repeat(32_767));
+        let many = || av(Array::filled(100_000, 1, long.clone()).into());
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0);
+        is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA);
+        // Delimiters cycle and are converted only where used.
+        is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d");
+    }
+
     fn row_t(xs: &[&str]) -> Value {
         arr(vec![xs.iter().map(|x| tv(x)).collect()])
     }