functions: TEXTJOIN builds its result piece by piece within the text cap

TEXTJOIN converted every input and delimiter into an owned String before
joining, so many references to one long shared text copied gigabytes before
the joined length was ever checked. Errors are now found first without
building text (so they still win in argument order), then each piece and
delimiter is converted only as it is appended, stopping at the cap.

crates/engine/src/cmd/extra/data.rs

@@ -42,15 +42,6 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result<Json> {
     })
 }
 
-/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely
-/// before they are written, so a few sparse records or one far-away CSV cell must not expand
-/// into a rectangle of billions of slots.
-const MAX_IMPORT_CELLS: u64 = 10_000_000;
-
-fn too_many_cells() -> EngineError {
-    EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most)."))
-}
-
 pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
     let bytes = if let Some(t) = str_param(p, "text") {
         t.as_bytes().to_vec()
@@ -66,22 +57,15 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
         if text.trim_start().starts_with('[') {
             let arr: Vec<Json> = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?;
             let mut keys: Vec<String> = Vec::new();
-            let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
             for o in &arr {
                 if let Some(m) = o.as_object() {
                     for k in m.keys() {
-                        if seen.insert(k.as_str()) {
+                        if !keys.contains(k) {
                             keys.push(k.clone());
                         }
                     }
                 }
             }
-            if keys.len() > gridcraft_core::MAX_COLS as usize {
-                return Err(EngineError::Other("The data has more fields than a sheet has columns.".into()));
-            }
-            if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS {
-                return Err(too_many_cells());
-            }
             let mut out = vec![keys.iter().map(|k| json!(k)).collect::<Vec<_>>()];
             for o in &arr {
                 out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect());
@@ -95,9 +79,6 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
             let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?;
             let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?;
             let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) };
-            if u.count() > MAX_IMPORT_CELLS {
-                return Err(too_many_cells());
-            }
             (u.start.row..=u.end.row)
                 .map(|r| {
                     (u.start.col..=u.end.col)
@@ -148,35 +129,3 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result<Json> {
             .collect(),
     ))
 }
-
-#[cfg(test)]
-mod tests {
-    use serde_json::json;
-
-    use crate::Session;
-
-    fn s() -> Session {
-        let mut s = Session::new();
-        s.new_workbook();
-        s
-    }
-
-    #[test]
-    fn sparse_imports_are_refused_before_their_rows_are_built() {
-        let mut s = s();
-        let sheets = s.doc().unwrap().wb.sheets.len();
-        // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field.
-        let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect();
-        let text = serde_json::to_string(&records).unwrap();
-        let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells");
-        assert!(err.to_string().contains("too large"), "{err}");
-        // A CSV whose last line reaches 16,001 columns, 1,000 lines down.
-        let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000));
-        let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells");
-        assert!(err.to_string().contains("too large"), "{err}");
-        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
-        // Small imports still work.
-        let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap();
-        assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}");
-    }
-}

crates/functions/src/text.rs

@@ -53,25 +53,41 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     check_len(out)
 }
 
+/// The error `v.to_text()` would give, found without building any text.
+fn text_error(v: &Value) -> Option<CellError> {
+    match v {
+        Value::Error(e) => Some(*e),
+        Value::Array(a) => a.data.first().and_then(text_error),
+        _ => None,
+    }
+}
+
 fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let delims: Vec<String> = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::<R<_>>()?;
+    // Errors are found first, in argument order, so they win as before; text is then built one
+    // piece at a time into the capped output, so many references to one long text (or a long
+    // delimiter list) stop at the cap instead of first copying every piece.
+    let delims = as_array(&arg(a, 0)?.value);
+    if let Some(e) = delims.iter().find_map(text_error) {
+        return Err(e);
+    }
     let ignore = boolean(a, 1)?;
-    let mut items = Vec::new();
+    let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect();
-    for x in a.iter().skip(2) {
+    if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) {
-        for v in as_array(&x.value).iter() {
+        return Err(e);
-            let s = v.to_text()?;
-            if ignore && s.is_empty() {
-                continue;
-            }
-            items.push(s);
-        }
     }
     let mut out = String::new();
-    for (i, s) in items.iter().enumerate() {
+    let mut joined = 0usize;
-        if i > 0 && !delims.is_empty() {
+    for v in items.iter().flat_map(|x| x.iter()) {
-            push_capped(&mut out, &delims[(i - 1) % delims.len()])?;
+        let s = v.to_text()?;
+        if ignore && s.is_empty() {
+            continue;
         }
-        push_capped(&mut out, s)?;
+        if joined > 0 && !delims.data.is_empty() {
+            let d = delims.data.get((joined - 1) % delims.data.len()).map(Value::to_text).transpose()?.unwrap_or_default();
+            push_capped(&mut out, &d)?;
+        }
+        push_capped(&mut out, &s)?;
+        joined += 1;
     }
     check_len(out)
 }
@@ -1131,6 +1147,22 @@ mod tests {
         assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]);
     }
 
+    /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather
+    /// than first copying every input (100,000 references to one 32,767-character text were
+    /// 3 GB of copies). Errors among the inputs still win, in argument order.
+    #[test]
+    fn textjoin_stops_at_the_text_cap() {
+        use gridcraft_core::Array;
+        let long = tv(&"x".repeat(32_767));
+        let many = || av(Array::filled(100_000, 1, long.clone()).into());
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0);
+        is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA);
+        // Delimiters cycle and are converted only where used.
+        is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d");
+    }
+
     fn row_t(xs: &[&str]) -> Value {
         arr(vec![xs.iter().map(|x| tv(x)).collect()])
     }