Get Data: refuse an import whose dense rows would be too large

data.getData and data.fromTextCsv built every row densely before
writing it: JSON records were padded to the union of all keys (found
with a quadratic Vec::contains scan), and CSV cells were expanded over
the whole used rectangle. A few sparse records or one far-away CSV
cell therefore allocated billions of slots.

The importer now collects JSON keys with a HashSet, refuses more keys
than a sheet has columns, and checks the projected rows x columns
against MAX_IMPORT_CELLS (10,000,000) before building any row.

crates/engine/src/cmd/extra/data.rs

@@ -42,6 +42,15 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result<Json> {
     })
 }
 
+/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely
+/// before they are written, so a few sparse records or one far-away CSV cell must not expand
+/// into a rectangle of billions of slots.
+const MAX_IMPORT_CELLS: u64 = 10_000_000;
+
+fn too_many_cells() -> EngineError {
+    EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most)."))
+}
+
 pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
     let bytes = if let Some(t) = str_param(p, "text") {
         t.as_bytes().to_vec()
@@ -57,15 +66,22 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
         if text.trim_start().starts_with('[') {
             let arr: Vec<Json> = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?;
             let mut keys: Vec<String> = Vec::new();
+            let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
             for o in &arr {
                 if let Some(m) = o.as_object() {
                     for k in m.keys() {
-                        if !keys.contains(k) {
+                        if seen.insert(k.as_str()) {
                             keys.push(k.clone());
                         }
                     }
                 }
             }
+            if keys.len() > gridcraft_core::MAX_COLS as usize {
+                return Err(EngineError::Other("The data has more fields than a sheet has columns.".into()));
+            }
+            if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS {
+                return Err(too_many_cells());
+            }
             let mut out = vec![keys.iter().map(|k| json!(k)).collect::<Vec<_>>()];
             for o in &arr {
                 out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect());
@@ -79,6 +95,9 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
             let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?;
             let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?;
             let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) };
+            if u.count() > MAX_IMPORT_CELLS {
+                return Err(too_many_cells());
+            }
             (u.start.row..=u.end.row)
                 .map(|r| {
                     (u.start.col..=u.end.col)
@@ -129,3 +148,35 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result<Json> {
             .collect(),
     ))
 }
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+
+    use crate::Session;
+
+    fn s() -> Session {
+        let mut s = Session::new();
+        s.new_workbook();
+        s
+    }
+
+    #[test]
+    fn sparse_imports_are_refused_before_their_rows_are_built() {
+        let mut s = s();
+        let sheets = s.doc().unwrap().wb.sheets.len();
+        // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field.
+        let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect();
+        let text = serde_json::to_string(&records).unwrap();
+        let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells");
+        assert!(err.to_string().contains("too large"), "{err}");
+        // A CSV whose last line reaches 16,001 columns, 1,000 lines down.
+        let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000));
+        let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells");
+        assert!(err.to_string().contains("too large"), "{err}");
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
+        // Small imports still work.
+        let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap();
+        assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}");
+    }
+}

crates/functions/src/text.rs

@@ -53,41 +53,25 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     check_len(out)
 }
 
-/// The error `v.to_text()` would give, found without building any text.
-fn text_error(v: &Value) -> Option<CellError> {
-    match v {
-        Value::Error(e) => Some(*e),
-        Value::Array(a) => a.data.first().and_then(text_error),
-        _ => None,
-    }
-}
-
 fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    // Errors are found first, in argument order, so they win as before; text is then built one
-    // piece at a time into the capped output, so many references to one long text (or a long
-    // delimiter list) stop at the cap instead of first copying every piece.
-    let delims = as_array(&arg(a, 0)?.value);
-    if let Some(e) = delims.iter().find_map(text_error) {
-        return Err(e);
-    }
+    let delims: Vec<String> = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::<R<_>>()?;
     let ignore = boolean(a, 1)?;
-    let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect();
-    if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) {
-        return Err(e);
+    let mut items = Vec::new();
+    for x in a.iter().skip(2) {
+        for v in as_array(&x.value).iter() {
+            let s = v.to_text()?;
+            if ignore && s.is_empty() {
+                continue;
+            }
+            items.push(s);
+        }
     }
     let mut out = String::new();
-    let mut joined = 0usize;
-    for v in items.iter().flat_map(|x| x.iter()) {
-        let s = v.to_text()?;
-        if ignore && s.is_empty() {
-            continue;
+    for (i, s) in items.iter().enumerate() {
+        if i > 0 && !delims.is_empty() {
+            push_capped(&mut out, &delims[(i - 1) % delims.len()])?;
         }
-        if joined > 0 && !delims.data.is_empty() {
-            let d = delims.data.get((joined - 1) % delims.data.len()).map(Value::to_text).transpose()?.unwrap_or_default();
-            push_capped(&mut out, &d)?;
-        }
-        push_capped(&mut out, &s)?;
-        joined += 1;
+        push_capped(&mut out, s)?;
     }
     check_len(out)
 }
@@ -1147,22 +1131,6 @@ mod tests {
         assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]);
     }
 
-    /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather
-    /// than first copying every input (100,000 references to one 32,767-character text were
-    /// 3 GB of copies). Errors among the inputs still win, in argument order.
-    #[test]
-    fn textjoin_stops_at_the_text_cap() {
-        use gridcraft_core::Array;
-        let long = tv(&"x".repeat(32_767));
-        let many = || av(Array::filled(100_000, 1, long.clone()).into());
-        is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value);
-        is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value);
-        is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0);
-        is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA);
-        // Delimiters cycle and are converted only where used.
-        is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d");
-    }
-
     fn row_t(xs: &[&str]) -> Value {
         arr(vec![xs.iter().map(|x| tv(x)).collect()])
     }