Analysis ToolPak: stop building a report once it outgrows the output limit

Report builders filled their whole output map before write_report
checked its size, so Descriptive Statistics grouped by rows over a
tall, nearly empty column allocated about 16 entries for each of up
to a million variables before refusing the report.

Grid now tracks its extent as values are stored and keeps nothing
once the report's rows x columns pass MAX_OUTPUT_CELLS; write_report
refuses such a report. extent() no longer rescans every entry.

crates/engine/src/cmd/analysis.rs

@@ -272,11 +272,25 @@ struct Grid {
     cells: BTreeMap<(u32, u32), Value>,
     /// Cells shown as percentages.
     pct: Vec<(u32, u32)>,
+    /// Rows and columns spanned by `cells`.
+    rows: u32,
+    cols: u32,
+    /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused.
+    too_large: bool,
 }
 
 impl Grid {
+    /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept,
+    /// so a builder that would produce millions of entries stops allocating them, and
+    /// [`write_report`] refuses the report.
     fn put(&mut self, r: u32, c: u32, v: Value) {
         if !v.is_empty() {
+            let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1)));
+            if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+                self.too_large = true;
+                return;
+            }
+            (self.rows, self.cols) = (rows, cols);
             self.cells.insert((r, c), v);
         }
     }
@@ -296,9 +310,7 @@ impl Grid {
     }
     /// (rows, cols) spanned.
     fn extent(&self) -> (u32, u32) {
-        let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0);
-        let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0);
-        (rows, cols)
+        (self.rows, self.cols)
     }
 }
 
@@ -1749,7 +1761,7 @@ fn sampling(s: &Session, p: &Json, seed: Option<u64>) -> Result<Grid> {
 fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result<Json> {
     let (rows, cols) = g.extent();
     let (rows, cols) = (rows.max(1), cols.max(1));
-    if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+    if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
         return Err(bad(CMD, "the report is too large"));
     }
     let d = s.doc()?;
@@ -1914,6 +1926,28 @@ mod tests {
         x
     }
 
+    #[test]
+    fn a_report_stops_growing_past_the_output_limit() {
+        // Entries past the limit are not kept, and the report is refused.
+        let mut g = super::Grid::default();
+        g.num(1999, 1999, 1.0);
+        assert!(!g.too_large);
+        g.num(0, 2000, 1.0);
+        assert!(g.too_large);
+        assert_eq!(g.cells.len(), 1);
+        assert_eq!(g.extent(), (2000, 2000));
+        // Row-grouped statistics of a tall, almost empty column: one variable per row would make
+        // millions of report entries; the report is refused instead of built.
+        let mut s = s();
+        s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap();
+        s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap();
+        let err = s
+            .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"}))
+            .expect_err("report too large");
+        assert!(err.to_string().contains("too large"), "{err}");
+        assert_eq!(v(&s, "C1"), Value::Empty);
+    }
+
     #[test]
     fn forecast_standard_errors_match_the_full_window_rescan() {
         // #1043: the running sum must give what re-adding every window gave, gaps included.

crates/engine/src/cmd/extra/data.rs

@@ -42,15 +42,6 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result<Json> {
     })
 }
 
-/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely
-/// before they are written, so a few sparse records or one far-away CSV cell must not expand
-/// into a rectangle of billions of slots.
-const MAX_IMPORT_CELLS: u64 = 10_000_000;
-
-fn too_many_cells() -> EngineError {
-    EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most)."))
-}
-
 pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
     let bytes = if let Some(t) = str_param(p, "text") {
         t.as_bytes().to_vec()
@@ -66,22 +57,15 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
         if text.trim_start().starts_with('[') {
             let arr: Vec<Json> = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?;
             let mut keys: Vec<String> = Vec::new();
-            let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
             for o in &arr {
                 if let Some(m) = o.as_object() {
                     for k in m.keys() {
-                        if seen.insert(k.as_str()) {
+                        if !keys.contains(k) {
                             keys.push(k.clone());
                         }
                     }
                 }
             }
-            if keys.len() > gridcraft_core::MAX_COLS as usize {
-                return Err(EngineError::Other("The data has more fields than a sheet has columns.".into()));
-            }
-            if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS {
-                return Err(too_many_cells());
-            }
             let mut out = vec![keys.iter().map(|k| json!(k)).collect::<Vec<_>>()];
             for o in &arr {
                 out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect());
@@ -95,9 +79,6 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
             let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?;
             let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?;
             let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) };
-            if u.count() > MAX_IMPORT_CELLS {
-                return Err(too_many_cells());
-            }
             (u.start.row..=u.end.row)
                 .map(|r| {
                     (u.start.col..=u.end.col)
@@ -148,35 +129,3 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result<Json> {
             .collect(),
     ))
 }
-
-#[cfg(test)]
-mod tests {
-    use serde_json::json;
-
-    use crate::Session;
-
-    fn s() -> Session {
-        let mut s = Session::new();
-        s.new_workbook();
-        s
-    }
-
-    #[test]
-    fn sparse_imports_are_refused_before_their_rows_are_built() {
-        let mut s = s();
-        let sheets = s.doc().unwrap().wb.sheets.len();
-        // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field.
-        let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect();
-        let text = serde_json::to_string(&records).unwrap();
-        let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells");
-        assert!(err.to_string().contains("too large"), "{err}");
-        // A CSV whose last line reaches 16,001 columns, 1,000 lines down.
-        let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000));
-        let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells");
-        assert!(err.to_string().contains("too large"), "{err}");
-        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
-        // Small imports still work.
-        let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap();
-        assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}");
-    }
-}