Analysis ToolPak: stop building a report once it outgrows the output limit

Report builders filled their whole output map before write_report
checked its size, so Descriptive Statistics grouped by rows over a
tall, nearly empty column allocated about 16 entries for each of up
to a million variables before refusing the report.

Grid now tracks its extent as values are stored and keeps nothing
once the report's rows x columns pass MAX_OUTPUT_CELLS; write_report
refuses such a report. extent() no longer rescans every entry.

crates/engine/src/cmd/analysis.rs

@@ -272,11 +272,25 @@ struct Grid {
     cells: BTreeMap<(u32, u32), Value>,
     /// Cells shown as percentages.
     pct: Vec<(u32, u32)>,
+    /// Rows and columns spanned by `cells`.
+    rows: u32,
+    cols: u32,
+    /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused.
+    too_large: bool,
 }
 
 impl Grid {
+    /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept,
+    /// so a builder that would produce millions of entries stops allocating them, and
+    /// [`write_report`] refuses the report.
     fn put(&mut self, r: u32, c: u32, v: Value) {
         if !v.is_empty() {
+            let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1)));
+            if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+                self.too_large = true;
+                return;
+            }
+            (self.rows, self.cols) = (rows, cols);
             self.cells.insert((r, c), v);
         }
     }
@@ -296,9 +310,7 @@ impl Grid {
     }
     /// (rows, cols) spanned.
     fn extent(&self) -> (u32, u32) {
-        let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0);
-        let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0);
-        (rows, cols)
+        (self.rows, self.cols)
     }
 }
 
@@ -1749,7 +1761,7 @@ fn sampling(s: &Session, p: &Json, seed: Option<u64>) -> Result<Grid> {
 fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result<Json> {
     let (rows, cols) = g.extent();
     let (rows, cols) = (rows.max(1), cols.max(1));
-    if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
+    if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS {
         return Err(bad(CMD, "the report is too large"));
     }
     let d = s.doc()?;
@@ -1914,6 +1926,28 @@ mod tests {
         x
     }
 
+    #[test]
+    fn a_report_stops_growing_past_the_output_limit() {
+        // Entries past the limit are not kept, and the report is refused.
+        let mut g = super::Grid::default();
+        g.num(1999, 1999, 1.0);
+        assert!(!g.too_large);
+        g.num(0, 2000, 1.0);
+        assert!(g.too_large);
+        assert_eq!(g.cells.len(), 1);
+        assert_eq!(g.extent(), (2000, 2000));
+        // Row-grouped statistics of a tall, almost empty column: one variable per row would make
+        // millions of report entries; the report is refused instead of built.
+        let mut s = s();
+        s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap();
+        s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap();
+        let err = s
+            .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"}))
+            .expect_err("report too large");
+        assert!(err.to_string().contains("too large"), "{err}");
+        assert_eq!(v(&s, "C1"), Value::Empty);
+    }
+
     #[test]
     fn forecast_standard_errors_match_the_full_window_rescan() {
         // #1043: the running sum must give what re-adding every window gave, gaps included.

crates/functions/src/stat/descriptive.rs

@@ -50,27 +50,12 @@ pub(super) fn median(v: Vec<f64>) -> R<f64> {
 
 /// Values occurring at least twice, ordered by first occurrence, with their (equal, maximal) count.
 fn modes(v: &[f64]) -> Vec<f64> {
-    // Index of each distinct number in `counts`, so counting is linear rather than a search of
-    // every distinct number so far. Keyed by bits with -0 folded into 0 (they compare equal); NaN
-    // never equals anything, so it is never merged.
-    let mut index: std::collections::HashMap<u64, usize> = std::collections::HashMap::new();
     let mut counts: Vec<(f64, usize)> = Vec::new();
     for &x in v {
-        if x.is_nan() {
+        if let Some(e) = counts.iter_mut().find(|e| e.0 == x) {
+            e.1 += 1;
+        } else {
             counts.push((x, 1));
-            continue;
-        }
-        let key = if x == 0.0 { 0 } else { x.to_bits() };
-        match index.entry(key) {
-            std::collections::hash_map::Entry::Occupied(e) => {
-                if let Some(c) = counts.get_mut(*e.get()) {
-                    c.1 += 1;
-                }
-            }
-            std::collections::hash_map::Entry::Vacant(e) => {
-                e.insert(counts.len());
-                counts.push((x, 1));
-            }
         }
     }
     let best = counts.iter().map(|e| e.1).max().unwrap_or(0);
@@ -470,10 +455,7 @@ pub(super) fn frequency(a: &[Arg], _: C) -> R<Value> {
     order.sort_by(|&i, &j| bins.get(i).copied().unwrap_or(0.0).total_cmp(&bins.get(j).copied().unwrap_or(0.0)));
     let mut counts = vec![0.0; bins.len() + 1];
     for x in data {
-        // `order` sorts the bins ascending, so the bins `x` exceeds come first: binary search for
-        // the first bin at or above `x` (the first of equal bins, as a linear scan would find).
-        let pos = order.partition_point(|&i| bins.get(i).copied().unwrap_or(f64::INFINITY) < x);
-        let slot = order.get(pos).copied().unwrap_or(bins.len());
+        let slot = order.iter().find(|&&i| x <= bins.get(i).copied().unwrap_or(f64::INFINITY)).copied().unwrap_or(bins.len());
         if let Some(c) = counts.get_mut(slot) {
             *c += 1.0;
         }

crates/functions/src/stat/tests.rs

@@ -617,20 +617,3 @@ fn aggregate_helper() {
     close(aggregate_values_k(9, &vals, &nv(2.0)), 9.0);
     is_err(aggregate_values_k(14, &vals, &nv(10.0)), CellError::Num);
 }
-
-/// Security: FREQUENCY finds each value's bin by binary search, and MODE counts distinct numbers
-/// through an index, so neither is quadratic in its inputs. Results are as before: unsorted and
-/// duplicate bins keep their positions (a value goes to the first of equal bins), and tied modes
-/// come in first-occurrence order.
-#[test]
-fn frequency_and_mode_scale_with_their_inputs() {
-    let f = ev("FREQUENCY", vec![c(&[1.0, 2.0, 2.0, 3.0, 5.0, 9.0, -1.0]), c(&[3.0, 2.0, 2.0, 0.0])]);
-    assert_eq!(rows_of(&f), vec![vec![nv(1.0)], vec![nv(3.0)], vec![nv(0.0)], vec![nv(1.0)], vec![nv(2.0)]]);
-    let data: Vec<f64> = (0..200_000).map(|i| f64::from(i) + 1e6).collect();
-    let bins: Vec<f64> = (0..200_000).map(f64::from).collect();
-    let f = ev("FREQUENCY", vec![c(&data), c(&bins)]);
-    assert_eq!(rows_of(&f).last(), Some(&vec![nv(200_000.0)]));
-    let distinct = ev("MODE", vec![c(&data)]);
-    is_err(distinct, CellError::NA);
-    assert_eq!(rows_of(&ev("MODE.MULT", vec![c(&[3.0, 0.0, 1.0, -0.0, 3.0, 1.0])])), vec![vec![nv(3.0)], vec![nv(0.0)], vec![nv(1.0)]]);
-}