mirror of
https://github.com/storytold/gridcraft.git
synced 2026-10-25 03:06:53 -06:00
calc: cap LINEST-family regressor matrices (#2177)
LINEST, LOGEST, TREND and GROWTH built k x k normal-equation matrices (plus two copies in sym_inverse, O(k^3)) with k taken straight from the known_xs shape, so one formula could allocate gigabytes and abort on open. More than 255 predictors is now #VALUE!, and more than 1e9 rows x k^2 multiply-adds is #NUM!, both checked before any allocation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
912b817abe
commit
e1b8aad143
@@ -534,6 +534,25 @@ fn strict_num(v: &Value) -> R<f64> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Most predictor columns LINEST, LOGEST, TREND and GROWTH accept. The normal equations are a
|
||||
/// k×k matrix inverted in O(k³), so an unbounded k (one `SEQUENCE(1,16384)` argument) allocated
|
||||
/// gigabytes and could abort the process.
|
||||
const MAX_REGRESSORS: usize = 255;
|
||||
/// Most multiply-adds (rows × k²) a regression may spend building its normal equations.
|
||||
const MAX_REGRESSION_WORK: usize = 1_000_000_000;
|
||||
|
||||
/// Rejects regressions too large to fit: more than [`MAX_REGRESSORS`] predictors is `#VALUE!`,
|
||||
/// more than [`MAX_REGRESSION_WORK`] multiply-adds is `#NUM!`.
|
||||
fn check_regression_size(n: usize, k: usize) -> R<()> {
|
||||
if k > MAX_REGRESSORS {
|
||||
return Err(CellError::Value);
|
||||
}
|
||||
match n.checked_mul(k).and_then(|nk| nk.checked_mul(k)) {
|
||||
Some(work) if work <= MAX_REGRESSION_WORK => Ok(()),
|
||||
_ => Err(CellError::Num),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_xy(y: &Value, x: Option<&Value>) -> R<Xy> {
|
||||
let ya = as_array(y);
|
||||
let n = ya.data.len();
|
||||
@@ -546,12 +565,14 @@ fn parse_xy(y: &Value, x: Option<&Value>) -> R<Xy> {
|
||||
if xa.rows == ya.rows && xa.cols == ya.cols {
|
||||
(xa.data.iter().map(|v| strict_num(v).map(|f| vec![f])).collect::<R<Vec<_>>>()?, 1)
|
||||
} else if !y_is_row && xa.rows == n {
|
||||
check_regression_size(n, xa.cols)?;
|
||||
let mut rows = Vec::with_capacity(n);
|
||||
for r in 0..n {
|
||||
rows.push((0..xa.cols).map(|c| strict_num(xa.get(r, c).unwrap_or(&Value::Empty))).collect::<R<Vec<_>>>()?);
|
||||
}
|
||||
(rows, xa.cols)
|
||||
} else if y_is_row && xa.cols == n {
|
||||
check_regression_size(n, xa.rows)?;
|
||||
let mut rows = Vec::with_capacity(n);
|
||||
for c in 0..n {
|
||||
rows.push((0..xa.rows).map(|r| strict_num(xa.get(r, c).unwrap_or(&Value::Empty))).collect::<R<Vec<_>>>()?);
|
||||
@@ -638,6 +659,7 @@ fn regress(y: &[f64], x: &[Vec<f64>], k: usize, constant: bool) -> R<Fit> {
|
||||
if n == 0 || x.len() != n {
|
||||
return Err(CellError::Value);
|
||||
}
|
||||
check_regression_size(n, k)?;
|
||||
let nf = n as f64;
|
||||
let (xm, ym) = if constant {
|
||||
let xm: Vec<f64> = (0..k).map(|j| x.iter().map(|r| r.get(j).copied().unwrap_or(0.0)).sum::<f64>() / nf).collect();
|
||||
@@ -2245,7 +2267,7 @@ pub(crate) fn specs() -> Vec<FnSpec> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{aggregate_values, aggregate_values_k};
|
||||
use super::{MAX_REGRESSORS, aggregate_values, aggregate_values_k, check_regression_size};
|
||||
use crate::util::testutil::*;
|
||||
use gridcraft_core::{CellError, Value};
|
||||
|
||||
@@ -2531,6 +2553,28 @@ mod tests {
|
||||
is_err(ev("LINEST", vec![c(&[1.0, 2.0]), c(&[1.0, 2.0, 3.0])]), CellError::Ref);
|
||||
}
|
||||
|
||||
// #2177: one predictor row of 16,384 columns built three 16,384² matrices (6 GB) and an O(k³)
|
||||
// inversion; a wider TOROW aborted the process. It is now refused before allocating.
|
||||
#[test]
|
||||
fn linest_family_caps_regressor_count() {
|
||||
let wide = |k: usize| av(arr(vec![(0..k).map(|i| nv(i as f64)).collect()]));
|
||||
let tall = |k: usize| av(arr((0..k).map(|i| vec![nv(i as f64), nv(1.0)]).collect()));
|
||||
for f in ["LINEST", "LOGEST", "TREND", "GROWTH"] {
|
||||
// y is one cell, so every column of x is a predictor.
|
||||
is_err(ev(f, vec![n(1.0), wide(16_384)]), CellError::Value);
|
||||
// y is a row, so every row of x is a predictor.
|
||||
is_err(ev(f, vec![av(arr(vec![vec![nv(1.0), nv(2.0)]])), tall(16_384)]), CellError::Value);
|
||||
}
|
||||
is_err(ev("LINEST", vec![n(1.0), wide(MAX_REGRESSORS + 1)]), CellError::Value);
|
||||
// At the cap the fit still runs (one observation: coefficients are zero, not an error).
|
||||
assert_eq!(rows_of(&ev("LINEST", vec![n(1.0), wide(MAX_REGRESSORS)]))[0].len(), MAX_REGRESSORS + 1);
|
||||
// The rows × k² work bound.
|
||||
assert_eq!(check_regression_size(1_000, MAX_REGRESSORS), Ok(()));
|
||||
assert_eq!(check_regression_size(1_000_000, 100), Err(CellError::Num));
|
||||
assert_eq!(check_regression_size(usize::MAX, 2), Err(CellError::Num));
|
||||
assert_eq!(check_regression_size(10_000_000, 1), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frequency_prob() {
|
||||
let f = ev("FREQUENCY", vec![c(&[79.0, 85.0, 78.0, 85.0, 50.0, 81.0, 95.0, 88.0, 97.0]), c(&[70.0, 79.0, 89.0])]);
|
||||
|
||||
Reference in New Issue
Block a user