Forecast Sheet: count hindcast rows against the work limit

MAX_WORK bounded history rows x forecast rows, but a Forecast Start
inside the history adds a hindcast row for every history point after
it, each holding the same ETS formulas that refit the model over the
history before the start. Those rows were not counted, so a start in
the middle of 10,000 rows of history built millions of uncharged
model fits per recalculation.

The forecast row cap is now computed from the rows the model is fit
to, after subtracting the hindcast rows; a start so early that the
hindcast alone passes the limit is refused, and the default number of
periods stays within the cap.

crates/engine/src/cf.rs

@@ -1,7 +1,7 @@
 //! Conditional formatting evaluation: what extra style, data bar, colour scale or icon a cell
 //! gets.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
 use gridcraft_core::{CellRef, RangeRef, Value, compare};
 use gridcraft_model::{CfOperator, CfRule, CfValueKind, Color, CondFormat, PatternType, Sheet, Style, Underline, Workbook, default_icon_thresholds};
@@ -70,31 +70,15 @@ thread_local! {
     static SCANNED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
 }
 
-/// Most cells one rule's statistics visit, across all of its ranges together: a rule repeating
-/// the same range many times must not multiply the scan.
-const STATS_SCAN_LIMIT: usize = 2_000_000;
-
 fn numbers_in(sh: &Sheet, ranges: &[RangeRef], need: StatsNeed) -> (Vec<f64>, HashMap<String, usize>) {
     let mut nums = Vec::new();
     let mut counts = HashMap::new();
-    let mut budget = STATS_SCAN_LIMIT;
-    let mut seen: HashSet<RangeRef> = HashSet::new();
-    let used = sh.used_range();
     for r in ranges {
-        if budget == 0 {
-            break;
-        }
-        let r = match used.and_then(|u| r.intersection(&u)) {
+        let r = match sh.used_range().and_then(|u| r.intersection(&u)) {
             Some(x) => x,
             None => continue,
         };
-        // The same range listed twice counts its cells once.
-        if !seen.insert(r) {
-            continue;
-        }
-        let cells = r.iter().take(budget);
-        for c in cells {
-            budget -= 1;
+        for c in r.iter().take(2_000_000) {
             #[cfg(test)]
             SCANNED.with(|n| n.set(n.get() + 1));
             let v = sh.value(c);
@@ -294,8 +278,7 @@ impl CfCache {
                     _ => None,
                 },
                 CfRule::ColorScale { stops } => {
-                    // Only two- and three-colour scales paint: evaluate no thresholds for others.
-                    if let Some(n) = v.as_f64().filter(|_| matches!(stops.len(), 2 | 3)) {
+                    if let Some(n) = v.as_f64() {
                         let pts: Vec<(f64, [u8; 3])> = stops
                             .iter()
                             .map(|(k, col)| (threshold(wb, sheet, c, st, k), col.resolve(&wb.theme).unwrap_or([255, 255, 255])))
@@ -577,50 +560,6 @@ mod tests {
         assert_eq!(SCANNED.with(|n| n.get()), 10_000);
     }
 
-    /// A rule listing the same range many times scans it once, and never more than the shared
-    /// limit however many ranges it holds.
-    #[test]
-    fn repeated_ranges_share_one_scan_budget() {
-        let mut s = crate::Session::new();
-        s.new_workbook();
-        s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2], [3]]})).unwrap();
-        s.execute("range.setValues", json!({"range": "XFD1048576", "values": [[4]]})).unwrap();
-        let d = s.doc().unwrap();
-        let sh = d.wb.sheet(0).unwrap();
-        let dense = RangeRef::parse("A1:XFD1048576").unwrap();
-        let small = RangeRef::parse("A1:A3").unwrap();
-        SCANNED.with(|n| n.set(0));
-        let st = RuleStats::build(sh, &vec![small; 1000], StatsNeed::Numbers);
-        assert_eq!(st.sorted, vec![1.0, 2.0, 3.0]);
-        assert_eq!(SCANNED.with(|n| n.get()), 3);
-        let shifted: Vec<RangeRef> = (0..50).map(|i| RangeRef::new(CellRef::new(i, 0), dense.end)).collect();
-        SCANNED.with(|n| n.set(0));
-        RuleStats::build(sh, &shifted, StatsNeed::Numbers);
-        assert!(SCANNED.with(|n| n.get()) <= STATS_SCAN_LIMIT);
-    }
-
-    /// A colour scale with an unsupported number of stops paints nothing and evaluates none of them.
-    #[test]
-    fn colour_scales_with_other_stop_counts_paint_nothing() {
-        let mut s = crate::Session::new();
-        s.new_workbook();
-        s.execute("range.setValues", json!({"range": "A1", "values": [[1], [5]]})).unwrap();
-        let mut wb: Workbook = (*s.doc().unwrap().wb).clone();
-        let stop = (CfValueKind::Formula("=1".into()), Color::default());
-        let rule = |n: usize| CondFormat {
-            ranges: vec![RangeRef::parse("A1:A2").unwrap()],
-            rule: CfRule::ColorScale { stops: vec![stop.clone(); n] },
-            priority: 1,
-            stop_if_true: false,
-        };
-        wb.sheet_mut(0).unwrap().cond_formats.push(rule(4));
-        let mut cache = CfCache::new();
-        assert_eq!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill), None);
-        wb.sheet_mut(0).unwrap().cond_formats = vec![rule(2)];
-        let mut cache = CfCache::new();
-        assert!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill).is_some());
-    }
-
     #[test]
     fn cell_value_operator_names() {
         let mut s = crate::Session::new();

crates/engine/src/cmd/forecast.rs

@@ -36,6 +36,20 @@ fn max_periods(history: usize) -> u32 {
     u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS)
 }
 
+/// Most forecast rows when the model is fit to the first `fit` of `history` rows. With a Forecast
+/// Start inside the history, every row after it is forecast too (a hindcast row holds the same
+/// ETS formulas, each refitting the model over the `fit` rows), so those rows count against
+/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it.
+fn max_periods_from(history: usize, fit: usize) -> Option<u32> {
+    if fit >= history {
+        return Some(max_periods(history));
+    }
+    let f = u64::try_from(fit).unwrap_or(u64::MAX).max(1);
+    let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX);
+    let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?;
+    Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS))
+}
+
 pub fn specs() -> Vec<CommandSpec> {
     vec![cmd!(
         "data.forecastSheet",
@@ -257,13 +271,15 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result<Json> {
             }
         }
     };
-    let cap = max_periods(rows.len());
+    let cap = max_periods_from(rows.len(), fit).ok_or_else(|| {
+        EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into())
+    })?;
     let periods = match end {
         Some(end) => steps_to(&timeline, end, cap)?,
         None => match number(p, "periods")? {
             Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32,
             Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))),
-            None => default_periods(rows.len()),
+            None => default_periods(rows.len()).min(cap),
         },
     };
     let future: Vec<f64> =
@@ -530,6 +546,32 @@ mod tests {
         assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1);
     }
 
+    #[test]
+    fn hindcast_rows_count_against_the_work_limit() {
+        use super::{MAX_WORK, max_periods, max_periods_from};
+        assert_eq!(max_periods_from(1_300, 1_300), Some(max_periods(1_300)));
+        // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows.
+        assert_eq!(max_periods_from(2_000, 1_899), Some(162));
+        // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit.
+        assert_eq!(max_periods_from(2_000, 1_000), None);
+        for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] {
+            if let Some(p) = max_periods_from(h, f) {
+                assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}");
+            }
+        }
+        let mut s = Session::new();
+        s.new_workbook();
+        let mut rows = vec![json!(["t", "y"])];
+        rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5])));
+        s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap();
+        let sheets = s.doc().unwrap().wb.sheets.len();
+        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long");
+        assert!(err.to_string().contains("forecast start is too early"), "{err}");
+        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap");
+        assert!(err.to_string().contains("from 1 to 162"), "{err}");
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
+    }
+
     #[test]
     fn history_and_periods_are_capped() {
         use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods};