Forecast Sheet: count hindcast rows against the work limit

MAX_WORK bounded history rows x forecast rows, but a Forecast Start
inside the history adds a hindcast row for every history point after
it, each holding the same ETS formulas that refit the model over the
history before the start. Those rows were not counted, so a start in
the middle of 10,000 rows of history built millions of uncharged
model fits per recalculation.

The forecast row cap is now computed from the rows the model is fit
to, after subtracting the hindcast rows; a start so early that the
hindcast alone passes the limit is refused, and the default number of
periods stays within the cap.

crates/chart/src/resolve.rs

@@ -11,10 +11,6 @@ use crate::{ChartData, Rgba, SeriesData, TrendlineData};
 pub(crate) const MAX_POINTS: usize = 100_000;
 /// Cap on series per chart.
 pub(crate) const MAX_SERIES: usize = 255;
-/// Cap on the points of all series of one chart together (values, X values and bubble sizes):
-/// [`MAX_SERIES`] × [`MAX_POINTS`] would let a few short whole-column references retain and
-/// draw tens of millions of points. Series past it get only what is left, or no points.
-pub(crate) const MAX_CHART_POINTS: usize = 1_000_000;
 /// Cap on trendlines drawn per series.
 pub(crate) const MAX_TRENDLINES: usize = 16;
 
@@ -81,15 +77,10 @@ fn union_parts(f: &str) -> Option<Vec<&str>> {
 /// Evaluates a series formula to a flat list of values (row-major), capped. A union of
 /// references (`(A1:A3,A5)`, as PivotCharts use) is read part by part.
 fn eval_list(wb: &Workbook, sheet: usize, formula: &str) -> Vec<Value> {
-    eval_list_in(wb, sheet, formula, 0, MAX_POINTS)
+    eval_list_in(wb, sheet, formula, 0)
 }
 
-/// [`eval_list`] keeping at most `cap` values (and never more than [`MAX_POINTS`]).
-fn eval_list_capped(wb: &Workbook, sheet: usize, formula: &str, cap: usize) -> Vec<Value> {
-    eval_list_in(wb, sheet, formula, 0, cap.min(MAX_POINTS))
-}
-
-fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, depth: u32, cap: usize) -> Vec<Value> {
+fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, depth: u32) -> Vec<Value> {
     let f = formula.trim();
     if f.is_empty() {
         return vec![];
@@ -99,19 +90,16 @@ fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, depth: u32, cap: usi
     {
         let mut out = Vec::new();
         for p in parts {
-            if out.len() >= cap {
+            if out.len() >= MAX_POINTS {
                 break;
             }
-            let rest = cap - out.len();
-            out.extend(eval_list_in(wb, sheet, p, depth + 1, rest).into_iter().take(rest));
+            let rest = MAX_POINTS - out.len();
+            out.extend(eval_list_in(wb, sheet, p, depth + 1).into_iter().take(rest));
         }
         return out;
     }
-    if cap == 0 {
-        return vec![];
-    }
     match evaluate(wb, sheet, CellRef::default(), f) {
-        Value::Array(a) => a.iter().take(cap).cloned().collect(),
+        Value::Array(a) => a.iter().take(MAX_POINTS).cloned().collect(),
         v => vec![v],
     }
 }
@@ -123,11 +111,8 @@ fn to_num(v: &Value) -> Option<f64> {
     }
 }
 
-/// A series formula's numbers, at most `cap` of them; `cap` is reduced by what was taken.
-fn nums(wb: &Workbook, sheet: usize, formula: &str, cap: &mut usize) -> Vec<Option<f64>> {
-    let out: Vec<Option<f64>> = eval_list_capped(wb, sheet, formula, *cap).iter().map(to_num).collect();
-    *cap -= out.len().min(*cap);
-    out
+fn nums(wb: &Workbook, sheet: usize, formula: &str) -> Vec<Option<f64>> {
+    eval_list(wb, sheet, formula).iter().map(to_num).collect()
 }
 
 /// Number format code of the first cell of a reference ("General" when not a reference).
@@ -243,21 +228,15 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData {
     let mut series = Vec::new();
     let mut categories: Option<Vec<String>> = None;
     let mut category_levels = Vec::new();
-    // One point budget for the whole chart, not one per series.
-    let mut budget = MAX_CHART_POINTS;
     for (i, s) in chart.series.iter().take(MAX_SERIES).enumerate() {
-        let values = nums(wb, sheet, &s.values, &mut budget);
+        let values = nums(wb, sheet, &s.values);
         let name = s.name.as_deref().map(|n| series_name(wb, sheet, n)).filter(|n| !n.is_empty()).unwrap_or_else(|| format!("Series{}", i + 1));
         let color =
             s.color.as_ref().and_then(|c| c.resolve(&wb.theme)).map(|[r, g, b]| [r, g, b, 0xFF]).unwrap_or_else(|| series_color(&wb.theme, i));
         let kind = effective_kind(chart.kind, s.kind, i);
         let x = match (&s.categories, numeric_x || matches!(kind, ChartKind::Scatter | ChartKind::ScatterLines | ChartKind::Bubble)) {
             // X values with text in them (labels) are ignored: the points are numbered 1..n.
-            (Some(f), true) => {
-                let xs = eval_list_capped(wb, sheet, f, budget);
-                budget -= xs.len().min(budget);
-                Some(xs).filter(|xs| !xs.iter().any(Value::is_text)).map(|xs| xs.iter().map(to_num).collect())
-            }
+            (Some(f), true) => Some(eval_list(wb, sheet, f)).filter(|xs| !xs.iter().any(Value::is_text)).map(|xs| xs.iter().map(to_num).collect()),
             _ => None,
         };
         if categories.is_none()
@@ -269,7 +248,7 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData {
                 category_levels = outer;
             }
         }
-        let sizes = s.bubble_sizes.as_deref().map(|f| nums(wb, sheet, f, &mut budget));
+        let sizes = s.bubble_sizes.as_deref().map(|f| nums(wb, sheet, f));
         let trendlines = s
             .trendlines
             .iter()

crates/chart/src/tests.rs

@@ -1258,21 +1258,3 @@ fn box_whisker_draws_each_series_quartile_method() {
     // The boxes span 3.5 and 2.5 data units on the same value axis.
     assert!((heights[0] / heights[1] - 1.4).abs() < 0.001);
 }
-
-#[test]
-fn series_share_one_point_budget() {
-    // Many series over the same long column each get a share of one chart-wide budget, not
-    // `MAX_POINTS` apiece.
-    let mut wb = Workbook::new();
-    let s = wb.sheet_mut(0).unwrap();
-    s.set_value(CellRef::new(0, 0), Value::Number(1.0));
-    s.set_value(CellRef::new(59_999, 0), Value::Number(2.0));
-    let mut c = chart(ChartKind::ColumnClustered);
-    c.series = (0..40).map(|_| Series { values: "Sheet1!$A$1:$A$60000".into(), ..Default::default() }).collect();
-    let d = resolve(&wb, 0, &c);
-    assert_eq!(d.series.len(), 40);
-    assert_eq!(d.series[0].values.len(), 60_000);
-    let total: usize = d.series.iter().map(|s| s.values.len()).sum();
-    assert_eq!(total, crate::resolve::MAX_CHART_POINTS);
-    assert!(d.series.last().is_some_and(|s| s.values.is_empty()));
-}

crates/engine/src/chart_svg.rs

@@ -10,10 +10,6 @@ use crate::io::esc;
 /// Widest or tallest chart written; larger anchors are clamped.
 const MAX_SIDE: f32 = 16_384.0;
 
-/// Most bytes of SVG written for the charts of one web page together. A chart past what is
-/// left is left out of the page rather than grown without bound.
-pub(crate) const MAX_PAGE_SVG_BYTES: usize = 64 * 1024 * 1024;
-
 struct HelveticaMeasure;
 
 impl gridcraft_chart::Measure for HelveticaMeasure {
@@ -22,34 +18,24 @@ impl gridcraft_chart::Measure for HelveticaMeasure {
     }
 }
 
-/// `chart` of sheet `sheet` as an `<svg>` element placed by `style`, or `None` for a degenerate
-/// size or an element longer than `budget` bytes.
-pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str, budget: usize) -> Option<String> {
+/// `chart` of sheet `sheet` as an `<svg>` element placed by `style`, or `None` for a degenerate size.
+pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str) -> Option<String> {
     let (w, h) = (chart.anchor.width.min(MAX_SIDE), chart.anchor.height.min(MAX_SIDE));
     if !(w >= 1.0 && h >= 1.0) {
         return None;
     }
     let data = gridcraft_chart::resolve(wb, sheet, chart);
     let prims = gridcraft_chart::render(chart, &data, w, h, &HelveticaMeasure);
-    prims_svg_within(&prims, w, h, style, budget)
+    Some(prims_svg(&prims, w, h, style))
 }
 
-#[cfg(test)]
 fn prims_svg(prims: &[Prim], w: f32, h: f32, style: &str) -> String {
-    prims_svg_within(prims, w, h, style, usize::MAX).unwrap_or_default()
-}
-
-/// The `<svg>` element, or `None` as soon as it grows past `budget` bytes.
-fn prims_svg_within(prims: &[Prim], w: f32, h: f32, style: &str, budget: usize) -> Option<String> {
     let mut out = String::new();
     let _ = write!(
         out,
         "<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"{w}\" height=\"{h}\" viewBox=\"0 0 {w} {h}\" font-family=\"Helvetica,Arial,sans-serif\" style=\"{style}\">"
     );
     for p in prims {
-        if out.len() > budget {
-            return None;
-        }
         match p {
             Prim::Rect { x, y, w, h, fill, stroke, radius } => {
                 let _ = write!(out, "<rect x=\"{}\" y=\"{}\" width=\"{}\" height=\"{}\"", num(*x), num(*y), num(*w), num(*h));
@@ -106,7 +92,7 @@ fn prims_svg_within(prims: &[Prim], w: f32, h: f32, style: &str, budget: usize)
         }
     }
     out.push_str("</svg>");
-    (out.len() <= budget).then_some(out)
+    out
 }
 
 /// Fill and stroke attributes of an open shape element; the caller closes it.
@@ -163,17 +149,6 @@ fn num(v: f32) -> String {
 mod tests {
     use super::*;
 
-    #[test]
-    fn an_svg_past_its_byte_budget_is_left_out() {
-        let rect = Prim::Rect { x: 1.0, y: 2.0, w: 3.0, h: 4.0, fill: Some([255, 0, 0, 255]), stroke: None, radius: 0.0 };
-        let prims = vec![rect; 10_000];
-        let full = prims_svg(&prims, 100.0, 50.0, "");
-        assert!(full.len() > 100_000);
-        assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len()), Some(full.clone()));
-        assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len() - 1), None);
-        assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", 1_000), None);
-    }
-
     #[test]
     fn every_primitive_becomes_an_svg_element() {
         let red = [255, 0, 0, 255];

crates/engine/src/cmd/forecast.rs

@@ -36,6 +36,20 @@ fn max_periods(history: usize) -> u32 {
     u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS)
 }
 
+/// Most forecast rows when the model is fit to the first `fit` of `history` rows. With a Forecast
+/// Start inside the history, every row after it is forecast too (a hindcast row holds the same
+/// ETS formulas, each refitting the model over the `fit` rows), so those rows count against
+/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it.
+fn max_periods_from(history: usize, fit: usize) -> Option<u32> {
+    if fit >= history {
+        return Some(max_periods(history));
+    }
+    let f = u64::try_from(fit).unwrap_or(u64::MAX).max(1);
+    let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX);
+    let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?;
+    Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS))
+}
+
 pub fn specs() -> Vec<CommandSpec> {
     vec![cmd!(
         "data.forecastSheet",
@@ -257,13 +271,15 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result<Json> {
             }
         }
     };
-    let cap = max_periods(rows.len());
+    let cap = max_periods_from(rows.len(), fit).ok_or_else(|| {
+        EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into())
+    })?;
     let periods = match end {
         Some(end) => steps_to(&timeline, end, cap)?,
         None => match number(p, "periods")? {
             Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32,
             Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))),
-            None => default_periods(rows.len()),
+            None => default_periods(rows.len()).min(cap),
         },
     };
     let future: Vec<f64> =
@@ -530,6 +546,32 @@ mod tests {
         assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1);
     }
 
+    #[test]
+    fn hindcast_rows_count_against_the_work_limit() {
+        use super::{MAX_WORK, max_periods, max_periods_from};
+        assert_eq!(max_periods_from(1_300, 1_300), Some(max_periods(1_300)));
+        // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows.
+        assert_eq!(max_periods_from(2_000, 1_899), Some(162));
+        // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit.
+        assert_eq!(max_periods_from(2_000, 1_000), None);
+        for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] {
+            if let Some(p) = max_periods_from(h, f) {
+                assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}");
+            }
+        }
+        let mut s = Session::new();
+        s.new_workbook();
+        let mut rows = vec![json!(["t", "y"])];
+        rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5])));
+        s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap();
+        let sheets = s.doc().unwrap().wb.sheets.len();
+        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long");
+        assert!(err.to_string().contains("forecast start is too early"), "{err}");
+        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap");
+        assert!(err.to_string().contains("from 1 to 162"), "{err}");
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
+    }
+
     #[test]
     fn history_and_periods_are_capped() {
         use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods};

crates/engine/src/io.rs

@@ -736,10 +736,8 @@ pub fn to_html(wb: &Workbook, sheet: usize) -> Result<String> {
                 place(&im.anchor)
             );
         }
-        let mut svg_budget = crate::chart_svg::MAX_PAGE_SVG_BYTES;
         for ch in &sh.charts {
-            if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor), svg_budget) {
-                svg_budget -= svg.len();
+            if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor)) {
                 out.push_str(&svg);
             }
         }