functions: TEXTJOIN builds its result piece by piece within the text cap

TEXTJOIN converted every input and delimiter into an owned String before
joining, so many references to one long shared text copied gigabytes before
the joined length was ever checked. Errors are now found first without
building text (so they still win in argument order), then each piece and
delimiter is converted only as it is appended, stopping at the cap.

crates/engine/src/cmd/forecast.rs

@@ -36,21 +36,6 @@ fn max_periods(history: usize) -> u32 {
     u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS)
 }
 
-/// Most forecast rows when the model is fit to the first `fit` of `history` rows, which span
-/// `slots` timeline slots once their gaps are filled. With a Forecast Start inside the history,
-/// every row after it is forecast too (a hindcast row holds the same ETS formulas, each refitting
-/// the model over the `slots` slots), so those rows (counted from the raw rows) count against
-/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it.
-fn max_periods_from(history: usize, fit: usize, slots: usize) -> Option<u32> {
-    if fit >= history {
-        return Some(max_periods(history));
-    }
-    let f = u64::try_from(slots.max(fit)).unwrap_or(u64::MAX).max(1);
-    let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX);
-    let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?;
-    Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS))
-}
-
 pub fn specs() -> Vec<CommandSpec> {
     vec![cmd!(
         "data.forecastSheet",
@@ -272,23 +257,13 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result<Json> {
             }
         }
     };
-    // The slots the model is fit over: the fit rows' span of the timeline, gaps included.
-    let slots = match (fit < rows.len(), times.get(fit.wrapping_sub(1)), times.first()) {
-        (true, Some(last_fit), Some(first)) => match (timeline.steps_after_last(*last_fit), timeline.steps_after_last(*first)) {
-            (Some(b), Some(a)) => ((b - a).round() + 1.0).clamp(0.0, 1e9) as usize,
-            _ => fit,
-        },
-        _ => fit,
-    };
-    let cap = max_periods_from(rows.len(), fit, slots).ok_or_else(|| {
-        EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into())
-    })?;
+    let cap = max_periods(rows.len());
     let periods = match end {
         Some(end) => steps_to(&timeline, end, cap)?,
         None => match number(p, "periods")? {
             Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32,
             Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))),
-            None => default_periods(rows.len()).min(cap),
+            None => default_periods(rows.len()),
         },
     };
     let future: Vec<f64> =
@@ -555,51 +530,6 @@ mod tests {
         assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1);
     }
 
-    #[test]
-    fn hindcast_rows_count_against_the_work_limit() {
-        use super::{MAX_WORK, max_periods, max_periods_from};
-        assert_eq!(max_periods_from(1_300, 1_300, 1_300), Some(max_periods(1_300)));
-        // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows.
-        assert_eq!(max_periods_from(2_000, 1_899, 1_899), Some(162));
-        // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit.
-        assert_eq!(max_periods_from(2_000, 1_000, 1_000), None);
-        // 800 rows fit to the first 400, which span 571 slots (gaps filled): each formula refits
-        // 571 slots, so 475 forecast rows after the 400 hindcast rows, not 850.
-        assert_eq!(max_periods_from(800, 400, 571), Some(475));
-        assert!((475 + 400) * 571 <= MAX_WORK as usize);
-        for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] {
-            if let Some(p) = max_periods_from(h, f, f) {
-                assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}");
-            }
-        }
-        let mut s = Session::new();
-        s.new_workbook();
-        let mut rows = vec![json!(["t", "y"])];
-        rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5])));
-        s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap();
-        let sheets = s.doc().unwrap().wb.sheets.len();
-        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long");
-        assert!(err.to_string().contains("forecast start is too early"), "{err}");
-        let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap");
-        assert!(err.to_string().contains("from 1 to 162"), "{err}");
-        assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets);
-    }
-
-    #[test]
-    fn the_hindcast_cap_follows_the_slots_the_fit_spans_not_its_rows() {
-        // 800 rows, 20% of the slots missing: the first 400 rows span 500 slots, so each ETS
-        // formula refits 500 slots and the cap is 1,000 - 400 = 600 (it was 850 by rows).
-        let mut s = Session::new();
-        s.new_workbook();
-        let times: Vec<u32> = (1u32..).filter(|t| !matches!(t % 10, 3 | 6)).take(800).collect();
-        let mut rows = vec![json!(["t", "y"])];
-        rows.extend(times.iter().map(|t| json!([t, (t % 7) as f64 + f64::from(*t) * 0.5])));
-        s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap();
-        let start = times[400];
-        let err = s.execute("data.forecastSheet", json!({"range": "A1:B801", "forecastStart": start, "periods": 601})).expect_err("past the cap");
-        assert!(err.to_string().contains("from 1 to 600"), "{err}");
-    }
-
     #[test]
     fn history_and_periods_are_capped() {
         use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods};

crates/functions/src/text.rs

@@ -53,25 +53,41 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     check_len(out)
 }
 
+/// The error `v.to_text()` would give, found without building any text.
+fn text_error(v: &Value) -> Option<CellError> {
+    match v {
+        Value::Error(e) => Some(*e),
+        Value::Array(a) => a.data.first().and_then(text_error),
+        _ => None,
+    }
+}
+
 fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let delims: Vec<String> = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::<R<_>>()?;
+    // Errors are found first, in argument order, so they win as before; text is then built one
+    // piece at a time into the capped output, so many references to one long text (or a long
+    // delimiter list) stop at the cap instead of first copying every piece.
+    let delims = as_array(&arg(a, 0)?.value);
+    if let Some(e) = delims.iter().find_map(text_error) {
+        return Err(e);
+    }
     let ignore = boolean(a, 1)?;
-    let mut items = Vec::new();
-    for x in a.iter().skip(2) {
-        for v in as_array(&x.value).iter() {
-            let s = v.to_text()?;
-            if ignore && s.is_empty() {
-                continue;
-            }
-            items.push(s);
-        }
+    let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect();
+    if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) {
+        return Err(e);
     }
     let mut out = String::new();
-    for (i, s) in items.iter().enumerate() {
-        if i > 0 && !delims.is_empty() {
-            push_capped(&mut out, &delims[(i - 1) % delims.len()])?;
+    let mut joined = 0usize;
+    for v in items.iter().flat_map(|x| x.iter()) {
+        let s = v.to_text()?;
+        if ignore && s.is_empty() {
+            continue;
         }
-        push_capped(&mut out, s)?;
+        if joined > 0 && !delims.data.is_empty() {
+            let d = delims.data.get((joined - 1) % delims.data.len()).map(Value::to_text).transpose()?.unwrap_or_default();
+            push_capped(&mut out, &d)?;
+        }
+        push_capped(&mut out, &s)?;
+        joined += 1;
     }
     check_len(out)
 }
@@ -1131,6 +1147,22 @@ mod tests {
         assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]);
     }
 
+    /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather
+    /// than first copying every input (100,000 references to one 32,767-character text were
+    /// 3 GB of copies). Errors among the inputs still win, in argument order.
+    #[test]
+    fn textjoin_stops_at_the_text_cap() {
+        use gridcraft_core::Array;
+        let long = tv(&"x".repeat(32_767));
+        let many = || av(Array::filled(100_000, 1, long.clone()).into());
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value);
+        is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0);
+        is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA);
+        // Delimiters cycle and are converted only where used.
+        is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d");
+    }
+
     fn row_t(xs: &[&str]) -> Value {
         arr(vec![xs.iter().map(|x| tv(x)).collect()])
     }