diff --git a/crates/engine/src/pivot/charts.rs b/crates/engine/src/pivot/charts.rs index bc5d8e597..97fea4f42 100644 --- a/crates/engine/src/pivot/charts.rs +++ b/crates/engine/src/pivot/charts.rs @@ -57,6 +57,10 @@ fn rows_ref(sheet: &str, origin: CellRef, col: u32, rows: &[u32]) -> Option Vec { @@ -64,6 +68,7 @@ pub fn chart_series(sheet: &str, origin: CellRef, out: &Output) -> Vec { let categories = out.label_col.and_then(|c| rows_ref(sheet, origin, c, &rows)); out.chart_cols .iter() + .take(MAX_CHART_SERIES) .filter_map(|(c, name)| { Some(Series { name: Some(format!("\"{}\"", name.replace('"', "\"\""))), @@ -106,3 +111,22 @@ pub fn clear_output(wb: &mut Workbook, sheet: usize, idx: usize) { p.last_range = None; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pivot_chart_gets_at_most_the_series_the_renderer_reads() { + let out = Output { + grid: vec![], + row_kinds: vec![], + body_offset: 0, + width: 1000, + chart_rows: vec![1, 2, 3], + chart_cols: (1..1000).map(|c| (c, format!("S{c}"))).collect(), + label_col: Some(0), + }; + assert_eq!(chart_series("Sheet1", CellRef::new(0, 0), &out).len(), MAX_CHART_SERIES); + } +} diff --git a/crates/engine/src/pivot/compute.rs b/crates/engine/src/pivot/compute.rs index 0ffa8cae5..f2d417b40 100644 --- a/crates/engine/src/pivot/compute.rs +++ b/crates/engine/src/pivot/compute.rs @@ -1,5 +1,7 @@ use super::items::{Acc, FieldItems, build_tree, collect_siblings, is_collapsed, item_label}; -use super::output::{AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, data_cols_exceed_sheet, flatten_cols}; +use super::output::{ + AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, caption, data_cols_exceed_sheet, flatten_cols, +}; use super::{ CalcSet, CellError, CellRef, HashMap, HashSet, MAX_AXIS_FIELDS, MAX_COLS, MAX_VALUE_FIELDS, PivotDateGroup, PivotField, PivotFilter, PivotFunc, PivotLayout, PivotShowAs, PivotTable, PivotValue, Source, Value, Workbook, read_source, @@ -92,6 +94,10 @@ struct AggregateData { col_leaves: HashSet>, } +/// Most accumulators (groups times value columns) one report may hold: twice what the 5 million +/// group limit allowed at one value field. +const MAX_ACCUMULATORS: usize = 16_000_000; + fn aggregate_record( input: &AggregateInput<'_>, data: &mut AggregateData, @@ -150,7 +156,9 @@ fn aggregate_record( if !input.cols.is_empty() && data_cols_exceed_sheet(data.col_leaves.len(), input.pt.grand_totals_rows, input.vals.len()) { return Err("The PivotTable report will not fit on the sheet.".into()); } - if data.aggs.len() > 5_000_000 { + // Each entry holds one accumulator per value field and calculated-field source column, so the + // width counts too: a handful of groups over hundreds of fields is large as well. + if data.aggs.len() > 5_000_000 || data.aggs.len().saturating_mul(input.vals.len() + input.extra.len()) > MAX_ACCUMULATORS { return Err("The PivotTable report is too large.".into()); } Ok(()) @@ -257,8 +265,8 @@ fn write_headers(g: &mut Grid, input: HeaderInput<'_>) -> Result<(), String> { let lv = spec.path.len().saturating_sub(1); let label = spec.path.last().and_then(|it| col_items.get(lv).map(|fi| fi.label(*it))).unwrap_or_default(); let text = match vi { - Some(vi) if k > 1 => format!("{label} {}", val_name(*vi)), - _ => format!("{label} Total"), + Some(vi) if k > 1 => caption(&label, &val_name(*vi)), + _ => caption(&label, "Total"), }; g.set(first_level + lv, c, Value::text(text), CellKind::ColLabel, 0, None); } @@ -316,6 +324,7 @@ struct ShowContext<'a> { value_index: usize, level0: &'a [u32], siblings: &'a HashMap, Vec>, + cache: &'a ShowCache, } fn number(value: &Option) -> Option { @@ -332,42 +341,65 @@ fn ratio(a: Value, b: Option) -> Value { } } +/// What Running Total and Rank need from other cells, computed once per sibling set instead of +/// once per displayed value (which rescanned the whole set every time). +type ShowKey = (Vec, Vec, usize); +/// Cumulative values by top-level item, and the total of all of them. +type RunningTotals = (HashMap, f64); + +#[derive(Default)] +struct ShowCache { + /// Ranked values' sibling numbers, sorted ascending without repeats, by (parent, column, value). + ranks: std::cell::RefCell>>, + /// Running totals by top-level item, with the total of all of them, by (rest of path, column, value). + running: std::cell::RefCell>, +} + fn running_total(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { - let Some((first, _)) = context.row_path.split_first() else { return raw }; + let Some((first, rest)) = context.row_path.split_first() else { return raw }; if !matches!(raw, Value::Number(_)) { return raw; } - let mut total = 0.0; - for item in context.level0 { - let mut path = context.row_path.to_vec(); - if let Some(first_item) = path.first_mut() { - *first_item = *item; + let mut cache = context.cache.running.borrow_mut(); + let (by_item, all) = cache.entry((rest.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut total = 0.0; + let mut by_item = HashMap::new(); + for item in context.level0 { + let mut path = Vec::with_capacity(rest.len() + 1); + path.push(*item); + path.extend_from_slice(rest); + total += number(&lookup.get(&path, context.col_path, context.value_index)).unwrap_or(0.0); + by_item.entry(*item).or_insert(total); } - total += number(&lookup.get(&path, context.col_path, context.value_index)).unwrap_or(0.0); - if item == first { - break; - } - } - Value::number(total) + (by_item, total) + }); + Value::number(by_item.get(first).copied().unwrap_or(*all)) } fn ranked_value(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { let Some((_, parent)) = context.row_path.split_last() else { return Value::Empty }; let Some(me) = raw.as_f64() else { return raw }; - let siblings = context.siblings.get(parent).cloned().unwrap_or_default(); + let mut cache = context.cache.ranks.borrow_mut(); + let sorted = cache.entry((parent.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut all: Vec = context + .siblings + .get(parent) + .into_iter() + .flatten() + .filter_map(|sibling| { + let mut path = parent.to_vec(); + path.push(*sibling); + number(&lookup.get(&path, context.col_path, context.value_index)) + }) + .collect(); + all.sort_by(f64::total_cmp); + all.dedup(); + all + }); // Excel's ranks are dense: ties share a rank and the next value takes the next one // (5, 4, 3, 3, 4 rank 1, 2, 3, 3, 2), so count the distinct larger values. - let mut larger: Vec = siblings - .into_iter() - .filter_map(|sibling| { - let mut path = parent.to_vec(); - path.push(sibling); - number(&lookup.get(&path, context.col_path, context.value_index)).filter(|value| *value > me) - }) - .collect(); - larger.sort_by(f64::total_cmp); - larger.dedup(); - Value::number(larger.len() as f64 + 1.0) + let larger = sorted.len() - sorted.partition_point(|value| *value <= me); + Value::number(larger as f64 + 1.0) } fn show_value(definition: &PivotValue, raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { @@ -637,6 +669,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro }) .collect(); let lookup = ValueLookup { wb, source: src, calc: &calc, vals: &vals, extra: &extra, aggs: &aggs, value_count: k }; + let show_cache = ShowCache::default(); let mut chart_rows: Vec = Vec::new(); let chart_cols: Vec<(u32, String)> = data_cols .iter() @@ -694,7 +727,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro let Some((_, vdef)) = vals.get(*vi) else { continue }; let cp = spec.path.as_slice(); let Some(raw) = lookup.get(rp, cp, *vi) else { continue }; - let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings }; + let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings, cache: &show_cache }; let shown = show_value(vdef, raw, &lookup, &context); if let Some(pr) = probe.as_deref_mut() { if pr.at == Some((r, l + j)) { diff --git a/crates/engine/src/pivot/output.rs b/crates/engine/src/pivot/output.rs index c3d6ee88d..ca680335b 100644 --- a/crates/engine/src/pivot/output.rs +++ b/crates/engine/src/pivot/output.rs @@ -121,6 +121,18 @@ pub(super) enum ColKind { Grand, } +/// Longest item caption Excel keeps; a subtotal caption repeats its parent's label for every value +/// field at every level, so a long source label is cut rather than multiplied. +const MAX_CAPTION_LABEL_CHARS: usize = 255; + +/// `label` (cut to [`MAX_CAPTION_LABEL_CHARS`]) and `suffix`, as in "East Total". +pub(super) fn caption(label: &str, suffix: &str) -> String { + let mut out: String = label.chars().take(MAX_CAPTION_LABEL_CHARS).collect(); + out.push(' '); + out.push_str(suffix); + out +} + pub(super) struct ColSpec { pub(super) kind: ColKind, pub(super) path: Vec, @@ -162,7 +174,7 @@ impl Ctx<'_> { leaf: false, path: prefix.to_vec(), values: true, - labels: vec![(col, Value::text(format!("{label} {name}")), indent, None)], + labels: vec![(col, Value::text(caption(label, name)), indent, None)], vi: Some(vi), }); } @@ -249,7 +261,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), indent, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), indent, None)], vi, }); } @@ -316,7 +328,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), 0, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), 0, None)], vi, }); } @@ -438,3 +450,17 @@ pub(super) fn cell_style(lk: &Look, rk: RowKind, cell: &OutCell, last_header: bo } s } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn captions_cut_a_long_parent_label() { + assert_eq!(caption("East", "Total"), "East Total"); + let long = "é".repeat(100_000); + let c = caption(&long, "Sum of Sales"); + assert_eq!(c.chars().count(), MAX_CAPTION_LABEL_CHARS + " Sum of Sales".len()); + assert!(c.ends_with(" Sum of Sales")); + } +}