engine: PivotTable filters test each record against a set of selected labels (#3279)

Every source record scanned the filter's whole selection, so a large
selection multiplied by many records was slow even when nothing passed.
Each filter's labels are lowercased into a set once per computation.

crates/chart/src/render/cartesian.rs

@@ -96,11 +96,8 @@ fn series_spans(data: &ChartData, fams: &[Fam], sec: &[bool], ncat: usize) -> Ve
         } else {
             None
         };
-        // A series without values has no spans: 255 empty series over 20,000 categories would
-        // otherwise fill ~5 million entries.
-        let n = if s.values.is_empty() { 0 } else { ncat };
-        let mut row = Vec::with_capacity(n);
-        for j in 0..n {
+        let mut row = Vec::with_capacity(ncat);
+        for j in 0..ncat {
             let v = s.values.get(j).copied().flatten();
             let span = match (si.and_then(|p| stacks.get_mut(p)), fam) {
                 (Some((_, pos, neg)), Fam::ColStack | Fam::Col100) => v.and_then(|v| {
@@ -453,18 +450,3 @@ pub(super) fn scatter(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx)
     }
     draw_trend_labels(ctx, plot, &trend_labels);
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn empty_series_get_no_span_entries() {
-        let empty = SeriesData::default();
-        let full = SeriesData { values: vec![Some(1.0); 3], ..SeriesData::default() };
-        let data = ChartData { series: vec![empty.clone(), full, empty], ..ChartData::default() };
-        let fams = [Fam::Col; 3];
-        let spans = series_spans(&data, &fams, &[false; 3], MAX_CATS);
-        assert_eq!(spans.iter().map(Vec::len).collect::<Vec<_>>(), [0, MAX_CATS, 0]);
-    }
-}

crates/engine/src/cmd/pivot/tests.rs

@@ -168,6 +168,19 @@ fn filters_and_report_filter_rows() {
     assert!(s.execute("pivot.filter", json!({"field": "Product", "selected": []})).is_err());
 }
 
+/// A selection of the most labels allowed (100,000) is looked up once per record, not scanned: the
+/// report still filters case-insensitively.
+#[test]
+fn a_huge_filter_selection_is_matched_through_a_set() {
+    let mut s = sales();
+    s.execute("insert.pivotTable", json!({"rows": ["Product"], "values": ["Sales"], "filters": ["Region"]})).unwrap();
+    let mut selected: Vec<String> = (0..99_999).map(|i| format!("Region {i}")).collect();
+    selected.push("EAST".into());
+    s.execute("pivot.filter", json!({"field": "Region", "selected": selected})).unwrap();
+    let gt = find_row(&s, "Grand Total").unwrap();
+    assert_eq!(val(&s, &CellRef::new(gt, 1).a1()), Value::Number(415.0));
+}
+
 #[test]
 fn show_values_as() {
     let mut s = sales();

crates/engine/src/pivot/compute.rs

@@ -77,6 +77,8 @@ struct AggregateInput<'a> {
     extra: &'a [usize],
     filters: &'a [(usize, &'a PivotFilter, PivotDateGroup, bool)],
     filter_fmts: &'a [String],
+    /// Each filter's selected labels, ASCII-lowercased, so a record's test is one probe.
+    filter_sets: &'a [Option<HashSet<String>>],
     row_fmts: &'a [String],
     col_fmts: &'a [String],
     empty: bool,
@@ -99,10 +101,10 @@ fn aggregate_record(
     ri: usize,
     rec: &[Value],
 ) -> Result<(), String> {
-    for ((c, field, group, _), fmt) in input.filters.iter().zip(input.filter_fmts) {
-        if let Some(selected) = &field.selected {
+    for (((c, _, group, _), fmt), selected) in input.filters.iter().zip(input.filter_fmts).zip(input.filter_sets) {
+        if let Some(selected) = selected {
             let label = item_label(input.wb, rec.get(*c).unwrap_or(&Value::Empty), *group, fmt);
-            if !selected.iter().any(|value| value.eq_ignore_ascii_case(&label)) {
+            if !selected.contains(&label.to_ascii_lowercase()) {
                 return Ok(());
             }
         }
@@ -475,6 +477,8 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro
 
     // ---- aggregate
     let filter_fmts: Vec<String> = filters.iter().map(|f| fmt_of(f.0)).collect();
+    let filter_sets: Vec<Option<HashSet<String>>> =
+        filters.iter().map(|(_, field, _, _)| field.selected.as_ref().map(|s| s.iter().map(|v| v.to_ascii_lowercase()).collect())).collect();
     let row_fmts: Vec<String> = rows.iter().map(|(c, _)| fmt_of(*c)).collect();
     let col_fmts: Vec<String> = cols.iter().map(|(c, _)| fmt_of(*c)).collect();
     let aggregate = aggregate_source(
@@ -488,6 +492,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro
             extra: &extra,
             filters: &filters,
             filter_fmts: &filter_fmts,
+            filter_sets: &filter_sets,
             row_fmts: &row_fmts,
             col_fmts: &col_fmts,
             empty,