Mail merge: cap the cells a recipient list keeps, not just its records and fields

Every record is padded to every field and the list is kept twice (as read,
and filtered and sorted), so a short CSV with a 1,000-field header and
100,000 one-field rows held 2 x 100 million empty cells, several gigabytes.

A list now keeps at most 4 million cells: wide lists keep fewer records (with
a warning), narrow ones keep the usual 100,000.

crates/engine/src/cmd/mailings/recipients.rs

@@ -72,6 +72,25 @@ const MAX_RECORDS: usize = 100_000;
 
 const MAX_FIELDS: usize = 1_000;
 
+/// Most cells (records × fields) a recipient list keeps. Every record is padded to every field
+/// and the list is kept twice (as read, and filtered and sorted), so a short file with a wide
+/// header and many one-field rows would otherwise claim gigabytes of empty cells.
+const MAX_CELLS: usize = 4_000_000;
+
+/// Most records a list with `fields` fields keeps: [`MAX_RECORDS`], fewer for a wide list.
+fn max_records(fields: usize) -> usize {
+    MAX_RECORDS.min(MAX_CELLS / fields.max(1))
+}
+
+/// Cut `rows` to what a list with `fields` fields keeps, saying so when it drops any.
+fn cap_records<T>(rows: &mut Vec<T>, fields: usize) {
+    let keep = max_records(fields);
+    if rows.len() > keep {
+        log::warn!("the recipient list keeps at most {keep} records with {fields} fields; the rest are left out");
+        rows.truncate(keep);
+    }
+}
+
 pub(super) fn cell(x: &Value) -> String {
     match x {
         Value::Null => String::new(),
@@ -102,7 +121,7 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
     };
     let rows = arr
         .iter()
-        .take(MAX_RECORDS)
+        .take(max_records(headers.len()))
         .filter_map(|r| match r {
             Value::Array(cells) => Some((0..headers.len()).map(|i| cells.get(i).map(cell).unwrap_or_default()).collect()),
             Value::Object(o) => Some(headers.iter().map(|h| o.get(h).map(cell).unwrap_or_default()).collect()),
@@ -223,7 +242,8 @@ fn read_file(path: &str) -> Result<Vec<u8>, CmdError> {
 
 /// Field names and records from a table's rows: with `headers`, the first row names the fields
 /// (blank names become `Column N`, repeats are numbered); otherwise every row is a record and
-/// the fields are `Column 1`…`Column N`. Records are padded or cut to the fields, and capped.
+/// the fields are `Column 1`…`Column N`. Records are padded or cut to the fields, and capped
+/// ([`max_records`]).
 fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>, Vec<Vec<String>>), CmdError> {
     if all.is_empty() {
         return Err(CmdError::Failed("the recipient list is empty".into()));
@@ -235,7 +255,7 @@ fn header_split(mut all: Vec<Vec<String>>, headers: bool) -> Result<(Vec<String>
         let width = all.iter().map(Vec::len).max().unwrap_or(0).min(MAX_FIELDS);
         (1..=width).map(|i| format!("Column {i}")).collect()
     };
-    all.truncate(MAX_RECORDS);
+    cap_records(&mut all, fields.len());
     for r in &mut all {
         r.resize(fields.len(), String::new());
     }

crates/engine/src/cmd/mailings/tests.rs

@@ -695,3 +695,21 @@ fn merge_uses_only_filtered_records_in_sort_order() {
     assert!(s.run("mailings.finish", &json!({})).is_err());
     assert!(s.run("mailings.editRecipients", &json!({})).is_ok());
 }
+
+/// A wide header and many one-field rows: every record is padded to every field (and the list
+/// kept twice), so the cells the list keeps are capped, not just its records and fields.
+#[test]
+fn a_wide_sparse_recipient_list_is_capped_by_cells() {
+    let header = (1..=1_000).map(|i| format!("F{i}")).collect::<Vec<_>>().join(",");
+    let csv = format!("{header}\n{}", "x\n".repeat(20_000));
+    let mut s = Session::new(Document::from_text("Hello"));
+    let r = s.run("mailings.recipients", &json!({"csv": csv})).unwrap();
+    assert_eq!(r["records"], 4_000);
+    let rows: Vec<Value> = (0..20_000).map(|_| json!({})).collect();
+    let fields: Vec<String> = (1..=1_000).map(|i| format!("F{i}")).collect();
+    let r = s.run("mailings.recipients", &json!({"rows": rows, "fields": fields})).unwrap();
+    assert_eq!(r["records"], 4_000);
+    // A narrow list keeps its usual number of records.
+    let r = s.run("mailings.recipients", &json!({"csv": format!("Name\n{}", "Ann\n".repeat(20_000))})).unwrap();
+    assert_eq!(r["records"], 20_000);
+}

crates/layout/src/para/lines.rs

@@ -90,8 +90,6 @@ struct Scan {
     j: usize,
     /// Tab cluster, stop, and x where the tab started.
     pending_tab: Option<(usize, TabStop, f32)>,
-    /// The text after a pending decimal tab looked at so far for its '.'.
-    decimal: Option<DecimalScan>,
     /// The width of the spaces so far on this line.
     space_w: f32,
     has_tab: bool,
@@ -113,15 +111,17 @@ fn takes_room(c: &Cluster) -> bool {
     c.kind != ClKind::Marker && !(matches!(c.kind, ClKind::Object(_)) && c.adv == 0.0 && c.obj_h == 0.0)
 }
 
-/// How far the text after a decimal tab has been looked at for its first '.' (see
-/// [`Scan::width_to_decimal`]): the tab, the next cluster to look at, the width before it, and
-/// whether that cluster is the '.'.
-#[derive(Clone, Copy, Default)]
-struct DecimalScan {
-    tab: usize,
-    next: usize,
-    width: f32,
-    found: bool,
+/// The width up to the first '.' of the text after the tab at `tj`, up to cluster `j`.
+fn width_to_decimal(pl: &ParaLayout, tj: usize, j: usize) -> Option<f32> {
+    let mut w = 0.0;
+    for k in tj + 1..=j {
+        let cc = pl.clusters.get(k)?;
+        if pl_text_at(cc) {
+            return Some(w);
+        }
+        w += cc.adv;
+    }
+    None
 }
 
 /// The objects' exclusions mirrored across a line of `width`, for a right-to-left paragraph.
@@ -276,7 +276,6 @@ impl<'e> Breaker<'e> {
             end: LineEnd::Para,
             j: self.i,
             pending_tab: None,
-            decimal: None,
             space_w: 0.0,
             has_tab: false,
             skip: false,
@@ -840,29 +839,6 @@ impl Scan {
         false
     }
 
-    /// The width up to the first '.' of the text after the tab at `tj`, up to cluster `j`. The
-    /// text is shifted after each of its clusters, so the clusters already looked at are not
-    /// walked again: a long run without a '.' after a decimal tab is looked at once, not once
-    /// per cluster.
-    fn width_to_decimal(&mut self, pl: &ParaLayout, tj: usize, j: usize) -> Option<f32> {
-        let start = DecimalScan { tab: tj, next: tj + 1, width: 0.0, found: false };
-        let d = self.decimal.get_or_insert(start);
-        if d.tab != tj || d.next > j.saturating_add(1) {
-            *d = start;
-        }
-        while !d.found && d.next <= j {
-            crate::work::steps(1);
-            let cc = pl.clusters.get(d.next)?;
-            if pl_text_at(cc) {
-                d.found = true;
-            } else {
-                d.width += cc.adv;
-                d.next += 1;
-            }
-        }
-        d.found.then_some(d.width)
-    }
-
     /// Text after a decimal/centre/right tab shifts as it grows.
     fn shift_pending_tab(&mut self, pl: &mut ParaLayout) {
         let Some((tj, stop, tx)) = self.pending_tab else { return };
@@ -871,7 +847,7 @@ impl Scan {
             TabAlign::Right => stop.pos - tx - seg_w,
             TabAlign::Center => stop.pos - tx - seg_w / 2.0,
             // Width up to the first '.' in the segment.
-            TabAlign::Decimal => stop.pos - tx - self.width_to_decimal(pl, tj, self.j).unwrap_or(seg_w),
+            TabAlign::Decimal => stop.pos - tx - width_to_decimal(pl, tj, self.j).unwrap_or(seg_w),
             _ => 0.0,
         };
         let shift = shift.max(0.0);

crates/layout/src/tests/linear.rs

@@ -482,25 +482,3 @@ fn selecting_a_header_on_the_hinted_page_does_not_walk_every_page() {
     eprintln!("hinted header: {small} steps for 20 pages, {big} for 160");
     assert!(small > 0 && big <= small + 8, "the pages aren't walked: {small} → {big}");
 }
-
-/// A paragraph of a decimal tab at 200 pt followed by `text`.
-fn after_decimal_tab(text: &str) -> Document {
-    use wordcraft_doc::props::{TabAlign, TabStop};
-    let mut d = Document::from_text(&format!("\t{text}"));
-    let at = Pos::body(0, 0);
-    d.format_paragraphs(&at, &at, &|p| p.tabs = Some(vec![TabStop { pos: 200.0, align: TabAlign::Decimal, leader: Default::default() }])).unwrap();
-    d
-}
-
-/// The text after a decimal tab was looked at again from the tab for its '.' after each of its
-/// clusters, so a long run of clusters without a '.' (soft hyphens take no room, so they stay on
-/// one line) took quadratic time.
-#[test]
-fn text_after_a_decimal_tab_is_looked_at_once() {
-    let opts = LayoutOptions::default();
-    // The text after the tab still ends at the stop, where it ended before.
-    let l = lay(&after_decimal_tab("12.5"), &opts);
-    let (Some(end), Some(start)) = (l.caret(&Pos::body(0, 5)), l.caret(&Pos::body(0, 0))) else { panic!("no carets") };
-    assert!((end.x - start.x - 200.0).abs() < 1.0, "the text ends at the stop: {} from {}", end.x, start.x);
-    assert_layout_linear("decimal tab", 2000, |n| after_decimal_tab(&format!("1{}", "\u{AD}".repeat(n))), &opts);
-}