Mailings: detect CSV delimiters across multiline quoted headers (#4249)

Lands #4249.

Co-Authored-By: OPUZZLES <109076338+OPUZZLES@users.noreply.github.com>

crates/engine/src/cmd/mailings/recipients.rs

@@ -1,16 +1,18 @@
 use super::filter::refresh_shown;
 use super::{CmdError, CmdResult, Session, Value, json, p};
 
-/// Parse CSV (RFC 4180-ish: quotes, doubled quotes, commas/semicolons/tabs).
-/// The field separator: whichever of `,` `;` and tab the first non-blank line (outside quotes) uses most.
+/// The field separator: whichever of `,` `;` and tab the first non-blank record (outside quotes) uses most.
 fn csv_separator(text: &str) -> char {
-    let first = text.lines().find(|l| !l.trim().is_empty()).unwrap_or("");
     let mut quoted = false;
     let mut unquoted = String::new();
-    for c in first.chars() {
+    // A quoted header may span physical lines; infer from the first complete record.
+    for c in text.split_inclusive(['\n', '\r']).skip_while(|line| line.trim().is_empty()).flat_map(str::chars) {
         if c == '"' {
             quoted = !quoted;
         } else if !quoted {
+            if c == '\n' || c == '\r' {
+                break;
+            }
             unquoted.push(c);
         }
     }
@@ -26,6 +28,7 @@ fn push_csv_row(rows: &mut Vec<Vec<String>>, row: &mut Vec<String>) {
     }
 }
 
+/// Parse CSV (RFC 4180-ish: quotes, doubled quotes, commas/semicolons/tabs).
 pub fn parse_csv(text: &str) -> Vec<Vec<String>> {
     let sep = csv_separator(text);
     let mut rows = Vec::new();
@@ -112,13 +115,23 @@ fn table_from_rows(fields: Option<&Vec<Value>>, arr: &[Value]) -> Result<(Vec<St
     Ok((headers, rows))
 }
 
+/// Skip unquoted blank prefix lines only when a text recipient list has a header.
+/// Headerless lists and the shared CSV reader keep whitespace records.
+fn recipient_csv(text: &str, headers: bool) -> Vec<Vec<String>> {
+    let prefix = text
+        .split_inclusive(['\n', '\r'])
+        .take_while(|line| headers && line.trim().is_empty())
+        .fold(0_usize, |bytes, line| bytes.saturating_add(line.len()));
+    parse_csv(text.get(prefix..).unwrap_or_default())
+}
+
 pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
     let mut sheet = Value::Null;
     let (headers, rows) = if let Some(arr) = v.get("rows").and_then(Value::as_array) {
         table_from_rows(v.get("fields").and_then(Value::as_array), arr)?
     } else {
         let all = if let Some(c) = p::str(v, "csv") {
-            parse_csv(c)
+            recipient_csv(c, p::bool(v, "headers").unwrap_or(true))
         } else {
             let (bytes, ext) = if let Some(d) = p::str(v, "data") {
                 let b = crate::cmd::insert::base64_decode(d.trim()).ok_or_else(|| CmdError::Params("`data` isn't valid base64".into()))?;
@@ -175,7 +188,7 @@ pub(super) fn recipients(s: &mut Session, v: &Value) -> CmdResult {
                 sheet = json!(chosen);
                 rows
             } else {
-                parse_csv(&decode(&bytes, v)?)
+                recipient_csv(&decode(&bytes, v)?, p::bool(v, "headers").unwrap_or(true))
             }
         };
         header_split(all, p::bool(v, "headers").unwrap_or(true))?

crates/engine/src/cmd/mailings/tests.rs

@@ -46,6 +46,76 @@ fn finishing_a_merge_to_a_file_is_not_an_edit() {
     let _ = std::fs::remove_dir_all(&dir);
 }
 
+#[test]
+fn csv_delimiter_reads_complete_header_record() {
+    for sep in [',', ';', '\t'] {
+        for newline in ["\n", "\r\n", "\r"] {
+            // All candidate separators and doubled quotes occur on both sides of the newline.
+            // The real separators follow the closing quote, beyond the first physical line.
+            let header = format!("Name,;\t \"given\"{newline}family,;\t \"last\"");
+            let quoted = header.replace('"', "\"\"");
+            let text = format!("\"{quoted}\"{sep}City{sep}{sep}\r\nAda{sep}Paris{sep}{sep}\nBo{sep}Oslo{sep}{sep}");
+            assert_eq!(
+                parse_csv(&text),
+                vec![vec![header.as_str(), "City", "", ""], vec!["Ada", "Paris", "", ""], vec!["Bo", "Oslo", "", ""]],
+                "separator {sep:?}, embedded newline {newline:?}"
+            );
+            // A multiline field after an ordinary column must not end detection either.
+            let text = format!("ID{sep}\"{quoted}\"{sep}City\n1{sep}Ada{sep}Paris\n2{sep}Bo{sep}Oslo");
+            assert_eq!(parse_csv(&text), vec![vec!["ID", header.as_str(), "City"], vec!["1", "Ada", "Paris"], vec!["2", "Bo", "Oslo"]]);
+        }
+    }
+}
+
+#[test]
+fn csv_delimiter_reads_multiline_headers_after_blank_lines() {
+    for sep in [',', ';', '\t'] {
+        for newline in ["\n", "\r\n", "\r"] {
+            let header = format!("Given{newline}Family");
+            for prefix in ["\n\r\n", "\r", "\r\r"] {
+                let text = format!("{prefix}\"{header}\"{sep}City\rAda{sep}Paris");
+                assert_eq!(parse_csv(&text), vec![vec![header.as_str(), "City"], vec!["Ada", "Paris"]]);
+            }
+        }
+    }
+}
+
+#[test]
+fn recipient_csv_blank_prefix_keeps_columns_without_dropping_literal_records() {
+    let mut s = Session::new(Document::new());
+    for sep in [',', ';', '\t'] {
+        for newline in ["\n", "\r\n", "\r"] {
+            let header = format!("First{newline}Name");
+            let csv = format!(" \t{newline}\"{header}\"{sep}Email{newline}Ada{sep}ada@example.com");
+            let result = s.run("mailings.recipients", &json!({"csv": csv})).unwrap();
+            assert_eq!(result, json!({"fields": [header, "Email"], "records": 1}));
+            assert_eq!(s.run("mailings.editRecipients", &json!({})).unwrap()["rows"], json!([["Ada", "ada@example.com"]]));
+        }
+    }
+    s.run("mailings.recipients", &json!({"csv": " \nName,Email\nAda,ada@example.com", "headers": false})).unwrap();
+    assert_eq!(s.run("mailings.editRecipients", &json!({})).unwrap()["rows"], json!([[" ", ""], ["Name", "Email"], ["Ada", "ada@example.com"]]));
+    s.run("mailings.recipients", &json!({"csv": "\" \",Email\nAda,ada@example.com"})).unwrap();
+    assert_eq!(s.run("mailings.editRecipients", &json!({})).unwrap()["rows"], json!([["Ada", "ada@example.com"]]));
+    let rows = s.merge.rows.clone();
+    assert!(s.run("mailings.recipients", &json!({"csv": " \t\r\n \r"})).is_err());
+    assert_eq!(s.merge.rows, rows, "an empty headered list must not replace existing recipients");
+    assert_eq!(parse_csv(" \nName,Email\nAda,ada@example.com")[0], vec![" "], "the shared CSV reader keeps literal whitespace rows");
+}
+
+#[test]
+fn csv_delimiter_keeps_record_boundaries_and_ties() {
+    // Later records cannot choose the header's delimiter.
+    assert_eq!(parse_csv("Name;City\rAda;Paris,,,,\rBo;Oslo"), vec![vec!["Name", "City"], vec!["Ada", "Paris,,,,"], vec!["Bo", "Oslo"]]);
+    // Existing ties prefer tab, then semicolon, then comma.
+    assert_eq!(parse_csv("A,B;C\n1,2;3"), vec![vec!["A,B", "C"], vec!["1,2", "3"]]);
+    assert_eq!(parse_csv("A;B\tC\n1;2\t3"), vec![vec!["A;B", "C"], vec!["1;2", "3"]]);
+    assert_eq!(parse_csv("Name\nAda,Paris;Oslo"), vec![vec!["Name"], vec!["Ada,Paris;Oslo"]]);
+    assert!(parse_csv("").is_empty());
+    assert_eq!(parse_csv("\nName\nAda"), vec![vec!["Name"], vec!["Ada"]]);
+    // The reader remains permissive about an unterminated quoted cell.
+    assert_eq!(parse_csv("Name,City\nAda,\"Paris\nOslo"), vec![vec!["Name", "City"], vec!["Ada", "Paris\nOslo"]]);
+}
+
 #[test]
 fn csv_delimiter_ignores_quoted_text() {
     let r = parse_csv("\"Name, first, last\";City\nAnn;Paris\n");