diff --git a/crates/xlsx/src/csv.rs b/crates/xlsx/src/csv.rs index b6221a0c4..315bc961a 100644 --- a/crates/xlsx/src/csv.rs +++ b/crates/xlsx/src/csv.rs @@ -42,32 +42,42 @@ pub fn decode_text(bytes: &[u8]) -> String { } /// Picks comma, semicolon or tab: the candidate with the most consistent count per line -/// (outside quotes) over the first lines. +/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins. pub fn sniff_delimiter(text: &str, quote: char) -> u8 { let mut best = (b',', 0usize, 0usize); for d in [',', ';', '\t'] { let mut counts = vec![]; let mut n = 0usize; let mut in_q = false; + let mut blank = true; for ch in text.chars().take(64 * 1024) { + if !in_q && ch == '\n' { + // Blank lines carry no delimiter evidence. + if !blank { + counts.push(n); + } + n = 0; + blank = true; + if counts.len() >= 20 { + break; + } + continue; + } + if ch != '\r' { + blank = false; + } if ch == quote { in_q = !in_q; } else if !in_q && ch == d { n += 1; - } else if !in_q && ch == '\n' { - counts.push(n); - n = 0; - if counts.len() >= 20 { - break; - } } } - if n > 0 { + if !blank { counts.push(n); } let min = counts.iter().copied().min().unwrap_or(0); let total: usize = counts.iter().sum(); - if (min, total) > (best.1, best.2) { + if min > 0 && (min, total) > (best.1, best.2) { best = (d as u8, min, total); } } diff --git a/crates/xlsx/src/tests/csv_tests.rs b/crates/xlsx/src/tests/csv_tests.rs index 451801fbd..12036e39e 100644 --- a/crates/xlsx/src/tests/csv_tests.rs +++ b/crates/xlsx/src/tests/csv_tests.rs @@ -74,6 +74,10 @@ fn delimiter_sniffing() { assert_eq!(sniff_delimiter("a\tb\n1\t2\n", '"'), b'\t'); assert_eq!(sniff_delimiter("a,b\n\"x;y;z\",2\n", '"'), b','); assert_eq!(sniff_delimiter("single", '"'), b','); + // A delimiter missing from some lines is text, not a separator (single-column export round trip). + assert_eq!(sniff_delimiter("alpha;beta\r\ngamma\r\n", '"'), b','); + assert_eq!(sniff_delimiter("alpha\tbeta\r\ngamma\r\n", '"'), b','); + assert_eq!(sniff_delimiter("a;b\r\n\r\nc;d\r\n", '"'), b';'); let wb = read_csv(b"x;y\n1,5;2", &CsvOptions { delimiter: 0, ..Default::default() }).unwrap(); assert_eq!(v(&wb, 1, 1), Value::Number(2.0)); assert!(read_csv(b"a", &CsvOptions { delimiter: b'"', ..Default::default() }).is_err());