mirror of
https://github.com/storytold/gridcraft.git
synced 2026-10-27 20:26:48 -06:00
Keep comma when a semicolon or tab is missing from some CSV lines (#127)
This commit is contained in:
+19
-9
@@ -42,32 +42,42 @@ pub fn decode_text(bytes: &[u8]) -> String {
|
||||
}
|
||||
|
||||
/// Picks comma, semicolon or tab: the candidate with the most consistent count per line
|
||||
/// (outside quotes) over the first lines.
|
||||
/// (outside quotes) over the first non-blank lines. A candidate must appear on every such line, otherwise comma wins.
|
||||
pub fn sniff_delimiter(text: &str, quote: char) -> u8 {
|
||||
let mut best = (b',', 0usize, 0usize);
|
||||
for d in [',', ';', '\t'] {
|
||||
let mut counts = vec![];
|
||||
let mut n = 0usize;
|
||||
let mut in_q = false;
|
||||
let mut blank = true;
|
||||
for ch in text.chars().take(64 * 1024) {
|
||||
if !in_q && ch == '\n' {
|
||||
// Blank lines carry no delimiter evidence.
|
||||
if !blank {
|
||||
counts.push(n);
|
||||
}
|
||||
n = 0;
|
||||
blank = true;
|
||||
if counts.len() >= 20 {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ch != '\r' {
|
||||
blank = false;
|
||||
}
|
||||
if ch == quote {
|
||||
in_q = !in_q;
|
||||
} else if !in_q && ch == d {
|
||||
n += 1;
|
||||
} else if !in_q && ch == '\n' {
|
||||
counts.push(n);
|
||||
n = 0;
|
||||
if counts.len() >= 20 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if n > 0 {
|
||||
if !blank {
|
||||
counts.push(n);
|
||||
}
|
||||
let min = counts.iter().copied().min().unwrap_or(0);
|
||||
let total: usize = counts.iter().sum();
|
||||
if (min, total) > (best.1, best.2) {
|
||||
if min > 0 && (min, total) > (best.1, best.2) {
|
||||
best = (d as u8, min, total);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +74,10 @@ fn delimiter_sniffing() {
|
||||
assert_eq!(sniff_delimiter("a\tb\n1\t2\n", '"'), b'\t');
|
||||
assert_eq!(sniff_delimiter("a,b\n\"x;y;z\",2\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("single", '"'), b',');
|
||||
// A delimiter missing from some lines is text, not a separator (single-column export round trip).
|
||||
assert_eq!(sniff_delimiter("alpha;beta\r\ngamma\r\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("alpha\tbeta\r\ngamma\r\n", '"'), b',');
|
||||
assert_eq!(sniff_delimiter("a;b\r\n\r\nc;d\r\n", '"'), b';');
|
||||
let wb = read_csv(b"x;y\n1,5;2", &CsvOptions { delimiter: 0, ..Default::default() }).unwrap();
|
||||
assert_eq!(v(&wb, 1, 1), Value::Number(2.0));
|
||||
assert!(read_csv(b"a", &CsvOptions { delimiter: b'"', ..Default::default() }).is_err());
|
||||
|
||||
Reference in New Issue
Block a user