diff --git a/crates/xlsx/src/csv.rs b/crates/xlsx/src/csv.rs index 47486bfa2..0e0e54c28 100644 --- a/crates/xlsx/src/csv.rs +++ b/crates/xlsx/src/csv.rs @@ -17,7 +17,107 @@ const CP1252: [u32; 32] = [ 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x9D, 0x017E, 0x0178, ]; -/// Decodes CSV bytes: UTF-8/UTF-16 BOMs, else UTF-8, else Windows-1252. +/// Windows-1256 code points for bytes 0x80–0xFF (unicode.org `CP1256.TXT`). +pub(crate) const CP1256: [u32; 128] = [ + 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, + 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, + 0x200F, 0x06D2, +]; + +/// Evidence that bytes which aren't UTF-8 are Windows-1256 Arabic rather than Windows-1252. +#[derive(Default)] +struct Cp1256Evidence { + /// High bytes that decode to different characters in the two code pages, apart from invisible + /// marks and ordinals after a digit. + differing: usize, + /// Of those, Arabic letters and Arabic punctuation (، ؛ ؟) in Windows-1256. + arabic: usize, + /// Arabic bytes next to an ASCII letter: Western accents sit inside Latin words, Arabic doesn't. + arabic_touching_latin: usize, + /// Accented Latin bytes the two code pages share, next to another high byte: Hebrew + /// Windows-1255 letters land there, Arabic and Western text rarely do. + shared_letters_in_runs: usize, +} + +impl Cp1256Evidence { + fn of(bytes: &[u8]) -> Self { + let mut e = Self::default(); + for (i, &b) in bytes.iter().enumerate() { + if b < 0x80 { + continue; + } + let before = i.checked_sub(1).and_then(|j| bytes.get(j)).copied(); + let after = bytes.get(i + 1).copied(); + let next_to = |pred: fn(u8) -> bool| before.is_some_and(pred) || after.is_some_and(pred); + if same_in_cp1252_and_cp1256(b) { + if is_shared_latin_letter(b) && next_to(|x| x >= 0x80) { + e.shared_letters_in_runs += 1; + } + continue; + } + if is_invisible_mark(b) || is_ordinal_after_digit(b, before) { + continue; + } + e.differing += 1; + if is_cp1256_arabic(b) { + e.arabic += 1; + if next_to(|x| x.is_ascii_alphabetic()) { + e.arabic_touching_latin += 1; + } + } + } + e + } + + /// Thresholds chosen against month, day and language names from 95 glibc locales (Western in + /// Windows-1252, Arabic, Persian and Urdu in 1256, Hebrew and Yiddish in 1255) plus CSV samples. + fn is_arabic(&self) -> bool { + self.arabic >= 2 + && self.arabic * 10 >= self.differing * 9 + && self.arabic_touching_latin * 10 <= self.arabic * 3 + && self.shared_letters_in_runs * 20 <= self.arabic + } +} + +fn same_in_cp1252_and_cp1256(b: u8) -> bool { + matches!( + b, + 0x80 | 0x82..=0x89 | 0x8B..=0x8C | 0x91..=0x97 | 0x99 | 0x9B..=0x9C | 0xA0 | 0xA2..=0xA9 | 0xAB..=0xB9 | 0xBB..=0xBE | 0xD7 | 0xE0 | 0xE2 | 0xE7..=0xEB + | 0xEE..=0xEF | 0xF4 | 0xF7 | 0xF9 | 0xFB..=0xFC + ) +} + +/// ZWNJ, ZWJ, LRM and RLM: common inside Persian words and around Arabic cells, but no evidence +/// either way. +fn is_invisible_mark(b: u8) -> bool { + matches!(b, 0x9D | 0x9E | 0xFD | 0xFE) +} + +/// `º` and `ª` after a digit (`1º`, `2ª`) are Western ordinals, though Windows-1256 reads the bytes +/// as `؛` and `ھ`; either way they are no evidence. +fn is_ordinal_after_digit(b: u8, before: Option) -> bool { + matches!(b, 0xAA | 0xBA) && before.is_some_and(|x| x.is_ascii_digit()) +} + +fn is_shared_latin_letter(b: u8) -> bool { + matches!(b, 0xE0 | 0xE2 | 0xE7..=0xEB | 0xEE..=0xEF | 0xF4 | 0xF9 | 0xFB..=0xFC) +} + +fn is_cp1256_arabic(b: u8) -> bool { + matches!( + b, + 0x81 | 0x8A | 0x8D..=0x90 | 0x98 | 0x9A | 0x9F | 0xA1 | 0xAA | 0xBA | 0xBF..=0xD6 | 0xD8..=0xDB | 0xDD..=0xDF | 0xE1 | 0xE3..=0xE6 | 0xEC..=0xED | 0xFF + ) +} + +/// Decodes CSV bytes with BOM-aware UTF-8 and UTF-16 handling. Without a BOM, valid UTF-8 is kept. +/// Other bytes decode as Windows-1256 when they read as Arabic (see [`Cp1256Evidence`]), else as +/// Windows-1252. pub fn decode_text(bytes: &[u8]) -> String { if let Some(rest) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) { return String::from_utf8_lossy(rest).into_owned(); @@ -34,13 +134,19 @@ pub fn decode_text(bytes: &[u8]) -> String { } match std::str::from_utf8(bytes) { Ok(s) => s.to_string(), - Err(_) => bytes - .iter() - .map(|&b| match b { - 0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'), - _ => b as char, - }) - .collect(), + Err(_) => { + let use_windows_1256 = Cp1256Evidence::of(bytes).is_arabic(); + bytes + .iter() + .map(|&b| match b { + 0x80..=0xFF if use_windows_1256 => { + char::from_u32(CP1256.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}') + } + 0x80..=0x9F => char::from_u32(CP1252.get((b - 0x80) as usize).copied().unwrap_or(0xFFFD)).unwrap_or('\u{FFFD}'), + _ => b as char, + }) + .collect() + } } } diff --git a/crates/xlsx/src/tests/csv_tests.rs b/crates/xlsx/src/tests/csv_tests.rs index cc52072e7..daaa81b80 100644 --- a/crates/xlsx/src/tests/csv_tests.rs +++ b/crates/xlsx/src/tests/csv_tests.rs @@ -68,6 +68,101 @@ fn encodings() { assert_eq!(v(&wb, 0, 1), Value::Number(1.0)); } +#[test] +fn windows_1256_table_matches_the_unicode_mapping() { + let at = |b: u8| crate::csv::CP1256.get(usize::from(b - 0x80)).and_then(|&c| char::from_u32(c)); + let expected = [ + (0x80, '€'), + (0x81, 'پ'), + (0x8D, 'چ'), + (0x98, 'ک'), + (0x9D, '\u{200C}'), + (0xA1, '،'), + (0xBA, '؛'), + (0xBF, '؟'), + (0xC7, 'ا'), + (0xC8, 'ب'), + (0xDC, 'ـ'), + (0xE1, 'ل'), + (0xE3, 'م'), + (0xE9, 'é'), + (0xFD, '\u{200E}'), + (0xFE, '\u{200F}'), + (0xFF, 'ے'), + ]; + for (b, c) in expected { + assert_eq!(at(b), Some(c), "0x{b:02X}"); + } +} + +#[test] +fn arabic_csv_in_windows_1256_reads_as_arabic() { + let bytes = [ + 0xC7, 0xE1, 0xC7, 0xD3, 0xE3, b',', 0xC7, 0xE1, 0xDE, 0xED, 0xE3, 0xC9, b'\n', 0xE3, 0xD1, 0xCD, 0xC8, 0xC7, b' ', b'1', b'2', b'3', 0xA1, + b' ', b'4', b'2', 0xBF, b',', 0xC8, 0xED, 0xC7, 0xE4, 0xC7, 0xCA, b' ', b'7', b'\n', + ]; + let wb = read_csv(&bytes, &CsvOptions::default()).unwrap(); + assert_eq!(v(&wb, 0, 0), Value::text("الاسم")); + assert_eq!(v(&wb, 0, 1), Value::text("القيمة")); + assert_eq!(v(&wb, 1, 0), Value::text("مرحبا 123، 42؟")); + assert_eq!(v(&wb, 1, 1), Value::text("بيانات 7")); +} + +fn reads_as_arabic(bytes: &[u8]) -> bool { + decode_text(bytes).chars().any(|c| ('\u{0600}'..='\u{06FF}').contains(&c)) +} + +#[test] +fn western_1252_text_stays_latin() { + let french = [ + b'C', b'r', 0xE8, b'm', b'e', b' ', b'b', b'r', 0xFB, b'l', 0xE9, b'e', b',', b' ', 0xE0, b' ', b'c', 0xF4, b't', 0xE9, b',', b' ', 0xE9, + b't', 0xE9, + ]; + let german = [ + b'G', b'r', 0xF6, 0xDF, b'e', b',', b' ', 0xDC, b'b', b'e', b'r', b'm', b'a', 0xDF, b',', b' ', b'F', b'u', 0xDF, b'g', 0xE4, b'n', b'g', + b'e', b'r', + ]; + assert_eq!(decode_text(&french), "Crème brûlée, à côté, été"); + assert_eq!(decode_text(&german), "Größe, Übermaß, Fußgänger"); + let portuguese = [ + b'A', 0xC7, 0xC3, b'O', b',', b' ', b'i', b'n', b'f', b'o', b'r', b'm', b'a', 0xE7, 0xE3, b'o', b',', b' ', b's', b'i', b't', b'u', b'a', + 0xE7, 0xE3, b'o', b',', b' ', b'P', 0xE3, b'o', + ]; + assert_eq!(decode_text(&portuguese), "AÇÃO, informação, situação, Pão"); + let nordic = [ + 0xC6, b'r', 0xF8, b',', 0xD8, b's', b't', b',', 0xC5, b's', b'e', b',', 0xDE, 0xF3, b'r', 0xF0, b'u', b'r', b',', b'G', b'u', 0xF0, b'r', + 0xFA, b'n', b',', 0xC4, 0xD6, 0xDC, + ]; + assert_eq!(decode_text(&nordic), "Ærø,Øst,Åse,Þórður,Guðrún,ÄÖÜ"); + let ordinals = [b'1', 0xBA, b',', b'2', 0xBA, b',', b'3', 0xBA, b'\n', b'1', 0xAA, b',', b'2', 0xAA]; + assert_eq!(decode_text(&ordinals), "1º,2º,3º\n1ª,2ª"); + let inside_words = [b'K', 0xE4, b's', b'e', b',', b'B', 0xE4, b'c', b'k', b'e', b'r']; + assert_eq!(decode_text(&inside_words), "Käse,Bäcker"); +} + +#[test] +fn short_arabic_words_read_as_arabic() { + assert_eq!(decode_text(&[0xDD, 0xED, b',', 0xE3, 0xE4, b',', 0xDA, 0xE4, b'\n']), "في,من,عن\n"); + assert_eq!(decode_text(&[0xE4, 0xDA, 0xE3, b',', 0xE1, 0xC7, b'\n', 0xE1, 0xC7, b',', 0xE4, 0xDA, 0xE3, b'\n']), "نعم,لا\nلا,نعم\n"); + assert_eq!(decode_text(&[0x98, 0xCA, 0xC7, 0xC8, b' ', 0x81, 0xC7, 0xD1, 0xD3, 0xED]), "کتاب پارسي"); + let with_marks = [ + 0xDF, 0xCA, 0xC7, 0xC8, 0x9D, 0xE5, 0xC7, b',', 0xCE, 0xC7, 0xE4, 0xE5, 0x9D, 0xE5, 0xC7, b',', 0xE3, 0xCD, 0xE3, 0xCF, 0xFE, b',', 0xDA, + 0xE1, 0xED, 0xFE, + ]; + assert_eq!(decode_text(&with_marks), "كتاب\u{200C}ها,خانه\u{200C}ها,محمد\u{200F},علي\u{200F}"); +} + +#[test] +fn hebrew_1255_bytes_are_not_read_as_arabic() { + let plain = [0xF9, 0xEC, 0xE5, 0xED, b',', 0xF2, 0xE5, 0xEC, 0xED]; + let pointed = [0xF9, 0xC8, 0xD1, 0xEC, 0xE5, 0xC9, 0xED, b' ', 0xF2, 0xE5, 0xC9, 0xEC, 0xC8, 0xED]; + let maqaf = [0xFA, 0xEC, 0xCE, 0xE0, 0xE1, 0xE9, 0xE1, b',', 0xE1, 0xEF, 0xCE, 0xE2, 0xE5, 0xF8, 0xE9, 0xE5, 0xEF]; + let gershayim = [0xF6, 0xE4, 0xD8, 0xEC, b',', 0xE3, 0xD8, 0xF8]; + for bytes in [&plain[..], &pointed, &maqaf, &gershayim] { + assert!(!reads_as_arabic(bytes), "{:?}", decode_text(bytes)); + } +} + #[test] fn delimiter_sniffing() { assert_eq!(sniff_delimiter("a;b;c\n1;2;3\n", '"'), b';'); diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 5a7588986..804d81441 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-19 · **Change:** minor (.xls dialog sheets are skipped instead of read as worksheets; `.xls` import reads each sheet's display options from WINDOW2: right-to-left, gridlines, headings, zeros, formulas; XLSX text boxes and shapes keep their paragraphs' right-to-left direction on save; CSV/TSV/TXT can open with every field kept as text via `file.open {csvAsText}`; formulas over Excel's 8,192 characters, and `%` or call chains past the nesting limit, are parse errors instead of stack overflows when a file is opened; XLSX PivotTables read and write `rowGrandTotals`/`colGrandTotals` the way Excel does, so a table saved with only one grand total opens with the right one; HTML export keeps right-to-left) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-19 · **Change:** minor (.xls dialog sheets are skipped instead of read as worksheets; `.xls` import reads each sheet's display options from WINDOW2: right-to-left, gridlines, headings, zeros, formulas; XLSX text boxes and shapes keep their paragraphs' right-to-left direction on save; CSV/TSV/TXT can open with every field kept as text via `file.open {csvAsText}`; formulas over Excel's 8,192 characters, and `%` or call chains past the nesting limit, are parse errors instead of stack overflows when a file is opened; XLSX PivotTables read and write `rowGrandTotals`/`colGrandTotals` the way Excel does, so a table saved with only one grand total opens with the right one; HTML export keeps right-to-left; CSV and text import detects likely Windows-1256 Arabic when invalid UTF-8 has Arabic byte runs; other invalid UTF-8 remains Windows-1252) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Every format Excel opens or saves, with GridCraft's read and write support, fidelity and tests. Part of [target-app-parity.md](target-app-parity.md). **File formats: ~50% ready, 80–120 h** @@ -41,10 +41,10 @@ and repair-free opening remain unverified. | Format | Excel | GridCraft read | GridCraft write | Notes | |---|---|---|---|---| -| `.csv` UTF-8 / Windows / Mac / MS-DOS | R/W | yes (UTF-8 and UTF-16 BOMs, UTF-8, Windows-1252 fallback; delimiter detection) | yes (UTF-8 without BOM, CRLF; a file opened as CSV is saved with the separator it was read with, so a `;` file stays `;`; otherwise `,`) | Excel writes the **displayed** values; we write raw values. No encoding choice on save. Re-import of a single-column export splits text (#127) A used range that would write more than 512 MiB (two far-apart cells span the whole rectangle) is refused with an error instead of being expanded (#528). | -| `.txt` tab-delimited, Unicode text | R/W | yes (`.txt`, `.tsv`, `.tab`) | yes | Text Import Wizard options partly via Text to Columns | -| `.csv` UTF-8 / Windows / Mac / MS-DOS | R/W | yes (UTF-8 and UTF-16 BOMs, UTF-8, Windows-1252 fallback; delimiter detection) | yes (UTF-8 without BOM, CRLF) | Writes the **displayed** values like Excel (number formats applied, formula results, never column-width `#####`). No encoding choice on save. Re-import of a single-column export splits text (#127) A used range that would write more than 512 MiB (two far-apart cells span the whole rectangle) is refused with an error instead of being expanded (#528). | -| `.txt` tab-delimited, Unicode text | R/W | yes (`.txt`, `.tsv`, `.tab`) | yes (displayed values) | Text Import Wizard options partly via Text to Columns | +| `.csv` UTF-8 / Windows / Mac / MS-DOS | R/W | yes (UTF-8 and UTF-16 BOMs, UTF-8, likely Windows-1256 Arabic detection, otherwise Windows-1252 fallback; delimiter detection) | yes (UTF-8 without BOM, CRLF; a file opened as CSV is saved with the separator it was read with, so a `;` file stays `;`; otherwise `,`) | Excel writes the **displayed** values; we write raw values. No encoding choice on save. Re-import of a single-column export splits text (#127) A used range that would write more than 512 MiB (two far-apart cells span the whole rectangle) is refused with an error instead of being expanded (#528). | +| `.txt` tab-delimited, Unicode text | R/W | yes (`.txt`, `.tsv`, `.tab`; likely Windows-1256 Arabic detection for invalid UTF-8) | yes | Text Import Wizard options partly via Text to Columns | +| `.csv` UTF-8 / Windows / Mac / MS-DOS | R/W | yes (UTF-8 and UTF-16 BOMs, UTF-8, likely Windows-1256 Arabic detection, otherwise Windows-1252 fallback; delimiter detection) | yes (UTF-8 without BOM, CRLF) | Writes the **displayed** values like Excel (number formats applied, formula results, never column-width `#####`). No encoding choice on save. Re-import of a single-column export splits text (#127) A used range that would write more than 512 MiB (two far-apart cells span the whole rectangle) is refused with an error instead of being expanded (#528). | +| `.txt` tab-delimited, Unicode text | R/W | yes (`.txt`, `.tsv`, `.tab`; likely Windows-1256 Arabic detection for invalid UTF-8) | yes (displayed values) | Text Import Wizard options partly via Text to Columns | | `.prn` formatted text (space-delimited) | R/W | no | no | | | `.dif`, `.slk` (SYLK) | R/W | no | no | Small, legacy | | `.htm` / `.html` web page | R/W (Windows) | yes, via Get Data ▸ From HTML | yes (one sheet as a table; a used range taller than 100,001 rows fails with an error instead of being cut short) | Formatting not exported | @@ -85,6 +85,7 @@ refresh. Counted under features (Get & Transform, ~10% ready), not here. | Date | Change | Summary | |---|---|---| +| 2026-10-19 | minor | CSV and text import decodes likely Windows-1256 Arabic; other invalid UTF-8 retains the Windows-1252 fallback | | 2026-10-19 | minor | .xls import skips dialog sheets (WsBool `fDialog`), which share the worksheet BOF type and were read as empty worksheets | | 2026-10-19 | minor | `.xls` import reads WINDOW2: right-to-left sheets (common in older Arabic and Hebrew files) open right-to-left, and hidden gridlines, headings and zero values and Show Formulas carry over | | 2026-10-19 | trivial | XLSX text boxes and shapes keep right-to-left paragraph direction (`a:pPr rtl`, ECMA-376 Part 1 §21.1.2.2.7): read from the first paragraph and written on every paragraph; output without it is unchanged | diff --git a/docs/localization-parity.md b/docs/localization-parity.md index 64876633d..e89c4d2b8 100644 --- a/docs/localization-parity.md +++ b/docs/localization-parity.md @@ -1,6 +1,6 @@ # Localization parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-19 · **Change:** minor (defined-name length limit counts characters, so long Arabic names are accepted; Table names are limited to 255 characters; right-to-left sheets in `.xls` files open right-to-left; text boxes and shapes keep right-to-left paragraph direction through XLSX save; Save as Web Page and copied HTML keep right-to-left; French and Serbian `context|label` translations no longer show the context) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-19 · **Change:** minor (defined-name length limit counts characters, so long Arabic names are accepted; Table names are limited to 255 characters; right-to-left sheets in `.xls` files open right-to-left; text boxes and shapes keep right-to-left paragraph direction through XLSX save; Save as Web Page and copied HTML keep right-to-left; French and Serbian `context|label` translations no longer show the context; CSV and text import decodes likely Windows-1256 Arabic; Arabic shaping and bidirectional UI remain unsupported) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Per-language status against Excel, which ships its UI in dozens of languages (Windows language packs cover 100+), with localized function names, argument separators, number and date formats, @@ -83,6 +83,7 @@ language (none so far) and of localized function names. | Date | Change | Summary | |---|---|---| +| 2026-10-19 | minor | Arabic CSV and text imports decode Windows-1256 text; Arabic shaping and bidirectional UI remain unsupported | | 2026-10-19 | minor | Defined-name length counts characters, not bytes: Arabic names up to 255 letters are accepted, and Create from Selection keeps long Arabic headers | | 2026-10-19 | minor | Table names are limited to 255 Unicode characters | | 2026-10-19 | minor | Right-to-left sheets in legacy `.xls` files (common in older Arabic workbooks) open right-to-left |