From 43958b537e8135d2857151bdbac4c22423f46121 Mon Sep 17 00:00:00 2001 From: Valentin Klein Date: Sun, 11 Oct 2026 17:47:42 +0200 Subject: [PATCH] =?UTF-8?q?DXF:=20read=20pre-2007=20files=20in=20their=20$?= =?UTF-8?q?DWGCODEPAGE=20(CJK,=20Cyrillic,=20Central=20European=E2=80=A6)?= =?UTF-8?q?=20(fixes=20#505)=20(#511)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squash of PR #511 by @vangel76. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + Cargo.toml | 1 + crates/dxf/Cargo.toml | 4 +- crates/dxf/src/codepage.rs | 294 +++++++++++++++++++++++++++++++++++ crates/dxf/src/lib.rs | 50 ++++-- crates/io/tests/codepages.rs | 204 ++++++++++++++++++++++++ 6 files changed, 539 insertions(+), 16 deletions(-) create mode 100644 crates/dxf/src/codepage.rs create mode 100644 crates/io/tests/codepages.rs diff --git a/Cargo.lock b/Cargo.lock index 98476e43..3a5c307d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,6 +606,8 @@ dependencies = [ name = "cadcraft-dxf" version = "0.5.0" dependencies = [ + "encoding_rs", + "log", "thiserror 2.0.21", ] diff --git a/Cargo.toml b/Cargo.toml index 07035407..d3290623 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ skrifa = "0.47" harfrust = "0.12" unicode-bidi = "0.3.18" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +encoding_rs = "0.8.35" # ui craft-ui = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "5c874d51eeb3de70e735c8ce87a59a7dedd47675", features = ["serde"] } diff --git a/crates/dxf/Cargo.toml b/crates/dxf/Cargo.toml index 84bfce4f..ef6703c5 100644 --- a/crates/dxf/Cargo.toml +++ b/crates/dxf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cadcraft-dxf" -description = "Standalone DXF tag reader/writer (ASCII and binary), from the published DXF reference" +description = "Standalone DXF tag reader/writer (ASCII and binary, pre-2007 code pages), from the published DXF reference" version.workspace = true edition.workspace = true license.workspace = true @@ -10,3 +10,5 @@ workspace = true [dependencies] thiserror = { workspace = true } +log = { workspace = true } +encoding_rs = { workspace = true } diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs new file mode 100644 index 00000000..6f27b5be --- /dev/null +++ b/crates/dxf/src/codepage.rs @@ -0,0 +1,294 @@ +//! Text decoding for DXF files. +//! +//! R2007 (AC1021) and later DXF files are UTF-8. Older ones store text in the code page named by +//! the `$DWGCODEPAGE` header variable (`ANSI_1252`, `ANSI_932`, `DOS850`, …); characters outside +//! it are written as `\U+XXXX` (any character) or `\M+nXXXX` (a double-byte character of an East +//! Asian code page) escapes, which [`crate::parse`] decodes after the bytes. +//! +//! The Windows, ISO and East Asian code pages are decoded with `encoding_rs`; DOS 437, 850 and +//! 852 with the tables below (the upper halves of the IBM PC code pages; the lower halves are +//! ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated +//! double-byte character, an unassigned byte) becomes U+FFFD and the bytes after it decode +//! normally, so line ends and the group structure survive. + +use std::borrow::Cow; + +use encoding_rs::Encoding; + +/// How the bytes of a code page become text. +#[derive(Clone, Copy, Debug)] +pub(crate) enum Decoder { + Utf8, + Encoding(&'static Encoding), + /// Single-byte DOS code page: ASCII, then these 128 characters for bytes 0x80–0xFF. + Table(&'static [char; 128]), +} + +/// The decoder for a `$DWGCODEPAGE` value (case-insensitive), or `None` when CADCraft doesn't +/// know the code page. +pub(crate) fn decoder(code_page: &str) -> Option { + use encoding_rs as e; + let name = code_page.trim().to_ascii_uppercase().replace(['-', ' '], "_"); + let enc = match name.as_str() { + "UTF8" | "UTF_8" | "UNICODE" => return Some(Decoder::Utf8), + "ASCII" | "ANSI_1252" | "ISO8859_1" | "ISO_8859_1" => e::WINDOWS_1252, + "ANSI_874" => e::WINDOWS_874, + "ANSI_932" | "DOS932" | "SHIFT_JIS" => e::SHIFT_JIS, + "ANSI_936" | "GB2312" | "GBK" => e::GBK, + "ANSI_949" | "KSC5601" | "KOREAN" => e::EUC_KR, + "ANSI_950" | "BIG5" => e::BIG5, + "ANSI_1250" => e::WINDOWS_1250, + "ANSI_1251" => e::WINDOWS_1251, + "ANSI_1253" => e::WINDOWS_1253, + "ANSI_1254" | "ISO8859_9" | "ISO_8859_9" => e::WINDOWS_1254, + "ANSI_1255" => e::WINDOWS_1255, + "ANSI_1256" => e::WINDOWS_1256, + "ANSI_1257" => e::WINDOWS_1257, + "ANSI_1258" => e::WINDOWS_1258, + "ISO8859_2" | "ISO_8859_2" => e::ISO_8859_2, + "ISO8859_3" | "ISO_8859_3" => e::ISO_8859_3, + "ISO8859_4" | "ISO_8859_4" => e::ISO_8859_4, + "ISO8859_5" | "ISO_8859_5" => e::ISO_8859_5, + "ISO8859_6" | "ISO_8859_6" => e::ISO_8859_6, + "ISO8859_7" | "ISO_8859_7" => e::ISO_8859_7, + "ISO8859_8" | "ISO_8859_8" => e::ISO_8859_8, + "DOS866" => e::IBM866, + "MACINTOSH" | "MAC_ROMAN" => e::MACINTOSH, + "DOS437" => return Some(Decoder::Table(&DOS437)), + "DOS850" => return Some(Decoder::Table(&DOS850)), + "DOS852" => return Some(Decoder::Table(&DOS852)), + _ => return None, + }; + Some(Decoder::Encoding(enc)) +} + +/// Decode `bytes` from the code page `code_page` names (a `$DWGCODEPAGE` value such as +/// `ANSI_1251`), or `None` when CADCraft doesn't know that code page. Invalid sequences become +/// U+FFFD; escapes (`\U+XXXX`, `\M+nXXXX`) are left as they are. +pub fn decode_code_page(bytes: &[u8], code_page: &str) -> Option { + decoder(code_page).map(|d| decode(bytes, d).into_owned()) +} + +/// Decode bytes with a decoder. UTF-8 that isn't valid falls back to Windows-1252. +pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { + match d { + Decoder::Utf8 => match std::str::from_utf8(bytes) { + Ok(s) => Cow::Borrowed(s), + Err(_) => encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes).0, + }, + Decoder::Encoding(e) => e.decode_without_bom_handling(bytes).0, + Decoder::Table(t) => match std::str::from_utf8(bytes) { + Ok(s) if s.is_ascii() => Cow::Borrowed(s), + _ => Cow::Owned( + bytes + .iter() + .map(|&b| if b.is_ascii() { char::from(b) } else { t.get(usize::from(b & 0x7F)).copied().unwrap_or('\u{FFFD}') }) + .collect(), + ), + }, + } +} + +/// The `$DWGCODEPAGE` value of an ASCII DXF file: the line two lines after a `$DWGCODEPAGE` +/// line (the group code `3` between them), looked for in the HEADER section only (the scan stops +/// at the first `ENDSEC`). Works on the raw bytes, before they are decoded. +fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { + let mut lines = bytes.split(|&b| b == b'\n').map(<[u8]>::trim_ascii); + while let Some(line) = lines.next() { + if line == b"ENDSEC" { + return None; + } + if line.eq_ignore_ascii_case(b"$DWGCODEPAGE") { + return match (lines.next(), lines.next()) { + (Some(b"3"), Some(value)) => Some(value), + _ => None, + }; + } + } + None +} + +/// Decode a whole ASCII DXF file. Valid UTF-8 is read as UTF-8: R2007+ files, pure-ASCII files, +/// and older files from writers that ignored the code page. Anything else is decoded from the +/// file's `$DWGCODEPAGE`, or from Windows-1252 (the usual `ANSI_1252`) when the header has none +/// or names a code page CADCraft doesn't know, which is logged as a warning. +pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { + if let Ok(s) = std::str::from_utf8(bytes) { + return Cow::Borrowed(s); + } + let d = match ascii_code_page(bytes) { + Some(name) => { + let name = String::from_utf8_lossy(name); + decoder(&name).unwrap_or_else(|| { + log::warn!("DXF: unknown code page `{name}`; reading the text as ANSI_1252"); + Decoder::Utf8 + }) + } + None => Decoder::Utf8, + }; + decode(bytes, d) +} + +/// The text state of a binary DXF file, learnt from its header as the tags go by: `$ACADVER` +/// (R2007+ strings are UTF-8) and `$DWGCODEPAGE` (the code page of older files). Both come +/// first in the HEADER section and are ASCII, so every later string is decoded right. +#[derive(Default)] +pub(crate) struct BinaryText { + /// The header variable whose value comes next (after a group 9). + var: Option, + utf8: bool, + code_page: Option, +} + +#[derive(Clone, Copy, PartialEq)] +enum HeaderVar { + AcadVer, + CodePage, +} + +impl BinaryText { + /// Decode one string value of group `code` and update the state from it. Strings of R2007+ + /// files, and of files without a code page, are UTF-8 when valid (else Windows-1252); older + /// files' strings are in their code page. + pub(crate) fn string(&mut self, code: i32, raw: &[u8]) -> String { + let d = if self.utf8 { Decoder::Utf8 } else { self.code_page.unwrap_or(Decoder::Utf8) }; + let s = decode(raw, d).into_owned(); + match (code, self.var.take()) { + (9, _) => { + self.var = match s.trim() { + "$ACADVER" => Some(HeaderVar::AcadVer), + "$DWGCODEPAGE" => Some(HeaderVar::CodePage), + _ => None, + } + } + (1, Some(HeaderVar::AcadVer)) => self.utf8 = is_utf8_version(&s), + (3, Some(HeaderVar::CodePage)) => { + self.code_page = decoder(&s); + if self.code_page.is_none() { + log::warn!("DXF: unknown code page `{}`; reading the text as ANSI_1252", s.trim()); + } + } + _ => {} + } + s + } +} + +/// True for an `$ACADVER` of R2007 (AC1021) or later, whose text is UTF-8. +fn is_utf8_version(acadver: &str) -> bool { + acadver.trim().strip_prefix("AC").and_then(|n| n.parse::().ok()).is_some_and(|n| n >= 1021) +} + +/// The double-byte code page of a `\M+n` escape: 1 Japanese (Shift-JIS), 2 Traditional Chinese +/// (Big5), 3 Korean (Wansung), 5 Simplified Chinese (GB 2312). 4 (Korean Johab) is not +/// supported, so its escapes stay literal. +pub(crate) fn mif_encoding(n: char) -> Option<&'static Encoding> { + match n { + '1' => Some(encoding_rs::SHIFT_JIS), + '2' => Some(encoding_rs::BIG5), + '3' => Some(encoding_rs::EUC_KR), + '5' => Some(encoding_rs::GBK), + _ => None, + } +} + +// Generated from the IBM PC code page definitions (bytes 0x80–0xFF in order). +const DOS437: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00A5}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}', + '\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}', + '\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}', + '\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}', + '\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; +const DOS850: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}', + '\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}', + '\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{00D7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{00C0}', '\u{00A9}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{00A2}', '\u{00A5}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{00E3}', '\u{00C3}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{00F0}', '\u{00D0}', '\u{00CA}', '\u{00CB}', + '\u{00C8}', '\u{0131}', '\u{00CD}', '\u{00CE}', '\u{00CF}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{00A6}', '\u{00CC}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{00D2}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{00FE}', '\u{00DE}', '\u{00DA}', '\u{00DB}', '\u{00D9}', + '\u{00FD}', '\u{00DD}', '\u{00AF}', '\u{00B4}', '\u{00AD}', '\u{00B1}', '\u{2017}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}', +]; +const DOS852: [char; 128] = [ + '\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{016F}', '\u{0107}', '\u{00E7}', '\u{0142}', '\u{00EB}', '\u{0150}', '\u{0151}', + '\u{00EE}', '\u{0179}', '\u{00C4}', '\u{0106}', '\u{00C9}', '\u{0139}', '\u{013A}', '\u{00F4}', '\u{00F6}', '\u{013D}', '\u{013E}', '\u{015A}', + '\u{015B}', '\u{00D6}', '\u{00DC}', '\u{0164}', '\u{0165}', '\u{0141}', '\u{00D7}', '\u{010D}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}', + '\u{0104}', '\u{0105}', '\u{017D}', '\u{017E}', '\u{0118}', '\u{0119}', '\u{00AC}', '\u{017A}', '\u{010C}', '\u{015F}', '\u{00AB}', '\u{00BB}', + '\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{011A}', '\u{015E}', '\u{2563}', '\u{2551}', '\u{2557}', + '\u{255D}', '\u{017B}', '\u{017C}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{0102}', '\u{0103}', + '\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{0111}', '\u{0110}', '\u{010E}', '\u{00CB}', + '\u{010F}', '\u{0147}', '\u{00CD}', '\u{00CE}', '\u{011B}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{0162}', '\u{016E}', '\u{2580}', + '\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{0143}', '\u{0144}', '\u{0148}', '\u{0160}', '\u{0161}', '\u{0154}', '\u{00DA}', '\u{0155}', '\u{0170}', + '\u{00FD}', '\u{00DD}', '\u{0163}', '\u{00B4}', '\u{00AD}', '\u{02DD}', '\u{02DB}', '\u{02C7}', '\u{02D8}', '\u{00A7}', '\u{00F7}', '\u{00B8}', + '\u{00B0}', '\u{00A8}', '\u{02D9}', '\u{0171}', '\u{0158}', '\u{0159}', '\u{25A0}', '\u{00A0}', +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn code_page_names() { + let d = |bytes: &[u8], cp: &str| decode_code_page(bytes, cp); + assert_eq!(d(b"\xCF\xF0\xE8\xE2\xE5\xF2", "ANSI_1251").as_deref(), Some("Привет")); + assert_eq!(d(b"\xCF\xF0\xE8\xE2\xE5\xF2", " ansi_1251 ").as_deref(), Some("Привет")); + assert_eq!(d(b"\x90\x7D\x96\xCA", "ANSI_932").as_deref(), Some("図面")); + assert_eq!(d(b"\xCD\xBC\xB2\xE3", "GB2312").as_deref(), Some("图层")); + assert_eq!(d(b"\xB9\xCF\xBC\x68", "BIG5").as_deref(), Some("圖層")); + assert_eq!(d(b"\xB5\xB5\xB8\xE9", "ANSI_949").as_deref(), Some("도면")); + assert_eq!(d(b"\xD3\xF7\xDD\xE4\xE9\xEF", "ANSI_1253").as_deref(), Some("Σχέδιο")); + assert_eq!(d(b"\xC0\xD2\xBE", "ANSI_874").as_deref(), Some("ภาพ")); + assert_eq!(d(b"\x80", "ANSI_1252").as_deref(), Some("€")); + // DOS 437 and 850 agree on letters but differ on 0x9B (¢ / ø). + assert_eq!(d(b"Gr\x94\xE1e \x9B", "DOS437").as_deref(), Some("Größe ¢")); + assert_eq!(d(b"Gr\x94\xE1e \x9B", "dos850").as_deref(), Some("Größe ø")); + assert_eq!(d(b"\xBD\xA2\x88w", "DOS852").as_deref(), Some("Żółw")); + assert_eq!(d(b"\x91\xAB\xAE\xA9", "DOS866").as_deref(), Some("Слой")); + assert_eq!(d("é".as_bytes(), "UTF8").as_deref(), Some("é")); + assert_eq!(d(b"x", "ANSI_1361"), None); + assert_eq!(d(b"x", "EBCDIC"), None); + assert_eq!(d(b"x", ""), None); + } + + #[test] + fn invalid_sequences_become_replacement_characters() { + // A Shift-JIS lead byte cut off by the line end keeps the line end. + assert_eq!(decode_code_page(b"\x90\n1\n", "ANSI_932").as_deref(), Some("\u{FFFD}\n1\n")); + assert_eq!(decode_code_page(b"\x90", "ANSI_932").as_deref(), Some("\u{FFFD}")); + assert_eq!(decode_code_page(b"\xFF\xFF", "ANSI_936").as_deref(), Some("\u{FFFD}\u{FFFD}")); + } + + #[test] + fn header_scan() { + let head = b"0\nSECTION\n2\nHEADER\n9\n$ACADVER\n1\nAC1015\n9\n$DWGCODEPAGE\n3\nANSI_1251\r\n0\nENDSEC\n"; + assert_eq!(ascii_code_page(head), Some(&b"ANSI_1251"[..])); + // Only the HEADER section counts. + assert_eq!(ascii_code_page(b"0\nSECTION\n2\nHEADER\n0\nENDSEC\n9\n$DWGCODEPAGE\n3\nANSI_1251\n"), None); + // Wrong group code, or the file ends early. + assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n1\nANSI_1251\n"), None); + assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n3"), None); + assert!(is_utf8_version("AC1021") && is_utf8_version(" AC1032 ")); + assert!(!is_utf8_version("AC1018") && !is_utf8_version("AC1009") && !is_utf8_version("garbage")); + } + + #[test] + fn mif_escapes() { + let one = |s: &str| crate::parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str(); + assert_eq!(one("\\M+5BCFE\\M+1907D\\M+2B9CF\\M+3B5B5!"), "件図圖도!"); + assert_eq!(one("\\M+5BCFE\\U+0021"), "件!"); + // Johab, unknown code pages, bad hex, short escapes and unassigned codes stay literal. + for s in ["\\M+4ABCD", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] { + assert_eq!(one(s), s); + } + } +} diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 6d7233bf..82ecf55e 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -166,14 +166,8 @@ const BINARY_SENTINEL: &[u8] = b"AutoCAD Binary DXF\r\n\x1a\0"; /// tag list to a few GB; real drawings stay far below (a 10 MB DWG is typically a few million). pub const MAX_TAGS: usize = 50_000_000; -/// Decode bytes as text: UTF-8 when valid (borrowed, no copy), otherwise Latin-1 (old ANSI_1252 -/// files). -fn decode_text(b: &[u8]) -> std::borrow::Cow<'_, str> { - match std::str::from_utf8(b) { - Ok(s) => std::borrow::Cow::Borrowed(s), - Err(_) => std::borrow::Cow::Owned(b.iter().map(|&c| char::from(c)).collect()), - } -} +mod codepage; +pub use codepage::decode_code_page; /// Parse a DXF file (ASCII or binary). pub fn parse(bytes: &[u8]) -> Result> { @@ -192,7 +186,7 @@ pub fn parse_with_limit(bytes: &[u8], max_tags: usize) -> Result> { if tags > max_tags { return Err(DxfError::TooLarge { tags, limit: max_tags }); } - parse_ascii_with_limit(&decode_text(bytes), max_tags) + parse_ascii_with_limit(&codepage::decode_ascii_file(bytes), max_tags) } /// The four hex digits of a `\U+XXXX` escape starting at `chars[i]`, if there is one. @@ -207,17 +201,41 @@ fn escape_at(chars: &[char], i: usize) -> Option { hex.iter().try_fold(0u32, |acc, c| Some(acc * 16 + c.to_digit(16)?)) } +/// The character of a `\M+nXXXX` escape starting at `chars[i]`, if there is one: `n` names a +/// double-byte code page and `XXXX` is the character's two bytes in hex. +fn mif_escape_at(chars: &[char], i: usize) -> Option { + if chars.get(i) != Some(&'\\') || chars.get(i + 1) != Some(&'M') || chars.get(i + 2) != Some(&'+') { + return None; + } + let enc = codepage::mif_encoding(*chars.get(i + 3)?)?; + let hex = chars.get(i + 4..i + 8)?; + if !hex.iter().all(char::is_ascii_hexdigit) { + return None; + } + let code = hex.iter().try_fold(0u16, |acc, c| Some(acc * 16 + u16::try_from(c.to_digit(16)?).ok()?))?; + let bytes = code.to_be_bytes(); + let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?; + let mut it = text.chars(); + it.next().filter(|_| it.next().is_none()) +} + /// Unescape the `\U+XXXX` sequences pre-R2007 DXF uses for characters outside the file's code /// page (four hex digits, one UTF-16 code unit; characters above U+FFFF are a surrogate pair of -/// escapes). Malformed escapes and unpaired surrogates are kept as literal text. +/// escapes), and the `\M+nXXXX` sequences for double-byte East Asian characters. Malformed +/// escapes, unpaired surrogates and unknown `\M+` code pages are kept as literal text. fn unescape_unicode(s: &str) -> String { - if !s.contains("\\U+") { + if !s.contains("\\U+") && !s.contains("\\M+") { return s.to_string(); } let mut out = String::with_capacity(s.len()); let chars: Vec = s.chars().collect(); let mut i = 0; while let Some(&c) = chars.get(i) { + if let Some(ch) = mif_escape_at(&chars, i) { + out.push(ch); + i += 8; + continue; + } if let Some(unit) = escape_at(&chars, i) { if let Some(ch) = char::from_u32(unit) { out.push(ch); @@ -279,7 +297,7 @@ struct Cur<'a> { p: usize, } -impl Cur<'_> { +impl<'a> Cur<'a> { fn take(&mut self, n: usize) -> Result<&[u8]> { let end = self.p.checked_add(n).ok_or(DxfError::Eof)?; let s = self.b.get(self.p..end).ok_or(DxfError::Eof)?; @@ -309,12 +327,13 @@ impl Cur<'_> { a.copy_from_slice(s); Ok(f64::from_le_bytes(a)) } - fn cstr(&mut self) -> Result { + /// The bytes of a zero-terminated string (without the zero), decoded by the caller. + fn cstr(&mut self) -> Result<&'a [u8]> { let start = self.p; while self.p < self.b.len() && self.b.get(self.p) != Some(&0) { self.p += 1; } - let s = decode_text(self.b.get(start..self.p).unwrap_or(&[])).into_owned(); + let s = self.b.get(start..self.p).unwrap_or(&[]); if self.p >= self.b.len() { return Err(DxfError::Eof); } @@ -330,6 +349,7 @@ pub fn parse_binary(b: &[u8]) -> Result> { fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { let mut c = Cur { b, p: 0 }; + let mut text = codepage::BinaryText::default(); let mut tags = Vec::new(); while c.p < b.len() { let mut code = i32::from(c.i16()?); @@ -337,7 +357,7 @@ fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { code = i32::from(c.i16()?); } let v = match kind_of(code) { - Kind::Str => Value::Str(unescape_unicode(&c.cstr()?)), + Kind::Str => Value::Str(unescape_unicode(&text.string(code, c.cstr()?))), Kind::Real => Value::Real(c.f64()?), Kind::Int16 => Value::Int(i64::from(c.i16()?)), Kind::Int32 => Value::Int(i64::from(c.i32()?)), diff --git a/crates/io/tests/codepages.rs b/crates/io/tests/codepages.rs new file mode 100644 index 00000000..40d85b3f --- /dev/null +++ b/crates/io/tests/codepages.rs @@ -0,0 +1,204 @@ +//! Pre-2007 DXF text is in the code page named by `$DWGCODEPAGE` (#505): Cyrillic, Central +//! European, Greek, Japanese, Chinese and Korean drawings open with their layer names, block names +//! and text intact, in ASCII and binary DXF, and a hostile file never panics. + +use cadcraft_doc::{Drawing, EntityKind}; + +/// An ASCII DXF of `version` declaring `code_page`, with a layer, a block and a TEXT whose names +/// and value are the raw bytes given. +fn ascii_dxf(version: &str, code_page: &str, layer: &[u8], block: &[u8], text: &[u8]) -> Vec { + let mut b = Vec::new(); + let mut put = |code: i32, v: &[u8]| { + b.extend_from_slice(format!("{code:>3}\r\n").as_bytes()); + b.extend_from_slice(v); + b.extend_from_slice(b"\r\n"); + }; + for (c, v) in [(0, &b"SECTION"[..]), (2, b"HEADER"), (9, b"$ACADVER"), (1, version.as_bytes()), (9, b"$DWGCODEPAGE"), (3, code_page.as_bytes())] { + put(c, v); + } + for (c, v) in [(0, &b"ENDSEC"[..]), (0, b"SECTION"), (2, b"TABLES"), (0, b"TABLE"), (2, b"LAYER"), (0, b"LAYER"), (2, layer), (70, b"0")] { + put(c, v); + } + for (c, v) in [(62, &b"7"[..]), (6, b"CONTINUOUS"), (0, b"ENDTAB"), (0, b"ENDSEC"), (0, b"SECTION"), (2, b"BLOCKS"), (0, b"BLOCK")] { + put(c, v); + } + for (c, v) in [(8, &b"0"[..]), (2, block), (70, b"0"), (10, b"0"), (20, b"0"), (30, b"0"), (0, b"LINE"), (8, b"0")] { + put(c, v); + } + for (c, v) in [(10, &b"0"[..]), (20, b"0"), (11, b"1"), (21, b"1"), (0, b"ENDBLK"), (0, b"ENDSEC"), (0, b"SECTION"), (2, b"ENTITIES")] { + put(c, v); + } + for (c, v) in [(0, &b"TEXT"[..]), (8, layer), (10, b"0"), (20, b"0"), (40, b"1"), (1, text), (0, b"INSERT"), (8, b"0")] { + put(c, v); + } + for (c, v) in [(2, block), (10, b"5"), (20, b"5"), (0, b"ENDSEC"), (0, b"EOF")] { + put(c, v); + } + b +} + +fn text_of(d: &Drawing) -> String { + d.model.iter().find_map(|e| if let EntityKind::Text(t) = &e.kind { Some(t.value.clone()) } else { None }).unwrap_or_default() +} + +fn names(d: &Drawing) -> (String, String, String) { + // The TEXT's layer, which the LAYER table defines. + let layer = d.model.iter().find(|e| matches!(e.kind, EntityKind::Text(_))).map(|e| e.common.layer.clone()).unwrap_or_default(); + assert!(d.layers.iter().any(|l| l.name == layer), "layer `{layer}` is defined"); + let insert = d.model.iter().find_map(|e| if let EntityKind::Insert(i) = &e.kind { Some(i.block.clone()) } else { None }); + let block = insert.unwrap_or_default(); + assert!(d.block(&block).is_some(), "block `{block}` is defined"); + (layer, block, text_of(d)) +} + +/// Text, layer and block names decode from each code page AutoCAD writes pre-2007 files in. +#[test] +fn reads_text_in_its_code_page() { + // (code page, layer bytes, layer, text bytes, text); byte sequences from the code page tables. + let cases: [(&str, &[u8], &str, &[u8], &str); 10] = [ + ("ANSI_1251", b"\xD1\xEB\xEE\xE9", "Слой", b"\xCF\xF0\xE8\xE2\xE5\xF2", "Привет"), + ("ANSI_1250", b"\xA3\xF3d\x9F", "Łódź", b"\x8Ail\xE1", "Šilá"), + ("ANSI_1253", b"\xD3\xF7\xDD\xE4\xE9\xEF", "Σχέδιο", b"\xE1\xE2\xE3", "αβγ"), + ("ANSI_1254", b"\xC7izim", "Çizim", b"\xDE\xF0", "Şğ"), + ("ANSI_874", b"\xC0\xD2\xBE", "ภาพ", b"\xC0\xD2\xBE", "ภาพ"), + ("ANSI_932", b"\x90\x7D\x96\xCA", "図面", b"\x90\x7D\x96\xCA", "図面"), + ("ANSI_936", b"\xCD\xBC\xB2\xE3", "图层", b"\xCD\xBC\xB2\xE3", "图层"), + ("ANSI_949", b"\xB5\xB5\xB8\xE9", "도면", b"\xB5\xB5\xB8\xE9", "도면"), + ("ANSI_950", b"\xB9\xCF\xBC\x68", "圖層", b"\xB9\xCF\xBC\x68", "圖層"), + ("DOS850", b"Gr\x94\xE1e", "Größe", b"\x9B\x9D", "øØ"), + ]; + for (cp, layer_bytes, layer, text_bytes, text) in cases { + let mut block_bytes = b"B_".to_vec(); + block_bytes.extend_from_slice(layer_bytes); + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", cp, layer_bytes, &block_bytes, text_bytes)).unwrap(); + assert_eq!(names(&d), (layer.to_string(), format!("B_{layer}"), text.to_string()), "{cp}"); + } +} + +/// A Shift-JIS trail byte can be 0x5C (`\`): 表 is 95 5C. Decoding the bytes before the escapes +/// keeps it a kanji instead of starting a `\U+` escape; real escapes still decode. +#[test] +fn shift_jis_backslash_trail_byte_is_not_an_escape() { + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_932", b"L", b"B", b"\x95\x5CU+0041 \\U+00E9 \\M+5BCFE")).unwrap(); + assert_eq!(text_of(&d), "表U+0041 é 件"); +} + +/// R2007+ files and files whose bytes are valid UTF-8 are UTF-8, whatever code page they declare +/// (the DWG bridge writes UTF-8 with the DWG's code page name); invalid UTF-8 in a 2007+ file +/// falls back to the declared code page. +#[test] +fn utf8_wins_when_valid() { + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_1251", "Слой".as_bytes(), b"B", "Привет".as_bytes())).unwrap(); + assert_eq!(names(&d), ("Слой".into(), "B".into(), "Привет".into())); + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1032", "ANSI_1252", "图层".as_bytes(), b"B", "Café".as_bytes())).unwrap(); + assert_eq!(names(&d), ("图层".into(), "B".into(), "Café".into())); + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1021", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"B", b"x")).unwrap(); + assert_eq!(names(&d).0, "Слой"); +} + +/// A code page CADCraft doesn't know (or none) reads as ANSI_1252, as before. +#[test] +fn unknown_code_page_falls_back_to_ansi_1252() { + for cp in ["ANSI_1361", "EBCDIC", "", "\u{1}"] { + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", cp, b"Caf\xE9", b"B", b"\x80 \xD8")).unwrap(); + assert_eq!(names(&d), ("Café".into(), "B".into(), "€ Ø".into()), "{cp:?}"); + } +} + +/// Reading converts to Unicode; saving writes pure ASCII with `\U+` escapes, which reads back the +/// same. +#[test] +fn code_page_text_survives_a_round_trip() { + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2")).unwrap(); + let out = cadcraft_io::write_dxf(&d); + assert!(out.is_ascii()); + let back = cadcraft_io::read_dxf(out.as_bytes()).unwrap(); + assert_eq!(names(&back), ("Слой".into(), "Блок".into(), "Привет".into())); +} + +/// A binary DXF: R2000 strings are in the code page, R2007+ strings UTF-8. +#[test] +fn binary_dxf_honours_the_code_page() { + let bin = |version: &str, cp: &str, text: &[u8]| { + let mut b = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec(); + let mut s = |code: i16, v: &[u8]| { + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(v); + b.push(0); + }; + s(0, b"SECTION"); + s(2, b"HEADER"); + s(9, b"$ACADVER"); + s(1, version.as_bytes()); + s(9, b"$DWGCODEPAGE"); + s(3, cp.as_bytes()); + s(0, b"ENDSEC"); + s(0, b"SECTION"); + s(2, b"ENTITIES"); + s(0, b"TEXT"); + s(8, b"0"); + b.extend_from_slice(&10i16.to_le_bytes()); + b.extend_from_slice(&0f64.to_le_bytes()); + b.extend_from_slice(&20i16.to_le_bytes()); + b.extend_from_slice(&0f64.to_le_bytes()); + b.extend_from_slice(&40i16.to_le_bytes()); + b.extend_from_slice(&1f64.to_le_bytes()); + let mut s = |code: i16, v: &[u8]| { + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(v); + b.push(0); + }; + s(1, text); + s(0, b"ENDSEC"); + s(0, b"EOF"); + b + }; + let read = |b: &[u8]| text_of(&cadcraft_io::read_dxf(b).unwrap()); + assert_eq!(read(&bin("AC1015", "ANSI_1251", b"\xCF\xF0\xE8\xE2\xE5\xF2")), "Привет"); + assert_eq!(read(&bin("AC1018", "ANSI_932", b"\x95\x5C\\U+00E9")), "表é"); + // A short Cyrillic string that is also valid UTF-8 (D1 B2) is still read in the code page. + assert_eq!(read(&bin("AC1015", "ANSI_1251", b"\xD1\xB2")), "СІ"); + assert_eq!(read(&bin("AC1021", "ANSI_1251", "Привет".as_bytes())), "Привет"); + assert_eq!(read(&bin("AC1015", "ANSI_1361", b"Caf\xE9")), "Café"); +} + +/// Truncated and invalid multibyte sequences become U+FFFD; nothing panics and the structure of +/// the file (line ends, group codes) survives. +#[test] +fn hostile_multibyte_never_panics() { + // A lead byte cut off by the line end, a lone trail byte, an invalid pair. + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_932", b"\x90", b"\x81", b"A\x90")).unwrap(); + assert_eq!(names(&d), ("\u{FFFD}".into(), "\u{FFFD}".into(), "A\u{FFFD}".into())); + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_936", b"\xFF\xFF", b"B", b"\xA1")).unwrap(); + assert_eq!(text_of(&d), "\u{FFFD}"); + // A file cut off inside a double-byte character, garbage after the header, every byte value. + let mut cut = ascii_dxf("AC1015", "ANSI_949", b"L", b"B", b"x"); + cut.truncate(cut.len() - 10); + cut.push(0xB5); + let _ = cadcraft_io::read_dxf(&cut); + let all: Vec = (0..=255u8).collect(); + for cp in ["ANSI_932", "ANSI_936", "ANSI_949", "ANSI_950", "ANSI_1251", "DOS437", "DOS852", "MACINTOSH"] { + let d = cadcraft_io::read_dxf(&ascii_dxf( + "AC1015", + cp, + b"L", + b"B", + &all.iter().copied().filter(|&b| b != b'\n' && b != b'\r').collect::>(), + )); + assert!(d.is_ok(), "{cp}"); + let mut junk = ascii_dxf("AC1015", cp, b"L", b"B", b"x"); + junk.extend_from_slice(&all); + let _ = cadcraft_io::read_dxf(&junk); + } +} + +/// A pre-2007 DWG keeps its code page through the acadrust bridge: a Cyrillic R2000 DXF saved as +/// DWG (acadrust writes 2004, text in ANSI_1251) opens with its text. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn pre_2007_dwg_text_decodes_through_the_bridge() { + let dxf = ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2"); + let dwg = cadcraft_dwg::dxf_to_dwg(&dxf).unwrap(); + let d = cadcraft_io::read(&dwg, "x.dwg").unwrap(); + assert_eq!(names(&d), ("Слой".into(), "Блок".into(), "Привет".into())); +}