From 771643094889ef6c468e4e6b6be5f7bddf3394ed Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:18:54 +0200 Subject: [PATCH] DXF: multi-byte East Asian code pages read from the code page unless the whole file is UTF-8 (#1025) Co-Authored-By: Claude Sonnet 5.5 --- crates/dxf/src/codepage.rs | 71 ++++++++++++++++++++++++++++++----- crates/dxf/src/lib.rs | 10 ++++- crates/dxf/tests/hardening.rs | 38 +++++++++++++++++++ 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index ddc4ca24f..f0d2a985d 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -108,10 +108,21 @@ fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> { None } +impl Decoder { + /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5). + /// Their short double-byte strings are often also valid UTF-8, so UTF-8 is only trusted for + /// a whole file there (see [`decode_ascii_file`] and [`BinaryText`]). + fn is_east_asian(self) -> bool { + use encoding_rs as e; + matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5) + } +} + /// Decode one string of a DXF file: valid UTF-8 is read as UTF-8 (R2007+ files, pure-ASCII text, /// and older files from writers that ignored the code page); anything else comes from the /// code page `d`. The same policy for ASCII lines and binary strings, so one stray byte spoils -/// one string, not the whole file. +/// one string, not the whole file. East Asian code pages don't use this rule (see +/// [`Decoder::is_east_asian`]). pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> { match std::str::from_utf8(bytes) { Ok(s) => Cow::Borrowed(s), @@ -119,10 +130,13 @@ pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> { } } -/// Decode a whole ASCII DXF file, line by line (see [`decode_text`]). Lines that aren't valid -/// UTF-8 are decoded from the file's `$DWGCODEPAGE`, or from Windows-1252 (the usual `ANSI_1252`) -/// when the header has none or names a code page CADCraft doesn't know, which is logged as a -/// warning. +/// Decode a whole ASCII DXF file. A file that is valid UTF-8 throughout is UTF-8. Otherwise, for +/// a single-byte code page, each line is decoded on its own (see [`decode_text`]): lines that +/// aren't valid UTF-8 come from the file's `$DWGCODEPAGE`, or from Windows-1252 (the usual +/// `ANSI_1252`) when the header has none or names a code page CADCraft doesn't know, which is +/// logged as a warning. For a multi-byte East Asian code page the whole file is decoded from +/// that code page, because a short double-byte string (two half-width katakana, many kanji +/// pairs) can be valid UTF-8 by accident and must not be misread line by line. pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { if let Ok(s) = std::str::from_utf8(bytes) { return Cow::Borrowed(s); @@ -137,6 +151,9 @@ pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> { } None => Decoder::Utf8, }; + if d.is_east_asian() { + return decode(bytes, d); + } // Split after each line terminator (CR, LF or CRLF are all ASCII, so they never occur inside // a multi-byte character of the supported code pages) and decode each line on its own. let mut out = String::with_capacity(bytes.len()); @@ -155,6 +172,12 @@ pub(crate) struct BinaryText { var: Option, utf8: bool, code_page: Option, + /// Decode every string of an East Asian code page file from the code page, never as UTF-8 + /// (the second pass of [`BinaryText::needs_code_page_pass`]). + code_page_only: bool, + /// Non-ASCII strings seen under an East Asian code page: valid UTF-8 / not valid UTF-8. + ea_utf8: bool, + ea_other: bool, } #[derive(Clone, Copy, PartialEq)] @@ -164,12 +187,42 @@ enum HeaderVar { } impl BinaryText { - /// Decode one string value of group `code` and update the state from it. A string that is - /// valid UTF-8 is read as such; otherwise it is in the file's code page (Windows-1252 for - /// R2007+ files and files without a known code page). + /// The state for the second pass over a file whose strings are a mix of valid UTF-8 and + /// not: every string of an East Asian code page is decoded from that code page. + pub(crate) fn code_page_pass() -> Self { + Self { code_page_only: true, ..Self::default() } + } + + /// True when the file declared a multi-byte East Asian code page and its non-ASCII strings are + /// not all valid UTF-8 although some are: those can be short double-byte strings that look + /// like UTF-8 by accident, so the file must be parsed again with [`Self::code_page_pass`]. (A + /// file whose strings are all valid UTF-8 is UTF-8; one with none is plainly the code page.) + pub(crate) fn needs_code_page_pass(&self) -> bool { + !self.code_page_only && self.ea_utf8 && self.ea_other + } + + /// Decode one string value of group `code` and update the state from it. For single-byte code + /// pages a string that is valid UTF-8 is read as such, otherwise it is in the file's code page + /// (Windows-1252 for R2007+ files and files without a known code page). East Asian code pages + /// see [`Self::needs_code_page_pass`]. pub(crate) fn string(&mut self, code: i32, raw: &[u8]) -> String { let d = if self.utf8 { Decoder::Utf8 } else { self.code_page.unwrap_or(Decoder::Utf8) }; - let s = decode_text(raw, d).into_owned(); + let s = if d.is_east_asian() { + if self.code_page_only { + decode(raw, d).into_owned() + } else { + if !raw.is_ascii() { + if std::str::from_utf8(raw).is_ok() { + self.ea_utf8 = true; + } else { + self.ea_other = true; + } + } + decode_text(raw, d).into_owned() + } + } else { + decode_text(raw, d).into_owned() + }; match (code, self.var.take()) { (9, _) => { self.var = match s.trim() { diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 77b0ad89d..bcb30db05 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -378,8 +378,16 @@ pub fn parse_binary(b: &[u8]) -> Result> { } fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result> { - let mut c = Cur { b, p: 0 }; let mut text = codepage::BinaryText::default(); + let tags = parse_binary_pass(b, max_tags, &mut text)?; + if text.needs_code_page_pass() { + return parse_binary_pass(b, max_tags, &mut codepage::BinaryText::code_page_pass()); + } + Ok(tags) +} + +fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) -> Result> { + let mut c = Cur { b, p: 0 }; let mut tags = Vec::new(); while c.p < b.len() { let mut code = i32::from(c.i16()?); diff --git a/crates/dxf/tests/hardening.rs b/crates/dxf/tests/hardening.rs index 24bfea082..27f494187 100644 --- a/crates/dxf/tests/hardening.rs +++ b/crates/dxf/tests/hardening.rs @@ -165,3 +165,41 @@ fn binary_sixteen_and_thirty_two_bit_values_wrap_in_range_and_clamp_outside() { assert_eq!(back[5], Tag::i(90, i64::from(i32::MAX))); assert_eq!(back[6], Tag::i(90, i64::from(i32::MIN))); } + +/// Strings of a `$DWGCODEPAGE`-declared file as ASCII text and as a binary file. +fn both_forms(code_page: &str, strings: &[&[u8]]) -> [Vec; 2] { + let mut a = format!("9\n$DWGCODEPAGE\n3\n{code_page}\n").into_bytes(); + for s in strings { + a.extend_from_slice(b"1\n"); + a.extend_from_slice(s); + a.push(b'\n'); + } + a.extend_from_slice(b"0\nEOF\n"); + let b = bin_with(|b| { + for (code, s) in [(9i16, &b"$DWGCODEPAGE"[..]), (3, code_page.as_bytes())].into_iter().chain(strings.iter().map(|s| (1, *s))) { + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(s); + b.push(0); + } + }); + [parse(&a).unwrap(), parse(&b).unwrap()] +} + +#[test] +fn east_asian_short_strings_that_look_like_utf8_stay_in_the_code_page() { + // C3 B1 is valid UTF-8 ("ñ") but is also two half-width katakana in Shift-JIS; the second + // string is plainly Shift-JIS, so the file is not UTF-8. + for t in both_forms("ANSI_932", &[b"\xC3\xB1", b"\x90\x7D\x96\xCA"]) { + assert_eq!(t[2].str(), "\u{FF83}\u{FF71}"); + assert_eq!(t[3].str(), "図面"); + } +} + +#[test] +fn east_asian_file_that_is_all_utf8_reads_as_utf8() { + for t in both_forms("ANSI_932", &["図面".as_bytes(), b"\xC3\xB1", b"plain"]) { + assert_eq!(t[2].str(), "図面"); + assert_eq!(t[3].str(), "ñ"); + assert_eq!(t[4].str(), "plain"); + } +}