DXF: multi-byte East Asian code pages read from the code page unless the whole file is UTF-8 (#1025)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/dxf/src/codepage.rs

@@ -108,10 +108,21 @@ fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> {
     None
 }
 
+impl Decoder {
+    /// True for the multi-byte East Asian code pages (Shift-JIS, GBK, EUC-KR/Wansung, Big5).
+    /// Their short double-byte strings are often also valid UTF-8, so UTF-8 is only trusted for
+    /// a whole file there (see [`decode_ascii_file`] and [`BinaryText`]).
+    fn is_east_asian(self) -> bool {
+        use encoding_rs as e;
+        matches!(self, Decoder::Encoding(x) if x == e::SHIFT_JIS || x == e::GBK || x == e::EUC_KR || x == e::BIG5)
+    }
+}
+
 /// Decode one string of a DXF file: valid UTF-8 is read as UTF-8 (R2007+ files, pure-ASCII text,
 /// and older files from writers that ignored the code page); anything else comes from the
 /// code page `d`. The same policy for ASCII lines and binary strings, so one stray byte spoils
-/// one string, not the whole file.
+/// one string, not the whole file. East Asian code pages don't use this rule (see
+/// [`Decoder::is_east_asian`]).
 pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> {
     match std::str::from_utf8(bytes) {
         Ok(s) => Cow::Borrowed(s),
@@ -119,10 +130,13 @@ pub(crate) fn decode_text(bytes: &[u8], d: Decoder) -> Cow<'_, str> {
     }
 }
 
-/// Decode a whole ASCII DXF file, line by line (see [`decode_text`]). Lines that aren't valid
-/// UTF-8 are decoded from the file's `$DWGCODEPAGE`, or from Windows-1252 (the usual `ANSI_1252`)
-/// when the header has none or names a code page CADCraft doesn't know, which is logged as a
-/// warning.
+/// Decode a whole ASCII DXF file. A file that is valid UTF-8 throughout is UTF-8. Otherwise, for
+/// a single-byte code page, each line is decoded on its own (see [`decode_text`]): lines that
+/// aren't valid UTF-8 come from the file's `$DWGCODEPAGE`, or from Windows-1252 (the usual
+/// `ANSI_1252`) when the header has none or names a code page CADCraft doesn't know, which is
+/// logged as a warning. For a multi-byte East Asian code page the whole file is decoded from
+/// that code page, because a short double-byte string (two half-width katakana, many kanji
+/// pairs) can be valid UTF-8 by accident and must not be misread line by line.
 pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> {
     if let Ok(s) = std::str::from_utf8(bytes) {
         return Cow::Borrowed(s);
@@ -137,6 +151,9 @@ pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> {
         }
         None => Decoder::Utf8,
     };
+    if d.is_east_asian() {
+        return decode(bytes, d);
+    }
     // Split after each line terminator (CR, LF or CRLF are all ASCII, so they never occur inside
     // a multi-byte character of the supported code pages) and decode each line on its own.
     let mut out = String::with_capacity(bytes.len());
@@ -155,6 +172,12 @@ pub(crate) struct BinaryText {
     var: Option<HeaderVar>,
     utf8: bool,
     code_page: Option<Decoder>,
+    /// Decode every string of an East Asian code page file from the code page, never as UTF-8
+    /// (the second pass of [`BinaryText::needs_code_page_pass`]).
+    code_page_only: bool,
+    /// Non-ASCII strings seen under an East Asian code page: valid UTF-8 / not valid UTF-8.
+    ea_utf8: bool,
+    ea_other: bool,
 }
 
 #[derive(Clone, Copy, PartialEq)]
@@ -164,12 +187,42 @@ enum HeaderVar {
 }
 
 impl BinaryText {
-    /// Decode one string value of group `code` and update the state from it. A string that is
-    /// valid UTF-8 is read as such; otherwise it is in the file's code page (Windows-1252 for
-    /// R2007+ files and files without a known code page).
+    /// The state for the second pass over a file whose strings are a mix of valid UTF-8 and
+    /// not: every string of an East Asian code page is decoded from that code page.
+    pub(crate) fn code_page_pass() -> Self {
+        Self { code_page_only: true, ..Self::default() }
+    }
+
+    /// True when the file declared a multi-byte East Asian code page and its non-ASCII strings are
+    /// not all valid UTF-8 although some are: those can be short double-byte strings that look
+    /// like UTF-8 by accident, so the file must be parsed again with [`Self::code_page_pass`]. (A
+    /// file whose strings are all valid UTF-8 is UTF-8; one with none is plainly the code page.)
+    pub(crate) fn needs_code_page_pass(&self) -> bool {
+        !self.code_page_only && self.ea_utf8 && self.ea_other
+    }
+
+    /// Decode one string value of group `code` and update the state from it. For single-byte code
+    /// pages a string that is valid UTF-8 is read as such, otherwise it is in the file's code page
+    /// (Windows-1252 for R2007+ files and files without a known code page). East Asian code pages
+    /// see [`Self::needs_code_page_pass`].
     pub(crate) fn string(&mut self, code: i32, raw: &[u8]) -> String {
         let d = if self.utf8 { Decoder::Utf8 } else { self.code_page.unwrap_or(Decoder::Utf8) };
-        let s = decode_text(raw, d).into_owned();
+        let s = if d.is_east_asian() {
+            if self.code_page_only {
+                decode(raw, d).into_owned()
+            } else {
+                if !raw.is_ascii() {
+                    if std::str::from_utf8(raw).is_ok() {
+                        self.ea_utf8 = true;
+                    } else {
+                        self.ea_other = true;
+                    }
+                }
+                decode_text(raw, d).into_owned()
+            }
+        } else {
+            decode_text(raw, d).into_owned()
+        };
         match (code, self.var.take()) {
             (9, _) => {
                 self.var = match s.trim() {

crates/dxf/src/lib.rs

@@ -378,8 +378,16 @@ pub fn parse_binary(b: &[u8]) -> Result<Vec<Tag>> {
 }
 
 fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
-    let mut c = Cur { b, p: 0 };
     let mut text = codepage::BinaryText::default();
+    let tags = parse_binary_pass(b, max_tags, &mut text)?;
+    if text.needs_code_page_pass() {
+        return parse_binary_pass(b, max_tags, &mut codepage::BinaryText::code_page_pass());
+    }
+    Ok(tags)
+}
+
+fn parse_binary_pass(b: &[u8], max_tags: usize, text: &mut codepage::BinaryText) -> Result<Vec<Tag>> {
+    let mut c = Cur { b, p: 0 };
     let mut tags = Vec::new();
     while c.p < b.len() {
         let mut code = i32::from(c.i16()?);

crates/dxf/tests/hardening.rs

@@ -165,3 +165,41 @@ fn binary_sixteen_and_thirty_two_bit_values_wrap_in_range_and_clamp_outside() {
     assert_eq!(back[5], Tag::i(90, i64::from(i32::MAX)));
     assert_eq!(back[6], Tag::i(90, i64::from(i32::MIN)));
 }
+
+/// Strings of a `$DWGCODEPAGE`-declared file as ASCII text and as a binary file.
+fn both_forms(code_page: &str, strings: &[&[u8]]) -> [Vec<Tag>; 2] {
+    let mut a = format!("9\n$DWGCODEPAGE\n3\n{code_page}\n").into_bytes();
+    for s in strings {
+        a.extend_from_slice(b"1\n");
+        a.extend_from_slice(s);
+        a.push(b'\n');
+    }
+    a.extend_from_slice(b"0\nEOF\n");
+    let b = bin_with(|b| {
+        for (code, s) in [(9i16, &b"$DWGCODEPAGE"[..]), (3, code_page.as_bytes())].into_iter().chain(strings.iter().map(|s| (1, *s))) {
+            b.extend_from_slice(&code.to_le_bytes());
+            b.extend_from_slice(s);
+            b.push(0);
+        }
+    });
+    [parse(&a).unwrap(), parse(&b).unwrap()]
+}
+
+#[test]
+fn east_asian_short_strings_that_look_like_utf8_stay_in_the_code_page() {
+    // C3 B1 is valid UTF-8 ("ñ") but is also two half-width katakana in Shift-JIS; the second
+    // string is plainly Shift-JIS, so the file is not UTF-8.
+    for t in both_forms("ANSI_932", &[b"\xC3\xB1", b"\x90\x7D\x96\xCA"]) {
+        assert_eq!(t[2].str(), "\u{FF83}\u{FF71}");
+        assert_eq!(t[3].str(), "図面");
+    }
+}
+
+#[test]
+fn east_asian_file_that_is_all_utf8_reads_as_utf8() {
+    for t in both_forms("ANSI_932", &["図面".as_bytes(), b"\xC3\xB1", b"plain"]) {
+        assert_eq!(t[2].str(), "図面");
+        assert_eq!(t[3].str(), "ñ");
+        assert_eq!(t[4].str(), "plain");
+    }
+}