From 0646dca86ef4624a6665b04025acaaa33e1a5c44 Mon Sep 17 00:00:00 2001 From: ffishman <20211677+ffishman@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:05:36 -0300 Subject: [PATCH] DXF: escape ASCII output whose code page bytes are valid UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ASCII DXF that is valid UTF-8 is read as UTF-8, so pre-2007 text such as СІ in ANSI_1251 (D1 B2) came back as U+0472. write_ascii_in now falls back to \U+ escapes for such a file; binary DXF, read by its header's code page, keeps the bytes. Reported by CodeRabbit on #1693. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/dxf/src/lib.rs | 10 ++++++++-- crates/dxf/tests/code_page_write.rs | 17 +++++++++++++++++ docs/file-format-parity.md | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 0bbb4ee68..72f834978 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -419,9 +419,15 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: /// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A /// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) -/// give [`write_ascii`]'s pure ASCII output. +/// give [`write_ascii`]'s pure ASCII output. So does a file whose code page bytes happen to be +/// valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, U+0472), since readers, ours included, take an ASCII +/// DXF that is valid UTF-8 as UTF-8. pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { - ascii(tags, Strings::Escaped(codepage::encoder(code_page))) + let bytes = ascii(tags, Strings::Escaped(codepage::encoder(code_page))); + if !bytes.is_ascii() && std::str::from_utf8(&bytes).is_ok() { + return ascii(tags, Strings::Escaped(None)); + } + bytes } /// How a writer stores string values. diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 67e7e7e57..bec95b73a 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -59,3 +59,20 @@ fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { assert!(write_binary_in(&tags, cp).iter().all(u8::is_ascii), "{cp} binary"); } } + +/// Code page bytes that happen to form valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, which is U+0472 +/// in UTF-8) would be read back as UTF-8 from an ASCII file, so that file is written with +/// escapes. Binary DXF reads strings by the header's code page and keeps the bytes. +#[test] +fn code_page_bytes_that_are_valid_utf8_are_escaped_in_ascii() { + let tags = file("ANSI_1251", "СІ"); + let ascii = write_ascii_in(&tags, "ANSI_1251"); + assert!(ascii.is_ascii(), "{:?}", String::from_utf8_lossy(&ascii)); + assert!(contains(&ascii, b"\\U+0421\\U+0406")); + assert_eq!(parse(&ascii).unwrap()[10].str(), "СІ"); + let binary = write_binary_in(&tags, "ANSI_1251"); + assert!(contains(&binary, b"\xD1\xB2")); + assert_eq!(parse(&binary).unwrap()[10].str(), "СІ"); + // Bytes that are not valid UTF-8 stay as they are. + assert!(contains(&write_ascii_in(&file("ANSI_1251", "Слой"), "ANSI_1251"), b"\xD1\xEB\xEE\xE9")); +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6dd6f783f..d919cf6ba 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure (later versions add only optional groups) with that `$ACADVER`; UTF-8 text from 2007; no CLASS instance counts (91) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page gives pure ASCII with escapes (#169) | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169) | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type