From 28fe95fff00fef5d9f8232272219746d0556a4cb Mon Sep 17 00:00:00 2001 From: Rodrigo Gressler Date: Mon, 19 Oct 2026 16:36:13 -0300 Subject: [PATCH 1/4] DXF: save pre-2007 text in the file's code page instead of all \U+ escapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before 2007, DXF text is in the code page `$DWGCODEPAGE` names. Since #169 every non-ASCII character was written as a `\U+XXXX` escape, so a Portuguese drawing saved at 2000/2004 had layers, blocks and text such as `A\U+00E7\U+00E3o` in the file. Characters the declared code page has are now written as its bytes (`Ação` in ANSI_1252); the others are still escaped, and an unknown code page still gives pure ASCII. The DWG bridge is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/dxf/src/codepage.rs | 24 +++++++++ crates/dxf/src/lib.rs | 79 ++++++++++++++++++++--------- crates/dxf/tests/code_page_write.rs | 57 +++++++++++++++++++++ crates/io/src/dxf_write.rs | 8 +-- crates/io/src/lib.rs | 11 ++-- crates/io/tests/codepages.rs | 30 +++++++++++ crates/io/tests/dxf_versions.rs | 16 ++++-- crates/io/tests/followups_626.rs | 7 ++- docs/file-format-parity.md | 5 +- docs/localization-parity.md | 4 +- 10 files changed, 197 insertions(+), 44 deletions(-) create mode 100644 crates/dxf/tests/code_page_write.rs diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index 6f27b5be4..53f24c3b6 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -69,6 +69,30 @@ pub fn decode_code_page(bytes: &[u8], code_page: &str) -> Option { decoder(code_page).map(|d| decode(bytes, d).into_owned()) } +/// Append the bytes of the non-ASCII character `c` in the code page `d`, and return true, when +/// the code page has it; otherwise append nothing and return false (the caller escapes it). +pub(crate) fn encode_char(c: char, d: Decoder, out: &mut Vec) -> bool { + let mut buf = [0; 4]; + match d { + Decoder::Utf8 => out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()), + Decoder::Encoding(e) => { + let (bytes, _, unmappable) = e.encode(c.encode_utf8(&mut buf)); + // A code page byte that is ASCII would read back as another character, and a NUL, + // CR or LF would break the group, so only all-high-bit sequences or proper double-byte + // characters are kept (their trail byte may be ASCII, as in Shift-JIS). + if unmappable || bytes.is_empty() || !bytes.first().is_some_and(|b| *b >= 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { + return false; + } + out.extend_from_slice(&bytes); + } + Decoder::Table(t) => { + let Some(i) = t.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; + out.push(0x80 | i); + } + } + true +} + /// Decode bytes with a decoder. UTF-8 that isn't valid falls back to Windows-1252. pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { match d { diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 82ecf55e8..e37e3e13e 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -8,6 +8,7 @@ #![forbid(unsafe_code)] use std::fmt::Write as _; +use std::io::Write as _; mod version; pub use version::Version; @@ -409,18 +410,41 @@ pub fn write_ascii(tags: &[Tag]) -> String { /// characters as they are) or escaped as `\U+XXXX` (older versions, see [`write_ascii`]). Line /// breaks inside strings become spaces either way. pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { - let mut out = String::with_capacity(tags.len() * 16); + let bytes = ascii(tags, if utf8 { Strings::Utf8 } else { Strings::Escaped(None) }); + // Both forms are UTF-8 (the escaped one is ASCII). + String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) +} + +/// Write tags as pre-R2007 ASCII DXF with text in the code page `code_page` names (a +/// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: +/// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A +/// code page CADCraft doesn't know gives [`write_ascii`]'s pure ASCII output. +pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { + ascii(tags, Strings::Escaped(codepage::decoder(code_page))) +} + +/// How a writer stores string values. +#[derive(Clone, Copy)] +enum Strings { + /// As they are (R2007+). + Utf8, + /// ASCII as it is, other characters in this code page when it has them, otherwise as + /// `\U+XXXX` escapes (before R2007). + Escaped(Option), +} + +fn ascii(tags: &[Tag], strings: Strings) -> Vec { + let mut out = Vec::with_capacity(tags.len() * 16); for t in tags { let _ = write!(out, "{:>3}\r\n", t.code); match &t.value { - Value::Str(s) if utf8 => out.extend(s.chars().map(|c| if matches!(c, '\r' | '\n') { ' ' } else { c })), - Value::Str(s) => escape_unicode(s, &mut out), - Value::Real(r) => out.push_str(&format_real(*r)), - Value::Int(i) => out.push_str(&i.to_string()), - Value::Bool(b) => out.push_str(&i64::from(*b).to_string()), - Value::Hex(h) => out.push_str(h), + Value::Str(s) => push_string(s, strings, &mut out), + Value::Real(r) => out.extend_from_slice(format_real(*r).as_bytes()), + Value::Int(i) => out.extend_from_slice(i.to_string().as_bytes()), + Value::Bool(b) => out.extend_from_slice(i64::from(*b).to_string().as_bytes()), + Value::Hex(h) => out.extend_from_slice(h.as_bytes()), } - out.push_str("\r\n"); + out.extend_from_slice(b"\r\n"); } out } @@ -433,6 +457,16 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// in [`write_ascii`]. A tag's value is converted to its code's kind, so a number written as text /// is stored as a number; a code outside the 16-bit range is skipped. pub fn write_binary(tags: &[Tag], utf8: bool) -> Vec { + binary(tags, if utf8 { Strings::Utf8 } else { Strings::Escaped(None) }) +} + +/// Write tags as pre-R2007 binary DXF with text in the code page `code_page` names, as +/// [`write_ascii_in`] does. +pub fn write_binary_in(tags: &[Tag], code_page: &str) -> Vec { + binary(tags, Strings::Escaped(codepage::decoder(code_page))) +} + +fn binary(tags: &[Tag], strings: Strings) -> Vec { let mut out = Vec::with_capacity(BINARY_SENTINEL.len() + tags.len() * 12); out.extend_from_slice(BINARY_SENTINEL); for t in tags { @@ -445,13 +479,9 @@ pub fn write_binary(tags: &[Tag], utf8: bool) -> Vec { Value::Str(s) => s.clone(), v => v.as_str(), }; - let mut text = String::with_capacity(s.len()); - if utf8 { - text.extend(s.chars().map(|c| if matches!(c, '\r' | '\n') { ' ' } else { c })); - } else { - escape_unicode(&s, &mut text); - } - out.extend(text.bytes().filter(|&b| b != 0)); + let mut text = Vec::with_capacity(s.len()); + push_string(&s, strings, &mut text); + out.extend(text.into_iter().filter(|&b| b != 0)); out.push(0); } Kind::Real => { @@ -505,15 +535,18 @@ pub fn is_binary(bytes: &[u8]) -> bool { bytes.starts_with(BINARY_SENTINEL) } -/// Escape a string for a pre-R2007 DXF, whose text is in the `$DWGCODEPAGE` code page: ASCII -/// stays as it is and every other character becomes `\U+XXXX` (a surrogate pair of escapes -/// above U+FFFF), so the bytes are valid in any code page. Line breaks become spaces. -fn escape_unicode(s: &str, out: &mut String) { +/// A string value as `strings` stores it. Line breaks become spaces. Before R2007 the text is +/// in the `$DWGCODEPAGE` code page: ASCII stays as it is, a character the code page has becomes +/// its bytes, and every other one `\U+XXXX` (a surrogate pair of escapes above U+FFFF). Without +/// a code page the result is ASCII, so it is valid in any code page. +fn push_string(s: &str, strings: Strings, out: &mut Vec) { for c in s.chars() { - match c { - '\r' | '\n' => out.push(' '), - c if c.is_ascii() => out.push(c), - c => { + match (c, strings) { + ('\r' | '\n', _) => out.push(b' '), + (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, Strings::Escaped(Some(d))) if codepage::encode_char(c, d, out) => {} + (c, Strings::Escaped(_)) => { let mut units = [0u16; 2]; for u in c.encode_utf16(&mut units) { let _ = write!(out, "\\U+{u:04X}"); diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs new file mode 100644 index 000000000..316f2edd5 --- /dev/null +++ b/crates/dxf/tests/code_page_write.rs @@ -0,0 +1,57 @@ +//! Pre-R2007 DXF text is written in the `$DWGCODEPAGE` code page: characters it has as their +//! bytes, the others as `\U+XXXX` escapes; every form reads back to the same text. + +use cadcraft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_binary_in}; + +fn file(code_page: &str, text: &str) -> Vec { + vec![ + Tag::s(0, "SECTION"), + Tag::s(2, "HEADER"), + Tag::s(9, "$ACADVER"), + Tag::s(1, "AC1015"), + Tag::s(9, "$DWGCODEPAGE"), + Tag::s(3, code_page), + Tag::s(0, "ENDSEC"), + Tag::s(0, "SECTION"), + Tag::s(2, "ENTITIES"), + Tag::s(0, "TEXT"), + Tag::s(1, text), + Tag::s(0, "ENDSEC"), + Tag::s(0, "EOF"), + ] +} + +fn contains(bytes: &[u8], part: &[u8]) -> bool { + bytes.windows(part.len()).any(|w| w == part) +} + +#[test] +fn text_is_written_in_the_code_page() { + // (code page, text, the bytes written for it) + let cases: [(&str, &str, &[u8]); 6] = [ + ("ANSI_1252", "Ação € 图", b"A\xE7\xE3o \x80 \\U+56FE"), + ("ANSI_1251", "Слой é", b"\xD1\xEB\xEE\xE9 \\U+00E9"), + ("ANSI_932", "表図面 é", b"\x95\x5C\x90\x7D\x96\xCA \\U+00E9"), + ("DOS850", "Größe €", b"Gr\x94\xE1e \\U+20AC"), + ("ANSI_1252", "😀", b"\\U+D83D\\U+DE00"), + // A code page CADCraft doesn't know: everything non-ASCII is escaped. + ("ANSI_1361", "Ação", b"A\\U+00E7\\U+00E3o"), + ]; + for (cp, text, raw) in cases { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, raw), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} + +/// Without a known code page the output is what `write_ascii` writes, line breaks included. +#[test] +fn unknown_code_page_is_pure_ascii() { + let tags = file("EBCDIC", "Ação\r\nfim"); + let bytes = write_ascii_in(&tags, "EBCDIC"); + assert!(bytes.is_ascii()); + assert_eq!(bytes, write_ascii(&tags).into_bytes()); + assert!(contains(&bytes, b"A\\U+00E7\\U+00E3o fim")); +} diff --git a/crates/io/src/dxf_write.rs b/crates/io/src/dxf_write.rs index 521654e3d..ec4b8595f 100644 --- a/crates/io/src/dxf_write.rs +++ b/crates/io/src/dxf_write.rs @@ -1234,10 +1234,10 @@ fn entity(w: &mut W, d: &Drawing, e: &Entity, owner: &str, paper: bool, cx: &Ctx } } -/// `$DWGCODEPAGE`: before 2007 the code page read from the file when CADCraft knows it (the text -/// is ASCII with `\U+XXXX` escapes, right in any code page, but the file keeps its declared one); -/// otherwise, and from 2007 (UTF-8 text), `ANSI_1252`. -fn code_page(d: &Drawing, version: Version) -> String { +/// `$DWGCODEPAGE`: before 2007 the code page read from the file when CADCraft knows it, in which +/// a saved file's text is written (characters outside it as `\U+XXXX` escapes); otherwise, and +/// from 2007 (UTF-8 text), `ANSI_1252`. +pub(crate) fn code_page(d: &Drawing, version: Version) -> String { let read = d.header.get("DWGCODEPAGE").and_then(HVal::as_str).map(str::trim).unwrap_or_default(); let known = read.is_ascii() && read.len() <= 32 && cadcraft_dxf::decode_code_page(b"", read).is_some(); if version < Version::R2007 && known { read.to_string() } else { "ANSI_1252".into() } diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index ef56e6602..605298db9 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -88,10 +88,13 @@ pub fn write_with(d: &Drawing, name: &str, opts: &SaveOptions) -> Result match ext(name).as_str() { "dxf" | "" => { let tags = dxf_write::tags(d, version); - Ok(if opts.binary { - cadcraft_dxf::write_binary(&tags, version.utf8()) - } else { - cadcraft_dxf::write_ascii_as(&tags, version.utf8()).into_bytes() + // Before 2007 the text is in the code page the header declares. + let code_page = dxf_write::code_page(d, version); + Ok(match (opts.binary, version.utf8()) { + (true, true) => cadcraft_dxf::write_binary(&tags, true), + (true, false) => cadcraft_dxf::write_binary_in(&tags, &code_page), + (false, true) => cadcraft_dxf::write_ascii_as(&tags, true).into_bytes(), + (false, false) => cadcraft_dxf::write_ascii_in(&tags, &code_page), }) } // The DWG bridge reads our R2000 DXF and writes the DWG at the version asked for. Without diff --git a/crates/io/tests/codepages.rs b/crates/io/tests/codepages.rs index 40d85b3f4..53c22e908 100644 --- a/crates/io/tests/codepages.rs +++ b/crates/io/tests/codepages.rs @@ -116,6 +116,36 @@ fn code_page_text_survives_a_round_trip() { assert_eq!(names(&back), ("Слой".into(), "Блок".into(), "Привет".into())); } +/// Saving a pre-2007 DXF writes the text in the file's code page, as the DXF Reference +/// describes: a Portuguese drawing keeps `Elétrica`, `Ação` and `Função` as ANSI_1252 bytes +/// instead of `\U+00E9` escapes, a Cyrillic one its ANSI_1251 bytes, and characters the code +/// page lacks are still escaped. ASCII and binary, R2000 and R2004, read back the same. +#[test] +fn saving_writes_text_in_the_code_page() { + use cadcraft_io::{SaveOptions, Version}; + let pt = ascii_dxf("AC1015", "ANSI_1252", b"El\xE9trica", b"A\xE7\xE3o", b"Fun\xE7\xE3o \\U+56FE"); + let ru = ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2 \\U+00E9"); + let cases: [(&[u8], (&str, &str, &str), &[&[u8]]); 2] = [ + (&pt, ("Elétrica", "Ação", "Função 图"), &[b"El\xE9trica", b"A\xE7\xE3o", b"Fun\xE7\xE3o \\U+56FE"]), + (&ru, ("Слой", "Блок", "Привет é"), &[b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2 \\U+00E9"]), + ]; + for (file, want, raw) in cases { + let d = cadcraft_io::read_dxf(file).unwrap(); + assert_eq!(names(&d), (want.0.into(), want.1.into(), want.2.into())); + for version in [Version::R2000, Version::R2004] { + for binary in [false, true] { + let out = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary }).unwrap(); + for r in raw { + assert!(out.windows(r.len()).any(|w| w == *r), "{version} binary={binary}: {}", String::from_utf8_lossy(r)); + } + assert!(!out.windows(7).any(|w| w == b"\\U+00E7"), "{version} binary={binary}: ç is written as its byte"); + let back = cadcraft_io::read_dxf(&out).unwrap(); + assert_eq!(names(&back), (want.0.into(), want.1.into(), want.2.into()), "{version} binary={binary}"); + } + } + } +} + /// A binary DXF: R2000 strings are in the code page, R2007+ strings UTF-8. #[test] fn binary_dxf_honours_the_code_page() { diff --git a/crates/io/tests/dxf_versions.rs b/crates/io/tests/dxf_versions.rs index 3fc2f1c73..eafd989e9 100644 --- a/crates/io/tests/dxf_versions.rs +++ b/crates/io/tests/dxf_versions.rs @@ -87,9 +87,11 @@ fn dxf_at_every_version_reads_back_the_same() { assert_eq!(acadver(&bytes), v.code(), "{what}"); assert_eq!(cadcraft_dxf::is_binary(&bytes), binary, "{what}"); let raw = bytes.windows("图号".len()).any(|w| w == "图号".as_bytes()); - assert_eq!(raw, v.utf8(), "{what}: UTF-8 text from 2007, \\U+ escapes before"); - if !v.utf8() && !binary { - assert!(bytes.is_ascii(), "{what}"); + assert_eq!(raw, v.utf8(), "{what}: UTF-8 text from 2007, the code page before"); + if !v.utf8() { + // ANSI_1252 has the Latin characters; the others are escaped. + let has = |b: &[u8]| bytes.windows(b.len()).any(|w| w == b); + assert!(has(b"\\U+56FE\\U+53F7 Caf\xe9 \xd8 \\U+D83D\\U+DE00"), "{what}"); } let back = read(&bytes, "x.dxf").unwrap(); assert_eq!(first_text(&back).as_deref(), Some(INTL), "{what}"); @@ -99,11 +101,15 @@ fn dxf_at_every_version_reads_back_the_same() { } } -/// R2000 ASCII is exactly the R2000 DXF CADCraft always wrote. +/// R2000 ASCII is the R2000 DXF CADCraft always wrote, tag for tag; only the text is in the +/// code page instead of all `\U+` escapes. #[test] fn r2000_ascii_is_unchanged() { let d = sample(); - assert_eq!(write_with(&d, "x.dxf", &opts(Version::R2000, false)).unwrap(), write_dxf(&d).into_bytes()); + let saved = write_with(&d, "x.dxf", &opts(Version::R2000, false)).unwrap(); + let escaped = write_dxf(&d); + assert_ne!(saved, escaped.as_bytes(), "Café is written in ANSI_1252"); + assert_eq!(cadcraft_dxf::parse(&saved).unwrap(), cadcraft_dxf::parse(escaped.as_bytes()).unwrap()); } /// Without a version asked for: 2018 for a new drawing, otherwise the file's own version (the diff --git a/crates/io/tests/followups_626.rs b/crates/io/tests/followups_626.rs index a44a27eb4..10e355cd3 100644 --- a/crates/io/tests/followups_626.rs +++ b/crates/io/tests/followups_626.rs @@ -177,10 +177,9 @@ fn code_page_is_kept_before_2007() { .unwrap(); for v in [Version::R2000, Version::R2004] { let bytes = save(&d, "x.dxf", v); - assert!(bytes.is_ascii(), "{v:?}: ASCII with escapes"); - let text = String::from_utf8(bytes).unwrap(); - assert_eq!(code_page(&text), "ANSI_1251", "{v:?}"); - let back = cadcraft_io::read_dxf(text.as_bytes()).unwrap(); + assert!(bytes.windows(6).any(|w| w == b"\xCF\xF0\xE8\xE2\xE5\xF2"), "{v:?}: the text in ANSI_1251"); + assert_eq!(code_page(&String::from_utf8_lossy(&bytes)), "ANSI_1251", "{v:?}"); + let back = cadcraft_io::read_dxf(&bytes).unwrap(); assert_eq!(back.header.str("DWGCODEPAGE", ""), "ANSI_1251"); assert!(back.model.iter().any(|e| matches!(&e.kind, EntityKind::Text(t) if t.value == "Привет"))); } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c598193d7..122b654a8 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-17 · **Change:** minor (DWG read leaves out orphaned DGN line style components) · **Target:** Autodesk AutoCAD 2027 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-19 · **Change:** minor (pre-2007 DXF text saved in its code page) · **Target:** Autodesk AutoCAD 2027 Every format AutoCAD 2027 reads or writes, with CADCraft's support, fidelity and tests. Work items are in [gaps.md](gaps.md#file-formats); the weighted score feeds @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure (later versions add only optional groups) with that `$ACADVER`; UTF-8 text from 2007; no CLASS instance counts (91) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | writes `\U+` escapes (#169), which is correct; declares `ANSI_1252` | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page gives pure ASCII with escapes (#169) | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type @@ -138,6 +138,7 @@ our output in AutoCAD black-box. | Date | Change | Summary | |---|---|---| +| 2026-10-19 | minor | Pre-2007 DXF saves write text in the file's code page (`Ação` as ANSI_1252 bytes, not `A\U+00E7\U+00E3o`); characters outside it stay `\U+` escapes | | 2026-10-17 | minor | DWG read: DGN line style components no line style definition reaches are left out before the DXF bridge, with a read note; an 83 MB site plan with 888,483 of them (1.2 GB of DXF, over the limit) now opens (#1482) | | 2026-10-12 | minor | PDF output: object transparency plotted as PDF alpha (`/ExtGState` `/ca` `/CA`) when the page setup's or the plot's "Plot transparency" is on; PNG plots blend it, SVG uses `opacity`; plots without it are unchanged byte for byte | | 2026-10-12 | minor | DXF/DWG save: the Model layout's plot settings are written as CADCraft plots model space, drawing extents scaled to fit (PLOTSETTINGS 74 = 1, 75 = 0), instead of plot type 5 "Layout information", which other programs' Plot dialogs can't use for model space | diff --git a/docs/localization-parity.md b/docs/localization-parity.md index 3dbd01e22..5c0574616 100644 --- a/docs/localization-parity.md +++ b/docs/localization-parity.md @@ -31,8 +31,8 @@ shaping and right-to-left text through TrueType, vertical SHX text styles. English. For scale: about 1,160 unique capitalised string literals exist in `crates/engine` and `crates/ui-egui` (grep, 2026-10-10). - **Drawing text:** TrueType/OpenType/`.ttc` through `skrifa` with per-character font fallback - and FONTALT, so CJK characters render (#173). DXF text is written with `\U+` escapes so - non-ASCII survives in R2000 files (#169). Lines with right-to-left text get the Unicode + and FONTALT, so CJK characters render (#173). Pre-2007 DXF text is written in the file's code page, + with `\U+` escapes for characters outside it, so non-ASCII survives in R2000 files (#169). Lines with right-to-left text get the Unicode Bidirectional Algorithm and OpenType shaping with `harfrust` (Arabic, Persian and Urdu joining, lam-alef, ZWNJ, marks; Hebrew), in TEXT and MTEXT including wrapping (#287). Left-to-right lines are still placed one glyph per code point: **no shaping** for Devanagari and other Indic From 936f08db1feba5515f5ea7d51a9d3a37dda3aecd Mon Sep 17 00:00:00 2001 From: Rodrigo Gressler Date: Mon, 19 Oct 2026 22:25:35 -0300 Subject: [PATCH 2/4] DXF: keep files that declare ASCII or UTF8 pure ASCII when saving A pre-2007 file whose header says $DWGCODEPAGE ASCII (or UTF8/UNICODE, which no pre-2007 reader expects) is still written with \U+ escapes only, instead of Windows-1252 (or UTF-8) bytes the header doesn't declare. Review of #1693. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/dxf/src/codepage.rs | 14 +++++++++++++- crates/dxf/src/lib.rs | 7 ++++--- crates/dxf/tests/code_page_write.rs | 18 +++++++++++------- crates/io/tests/codepages.rs | 19 +++++++++++++++++++ 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index 53f24c3b6..9377404d9 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -69,12 +69,24 @@ pub fn decode_code_page(bytes: &[u8], code_page: &str) -> Option { decoder(code_page).map(|d| decode(bytes, d).into_owned()) } +/// The code page a pre-2007 file's text is written in, or `None` for pure ASCII with escapes: +/// when CADCraft doesn't know `code_page`, and for `ASCII` (the reader takes such files as +/// Windows-1252, but a writer must keep them ASCII) and `UTF8`/`UNICODE`, which no pre-2007 +/// reader expects. +pub(crate) fn encoder(code_page: &str) -> Option { + if code_page.trim().eq_ignore_ascii_case("ASCII") { + return None; + } + decoder(code_page).filter(|d| !matches!(d, Decoder::Utf8)) +} + /// Append the bytes of the non-ASCII character `c` in the code page `d`, and return true, when /// the code page has it; otherwise append nothing and return false (the caller escapes it). pub(crate) fn encode_char(c: char, d: Decoder, out: &mut Vec) -> bool { let mut buf = [0; 4]; match d { - Decoder::Utf8 => out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()), + // Not a pre-2007 code page ([`encoder`] never gives it): escaped. + Decoder::Utf8 => return false, Decoder::Encoding(e) => { let (bytes, _, unmappable) = e.encode(c.encode_utf8(&mut buf)); // A code page byte that is ASCII would read back as another character, and a NUL, diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index e37e3e13e..0bbb4ee68 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -418,9 +418,10 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// Write tags as pre-R2007 ASCII DXF with text in the code page `code_page` names (a /// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: /// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A -/// code page CADCraft doesn't know gives [`write_ascii`]'s pure ASCII output. +/// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) +/// give [`write_ascii`]'s pure ASCII output. pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { - ascii(tags, Strings::Escaped(codepage::decoder(code_page))) + ascii(tags, Strings::Escaped(codepage::encoder(code_page))) } /// How a writer stores string values. @@ -463,7 +464,7 @@ pub fn write_binary(tags: &[Tag], utf8: bool) -> Vec { /// Write tags as pre-R2007 binary DXF with text in the code page `code_page` names, as /// [`write_ascii_in`] does. pub fn write_binary_in(tags: &[Tag], code_page: &str) -> Vec { - binary(tags, Strings::Escaped(codepage::decoder(code_page))) + binary(tags, Strings::Escaped(codepage::encoder(code_page))) } fn binary(tags: &[Tag], strings: Strings) -> Vec { diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 316f2edd5..67e7e7e57 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -46,12 +46,16 @@ fn text_is_written_in_the_code_page() { } } -/// Without a known code page the output is what `write_ascii` writes, line breaks included. +/// Without a known code page, and for `ASCII`, `UTF8` and `UNICODE` headers, the output is what +/// `write_ascii` writes (pure ASCII with escapes), line breaks included. #[test] -fn unknown_code_page_is_pure_ascii() { - let tags = file("EBCDIC", "Ação\r\nfim"); - let bytes = write_ascii_in(&tags, "EBCDIC"); - assert!(bytes.is_ascii()); - assert_eq!(bytes, write_ascii(&tags).into_bytes()); - assert!(contains(&bytes, b"A\\U+00E7\\U+00E3o fim")); +fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { + for cp in ["EBCDIC", "ASCII", "ascii ", "UTF8", "UNICODE"] { + let tags = file(cp, "Ação\r\nfim"); + let bytes = write_ascii_in(&tags, cp); + assert!(bytes.is_ascii(), "{cp}"); + assert_eq!(bytes, write_ascii(&tags).into_bytes(), "{cp}"); + assert!(contains(&bytes, b"A\\U+00E7\\U+00E3o fim"), "{cp}"); + assert!(write_binary_in(&tags, cp).iter().all(u8::is_ascii), "{cp} binary"); + } } diff --git a/crates/io/tests/codepages.rs b/crates/io/tests/codepages.rs index 53c22e908..56f9af80f 100644 --- a/crates/io/tests/codepages.rs +++ b/crates/io/tests/codepages.rs @@ -146,6 +146,25 @@ fn saving_writes_text_in_the_code_page() { } } +/// A file whose header declares `ASCII` keeps that code page and stays ASCII when saved: its +/// non-ASCII text is written as `\U+` escapes, not as Windows-1252 bytes. +#[test] +fn saving_an_ascii_file_keeps_it_ascii() { + use cadcraft_io::{SaveOptions, Version}; + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ASCII", b"El\\U+00E9trica", b"B", b"Eletrica \\U+00E9")).unwrap(); + assert_eq!(names(&d), ("Elétrica".into(), "B".into(), "Eletrica é".into())); + for version in [Version::R2000, Version::R2004] { + let ascii = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary: false }).unwrap(); + assert!(ascii.is_ascii(), "{version}"); + let binary = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary: true }).unwrap(); + for out in [&ascii, &binary] { + assert!(out.windows(14).any(|w| w == b"El\\U+00E9trica"), "{version}"); + assert!(out.windows(5).any(|w| w == b"ASCII"), "{version}: the header keeps its code page"); + assert_eq!(names(&cadcraft_io::read_dxf(out).unwrap()), ("Elétrica".into(), "B".into(), "Eletrica é".into()), "{version}"); + } + } +} + /// A binary DXF: R2000 strings are in the code page, R2007+ strings UTF-8. #[test] fn binary_dxf_honours_the_code_page() { From 0646dca86ef4624a6665b04025acaaa33e1a5c44 Mon Sep 17 00:00:00 2001 From: ffishman <20211677+ffishman@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:05:36 -0300 Subject: [PATCH 3/4] DXF: escape ASCII output whose code page bytes are valid UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ASCII DXF that is valid UTF-8 is read as UTF-8, so pre-2007 text such as СІ in ANSI_1251 (D1 B2) came back as U+0472. write_ascii_in now falls back to \U+ escapes for such a file; binary DXF, read by its header's code page, keeps the bytes. Reported by CodeRabbit on #1693. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/dxf/src/lib.rs | 10 ++++++++-- crates/dxf/tests/code_page_write.rs | 17 +++++++++++++++++ docs/file-format-parity.md | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 0bbb4ee68..72f834978 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -419,9 +419,15 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: /// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A /// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) -/// give [`write_ascii`]'s pure ASCII output. +/// give [`write_ascii`]'s pure ASCII output. So does a file whose code page bytes happen to be +/// valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, U+0472), since readers, ours included, take an ASCII +/// DXF that is valid UTF-8 as UTF-8. pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { - ascii(tags, Strings::Escaped(codepage::encoder(code_page))) + let bytes = ascii(tags, Strings::Escaped(codepage::encoder(code_page))); + if !bytes.is_ascii() && std::str::from_utf8(&bytes).is_ok() { + return ascii(tags, Strings::Escaped(None)); + } + bytes } /// How a writer stores string values. diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 67e7e7e57..bec95b73a 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -59,3 +59,20 @@ fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { assert!(write_binary_in(&tags, cp).iter().all(u8::is_ascii), "{cp} binary"); } } + +/// Code page bytes that happen to form valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, which is U+0472 +/// in UTF-8) would be read back as UTF-8 from an ASCII file, so that file is written with +/// escapes. Binary DXF reads strings by the header's code page and keeps the bytes. +#[test] +fn code_page_bytes_that_are_valid_utf8_are_escaped_in_ascii() { + let tags = file("ANSI_1251", "СІ"); + let ascii = write_ascii_in(&tags, "ANSI_1251"); + assert!(ascii.is_ascii(), "{:?}", String::from_utf8_lossy(&ascii)); + assert!(contains(&ascii, b"\\U+0421\\U+0406")); + assert_eq!(parse(&ascii).unwrap()[10].str(), "СІ"); + let binary = write_binary_in(&tags, "ANSI_1251"); + assert!(contains(&binary, b"\xD1\xB2")); + assert_eq!(parse(&binary).unwrap()[10].str(), "СІ"); + // Bytes that are not valid UTF-8 stay as they are. + assert!(contains(&write_ascii_in(&file("ANSI_1251", "Слой"), "ANSI_1251"), b"\xD1\xEB\xEE\xE9")); +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 6dd6f783f..d919cf6ba 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure (later versions add only optional groups) with that `$ACADVER`; UTF-8 text from 2007; no CLASS instance counts (91) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page gives pure ASCII with escapes (#169) | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169) | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type From 1e96a3d50c2e955da294a0d2dbf2bdb53c40d9f5 Mon Sep 17 00:00:00 2001 From: ffishman <20211677+ffishman@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:13:10 -0300 Subject: [PATCH 4/4] DXF: never let a trail byte 0x5C start an escape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Shift-JIS, Big5 and GBK a double-byte character's trail byte can be 0x5C, the backslash (表 95 5C, ソ 83 5C, 許 B3 5C, 乗 81 5C). Written as its bytes right before U+ or M+, the file holds a backslash byte followed by U+, an escape for a reader that looks for escapes before it decodes the code page. CADCraft decodes first, so it read back right, but such a character there is now written as a \U+XXXX escape and the file reads the same either way. Elsewhere it stays as its bytes. ANSI_949 has no such trail byte. Suggested by austin62jake on #1693. --- crates/dxf/src/lib.rs | 23 ++++++++++++++++++++--- crates/dxf/tests/code_page_write.rs | 22 ++++++++++++++++++++++ docs/file-format-parity.md | 2 +- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index ec350adaa..7f40c8f32 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -547,18 +547,21 @@ pub fn is_binary(bytes: &[u8]) -> bool { /// its bytes, and every other one `\U+XXXX` (a surrogate pair of escapes above U+FFFF). Without /// a code page the result is ASCII, so it is valid in any code page. In every form a backslash /// that would start a `\U+` or `\M+` sequence is written as `\U+005C`, so the reader gives back -/// the literal text (#2050). +/// the literal text (#2050). So is a double-byte character whose trail byte is 0x5C (`\`) in that +/// place, such as Shift-JIS `表` or Big5 `許`: CADCraft decodes the code page before it looks for +/// escapes, but a reader that looks first would take that byte and the `U+` after it for one. fn push_string(s: &str, strings: Strings, out: &mut Vec) { let chars: Vec = s.chars().collect(); + let escape_at = |j: usize| matches!(chars.get(j), Some('U' | 'M')) && chars.get(j + 1) == Some(&'+'); for (i, &c) in chars.iter().enumerate() { match (c, strings) { ('\r' | '\n', _) => out.push(b' '), - ('\\', _) if matches!(chars.get(i + 1), Some('U' | 'M')) && chars.get(i + 2) == Some(&'+') => { + ('\\', _) if escape_at(i + 1) => { out.extend_from_slice(b"\\U+005C"); } (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), - (c, Strings::Escaped(Some(d))) if codepage::encode_char(c, d, out) => {} + (c, Strings::Escaped(Some(d))) if code_page_bytes(c, d, escape_at(i + 1), out) => {} (c, Strings::Escaped(_)) => { let mut units = [0u16; 2]; for u in c.encode_utf16(&mut units) { @@ -569,6 +572,20 @@ fn push_string(s: &str, strings: Strings, out: &mut Vec) { } } +/// `c` in the code page `d`, if it has it, unless it is a double-byte character whose trail byte +/// is 0x5C and an escape's `U+` or `M+` follows (`before_escape`): see [`push_string`]. +fn code_page_bytes(c: char, d: codepage::Decoder, before_escape: bool, out: &mut Vec) -> bool { + let start = out.len(); + if !codepage::encode_char(c, d, out) { + return false; + } + if before_escape && out.len() - start > 1 && out.last() == Some(&b'\\') { + out.truncate(start); + return false; + } + true +} + /// A section: `0 SECTION / 2 NAME ... 0 ENDSEC`. #[derive(Clone, Debug, Default)] pub struct Section { diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 466a2b2a6..75d647c08 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -92,3 +92,25 @@ fn literal_escapes_survive_code_page_text() { } } } + +/// Double-byte code pages whose trail byte can be 0x5C (`\\`): Shift-JIS `表` (95 5C) and `ソ` +/// (83 5C), Big5 `許` (B3 5C), GBK `乗` (81 5C). Right before `U+` or `M+`, such a character is +/// written as an escape, so even a reader that looks for escapes before it decodes the code page +/// reads it right. Elsewhere it stays as its bytes. Both forms read back as written. +#[test] +fn a_trail_byte_backslash_never_starts_an_escape() { + let cases: [(&str, &str, &[u8], &[u8]); 4] = [ + ("ANSI_932", "表U+0041 表示", b"\\U+8868U+0041 \x95\x5C\x8E\xA6", b"\x95\x5CU+"), + ("ANSI_932", "ソM+1A2B", b"\\U+30BDM+1A2B", b"\x83\x5CM+"), + ("ANSI_950", "許U+0041", b"\\U+8A31U+0041", b"\xB3\x5CU+"), + ("ANSI_936", "乗U+0041", b"\\U+4E57U+0041", b"\x81\x5CU+"), + ]; + for (cp, text, written, never) in cases { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, written), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert!(!contains(&bytes, never), "{cp} {what}: a trail byte starts an escape"); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index d919cf6ba..eda4a0872 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure (later versions add only optional groups) with that `$ACADVER`; UTF-8 text from 2007; no CLASS instance counts (91) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169) | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169); a double-byte character whose trail byte is `\` (Shift-JIS, Big5, GBK) is escaped where `U+` or `M+` follows it | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type