diff --git a/crates/dxf/src/codepage.rs b/crates/dxf/src/codepage.rs index 6f27b5be4..9377404d9 100644 --- a/crates/dxf/src/codepage.rs +++ b/crates/dxf/src/codepage.rs @@ -69,6 +69,42 @@ pub fn decode_code_page(bytes: &[u8], code_page: &str) -> Option { decoder(code_page).map(|d| decode(bytes, d).into_owned()) } +/// The code page a pre-2007 file's text is written in, or `None` for pure ASCII with escapes: +/// when CADCraft doesn't know `code_page`, and for `ASCII` (the reader takes such files as +/// Windows-1252, but a writer must keep them ASCII) and `UTF8`/`UNICODE`, which no pre-2007 +/// reader expects. +pub(crate) fn encoder(code_page: &str) -> Option { + if code_page.trim().eq_ignore_ascii_case("ASCII") { + return None; + } + decoder(code_page).filter(|d| !matches!(d, Decoder::Utf8)) +} + +/// Append the bytes of the non-ASCII character `c` in the code page `d`, and return true, when +/// the code page has it; otherwise append nothing and return false (the caller escapes it). +pub(crate) fn encode_char(c: char, d: Decoder, out: &mut Vec) -> bool { + let mut buf = [0; 4]; + match d { + // Not a pre-2007 code page ([`encoder`] never gives it): escaped. + Decoder::Utf8 => return false, + Decoder::Encoding(e) => { + let (bytes, _, unmappable) = e.encode(c.encode_utf8(&mut buf)); + // A code page byte that is ASCII would read back as another character, and a NUL, + // CR or LF would break the group, so only all-high-bit sequences or proper double-byte + // characters are kept (their trail byte may be ASCII, as in Shift-JIS). + if unmappable || bytes.is_empty() || !bytes.first().is_some_and(|b| *b >= 0x80) || bytes.iter().any(|b| matches!(b, 0 | b'\r' | b'\n')) { + return false; + } + out.extend_from_slice(&bytes); + } + Decoder::Table(t) => { + let Some(i) = t.iter().position(|&x| x == c).and_then(|i| u8::try_from(i).ok()) else { return false }; + out.push(0x80 | i); + } + } + true +} + /// Decode bytes with a decoder. UTF-8 that isn't valid falls back to Windows-1252. pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> { match d { diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index b9e394766..953043cb9 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -8,6 +8,7 @@ #![forbid(unsafe_code)] use std::fmt::Write as _; +use std::io::Write as _; mod version; pub use version::Version; @@ -421,17 +422,48 @@ pub fn write_ascii(tags: &[Tag]) -> String { /// characters as they are) or escaped as `\U+XXXX` (older versions, see [`write_ascii`]). Line /// breaks inside strings become spaces either way. pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { - let mut out = String::with_capacity(tags.len() * 16); + let bytes = ascii(tags, if utf8 { Strings::Utf8 } else { Strings::Escaped(None) }); + // Both forms are UTF-8 (the escaped one is ASCII). + String::from_utf8(bytes).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) +} + +/// Write tags as pre-R2007 ASCII DXF with text in the code page `code_page` names (a +/// `$DWGCODEPAGE` value such as `ANSI_1252`), as the DXF Reference describes for those versions: +/// a character the code page has is written as its bytes, any other as a `\U+XXXX` escape. A +/// code page CADCraft doesn't know, `ASCII`, and `UTF8`/`UNICODE` (not a pre-2007 code page) +/// give [`write_ascii`]'s pure ASCII output. So does a file whose code page bytes happen to be +/// valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, U+0472), since readers, ours included, take an ASCII +/// DXF that is valid UTF-8 as UTF-8. +pub fn write_ascii_in(tags: &[Tag], code_page: &str) -> Vec { + let bytes = ascii(tags, Strings::Escaped(codepage::encoder(code_page))); + if !bytes.is_ascii() && std::str::from_utf8(&bytes).is_ok() { + return ascii(tags, Strings::Escaped(None)); + } + bytes +} + +/// How a writer stores string values. +#[derive(Clone, Copy)] +enum Strings { + /// As they are (R2007+). + Utf8, + /// ASCII as it is, other characters in this code page when it has them, otherwise as + /// `\U+XXXX` escapes (before R2007). + Escaped(Option), +} + +fn ascii(tags: &[Tag], strings: Strings) -> Vec { + let mut out = Vec::with_capacity(tags.len() * 16); for t in tags { let _ = write!(out, "{:>3}\r\n", t.code); match &t.value { - Value::Str(s) => escape_unicode(s, utf8, &mut out), - Value::Real(r) => out.push_str(&format_real(*r)), - Value::Int(i) => out.push_str(&i.to_string()), - Value::Bool(b) => out.push_str(&i64::from(*b).to_string()), - Value::Hex(h) => out.push_str(h), + Value::Str(s) => push_string(s, strings, &mut out), + Value::Real(r) => out.extend_from_slice(format_real(*r).as_bytes()), + Value::Int(i) => out.extend_from_slice(i.to_string().as_bytes()), + Value::Bool(b) => out.extend_from_slice(i64::from(*b).to_string().as_bytes()), + Value::Hex(h) => out.extend_from_slice(h.as_bytes()), } - out.push_str("\r\n"); + out.extend_from_slice(b"\r\n"); } out } @@ -444,6 +476,16 @@ pub fn write_ascii_as(tags: &[Tag], utf8: bool) -> String { /// in [`write_ascii`]. A tag's value is converted to its code's kind, so a number written as text /// is stored as a number; a code outside the 16-bit range is skipped. pub fn write_binary(tags: &[Tag], utf8: bool) -> Vec { + binary(tags, if utf8 { Strings::Utf8 } else { Strings::Escaped(None) }) +} + +/// Write tags as pre-R2007 binary DXF with text in the code page `code_page` names, as +/// [`write_ascii_in`] does. +pub fn write_binary_in(tags: &[Tag], code_page: &str) -> Vec { + binary(tags, Strings::Escaped(codepage::encoder(code_page))) +} + +fn binary(tags: &[Tag], strings: Strings) -> Vec { let mut out = Vec::with_capacity(BINARY_SENTINEL.len() + tags.len() * 12); out.extend_from_slice(BINARY_SENTINEL); for t in tags { @@ -456,9 +498,9 @@ pub fn write_binary(tags: &[Tag], utf8: bool) -> Vec { Value::Str(s) => s.clone(), v => v.as_str(), }; - let mut text = String::with_capacity(s.len()); - escape_unicode(&s, utf8, &mut text); - out.extend(text.bytes().filter(|&b| b != 0)); + let mut text = Vec::with_capacity(s.len()); + push_string(&s, strings, &mut text); + out.extend(text.into_iter().filter(|&b| b != 0)); out.push(0); } Kind::Real => { @@ -512,19 +554,27 @@ pub fn is_binary(bytes: &[u8]) -> bool { bytes.starts_with(BINARY_SENTINEL) } -/// Escape a string for a pre-R2007 DXF, whose text is in the `$DWGCODEPAGE` code page: ASCII -/// stays as it is and every other character becomes `\U+XXXX` (a surrogate pair of escapes -/// above U+FFFF), so the bytes are valid in any code page; with `utf8` they stay as they are. -/// A backslash that would start a `\U+` or `\M+` sequence is written as `\U+005C` so the reader -/// gives back the literal text. Line breaks become spaces. -fn escape_unicode(s: &str, utf8: bool, out: &mut String) { +/// A string value as `strings` stores it. Line breaks become spaces. Before R2007 the text is +/// in the `$DWGCODEPAGE` code page: ASCII stays as it is, a character the code page has becomes +/// its bytes, and every other one `\U+XXXX` (a surrogate pair of escapes above U+FFFF). Without +/// a code page the result is ASCII, so it is valid in any code page. In every form a backslash +/// that would start a `\U+` or `\M+` sequence is written as `\U+005C`, so the reader gives back +/// the literal text (#2050). So is a double-byte character whose trail byte is 0x5C (`\`) in that +/// place, such as Shift-JIS `表` or Big5 `許`: CADCraft decodes the code page before it looks for +/// escapes, but a reader that looks first would take that byte and the `U+` after it for one. +fn push_string(s: &str, strings: Strings, out: &mut Vec) { let chars: Vec = s.chars().collect(); + let escape_at = |j: usize| matches!(chars.get(j), Some('U' | 'M')) && chars.get(j + 1) == Some(&'+'); for (i, &c) in chars.iter().enumerate() { - match c { - '\r' | '\n' => out.push(' '), - '\\' if matches!(chars.get(i + 1), Some('U' | 'M')) && chars.get(i + 2) == Some(&'+') => out.push_str("\\U+005C"), - c if utf8 || c.is_ascii() => out.push(c), - c => { + match (c, strings) { + ('\r' | '\n', _) => out.push(b' '), + ('\\', _) if escape_at(i + 1) => { + out.extend_from_slice(b"\\U+005C"); + } + (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), + (c, Strings::Escaped(Some(d))) if code_page_bytes(c, d, escape_at(i + 1), out) => {} + (c, Strings::Escaped(_)) => { let mut units = [0u16; 2]; for u in c.encode_utf16(&mut units) { let _ = write!(out, "\\U+{u:04X}"); @@ -534,6 +584,20 @@ fn escape_unicode(s: &str, utf8: bool, out: &mut String) { } } +/// `c` in the code page `d`, if it has it, unless it is a double-byte character whose trail byte +/// is 0x5C and an escape's `U+` or `M+` follows (`before_escape`): see [`push_string`]. +fn code_page_bytes(c: char, d: codepage::Decoder, before_escape: bool, out: &mut Vec) -> bool { + let start = out.len(); + if !codepage::encode_char(c, d, out) { + return false; + } + if before_escape && out.len() - start > 1 && out.last() == Some(&b'\\') { + out.truncate(start); + return false; + } + true +} + /// A section: `0 SECTION / 2 NAME ... 0 ENDSEC`. #[derive(Clone, Debug, Default)] pub struct Section { diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs new file mode 100644 index 000000000..75d647c08 --- /dev/null +++ b/crates/dxf/tests/code_page_write.rs @@ -0,0 +1,116 @@ +//! Pre-R2007 DXF text is written in the `$DWGCODEPAGE` code page: characters it has as their +//! bytes, the others as `\U+XXXX` escapes; every form reads back to the same text. + +use cadcraft_dxf::{Tag, parse, write_ascii, write_ascii_in, write_binary_in}; + +fn file(code_page: &str, text: &str) -> Vec { + vec![ + Tag::s(0, "SECTION"), + Tag::s(2, "HEADER"), + Tag::s(9, "$ACADVER"), + Tag::s(1, "AC1015"), + Tag::s(9, "$DWGCODEPAGE"), + Tag::s(3, code_page), + Tag::s(0, "ENDSEC"), + Tag::s(0, "SECTION"), + Tag::s(2, "ENTITIES"), + Tag::s(0, "TEXT"), + Tag::s(1, text), + Tag::s(0, "ENDSEC"), + Tag::s(0, "EOF"), + ] +} + +fn contains(bytes: &[u8], part: &[u8]) -> bool { + bytes.windows(part.len()).any(|w| w == part) +} + +#[test] +fn text_is_written_in_the_code_page() { + // (code page, text, the bytes written for it) + let cases: [(&str, &str, &[u8]); 6] = [ + ("ANSI_1252", "Ação € 图", b"A\xE7\xE3o \x80 \\U+56FE"), + ("ANSI_1251", "Слой é", b"\xD1\xEB\xEE\xE9 \\U+00E9"), + ("ANSI_932", "表図面 é", b"\x95\x5C\x90\x7D\x96\xCA \\U+00E9"), + ("DOS850", "Größe €", b"Gr\x94\xE1e \\U+20AC"), + ("ANSI_1252", "😀", b"\\U+D83D\\U+DE00"), + // A code page CADCraft doesn't know: everything non-ASCII is escaped. + ("ANSI_1361", "Ação", b"A\\U+00E7\\U+00E3o"), + ]; + for (cp, text, raw) in cases { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, raw), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} + +/// Without a known code page, and for `ASCII`, `UTF8` and `UNICODE` headers, the output is what +/// `write_ascii` writes (pure ASCII with escapes), line breaks included. +#[test] +fn unknown_ascii_and_utf8_code_pages_are_pure_ascii() { + for cp in ["EBCDIC", "ASCII", "ascii ", "UTF8", "UNICODE"] { + let tags = file(cp, "Ação\r\nfim"); + let bytes = write_ascii_in(&tags, cp); + assert!(bytes.is_ascii(), "{cp}"); + assert_eq!(bytes, write_ascii(&tags).into_bytes(), "{cp}"); + assert!(contains(&bytes, b"A\\U+00E7\\U+00E3o fim"), "{cp}"); + assert!(write_binary_in(&tags, cp).iter().all(u8::is_ascii), "{cp} binary"); + } +} + +/// Code page bytes that happen to form valid UTF-8 (`СІ` in ANSI_1251 is D1 B2, which is U+0472 +/// in UTF-8) would be read back as UTF-8 from an ASCII file, so that file is written with +/// escapes. Binary DXF reads strings by the header's code page and keeps the bytes. +#[test] +fn code_page_bytes_that_are_valid_utf8_are_escaped_in_ascii() { + let tags = file("ANSI_1251", "СІ"); + let ascii = write_ascii_in(&tags, "ANSI_1251"); + assert!(ascii.is_ascii(), "{:?}", String::from_utf8_lossy(&ascii)); + assert!(contains(&ascii, b"\\U+0421\\U+0406")); + assert_eq!(parse(&ascii).unwrap()[10].str(), "СІ"); + let binary = write_binary_in(&tags, "ANSI_1251"); + assert!(contains(&binary, b"\xD1\xB2")); + assert_eq!(parse(&binary).unwrap()[10].str(), "СІ"); + // Bytes that are not valid UTF-8 stay as they are. + assert!(contains(&write_ascii_in(&file("ANSI_1251", "Слой"), "ANSI_1251"), b"\xD1\xEB\xEE\xE9")); +} + +/// A literal `\U+` or `\M+` in the text (#2050) reads back as written alongside code page bytes, +/// in ASCII and binary, including an ASCII file that falls back to escapes because its bytes +/// would read as UTF-8. +#[test] +fn literal_escapes_survive_code_page_text() { + for (cp, text) in + [("ANSI_1252", "Ação \\U+00E9 \\M+183A4F"), ("ANSI_1251", "Слой \\U+0041"), ("ANSI_1251", "СІ \\U+0041"), ("ANSI_932", "表 \\M+1")] + { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, b"\\U+005C"), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} + +/// Double-byte code pages whose trail byte can be 0x5C (`\\`): Shift-JIS `表` (95 5C) and `ソ` +/// (83 5C), Big5 `許` (B3 5C), GBK `乗` (81 5C). Right before `U+` or `M+`, such a character is +/// written as an escape, so even a reader that looks for escapes before it decodes the code page +/// reads it right. Elsewhere it stays as its bytes. Both forms read back as written. +#[test] +fn a_trail_byte_backslash_never_starts_an_escape() { + let cases: [(&str, &str, &[u8], &[u8]); 4] = [ + ("ANSI_932", "表U+0041 表示", b"\\U+8868U+0041 \x95\x5C\x8E\xA6", b"\x95\x5CU+"), + ("ANSI_932", "ソM+1A2B", b"\\U+30BDM+1A2B", b"\x83\x5CM+"), + ("ANSI_950", "許U+0041", b"\\U+8A31U+0041", b"\xB3\x5CU+"), + ("ANSI_936", "乗U+0041", b"\\U+4E57U+0041", b"\x81\x5CU+"), + ]; + for (cp, text, written, never) in cases { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, written), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert!(!contains(&bytes, never), "{cp} {what}: a trail byte starts an escape"); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} diff --git a/crates/io/src/dxf_write.rs b/crates/io/src/dxf_write.rs index a0ae2f95a..f2c125512 100644 --- a/crates/io/src/dxf_write.rs +++ b/crates/io/src/dxf_write.rs @@ -33,6 +33,7 @@ pub(crate) use blocks::is_generated_arrow; pub(crate) use blocks::table_block_entities; use ctx::Ctx; use entity::entity; +pub(crate) use header::code_page; use sections::tags_for; struct W { diff --git a/crates/io/src/dxf_write/header.rs b/crates/io/src/dxf_write/header.rs index 27a9019a5..354c293c8 100644 --- a/crates/io/src/dxf_write/header.rs +++ b/crates/io/src/dxf_write/header.rs @@ -134,10 +134,10 @@ pub(super) fn header_vars(w: &mut W, d: &Drawing) { dim_header_vars(w, d); } -/// `$DWGCODEPAGE`: before 2007 the code page read from the file when CADCraft knows it (the text -/// is ASCII with `\U+XXXX` escapes, right in any code page, but the file keeps its declared one); -/// otherwise, and from 2007 (UTF-8 text), `ANSI_1252`. -pub(super) fn code_page(d: &Drawing, version: Version) -> String { +/// `$DWGCODEPAGE`: before 2007 the code page read from the file when CADCraft knows it, in which +/// a saved file's text is written (characters outside it as `\U+XXXX` escapes); otherwise, and +/// from 2007 (UTF-8 text), `ANSI_1252`. +pub(crate) fn code_page(d: &Drawing, version: Version) -> String { let read = d.header.get("DWGCODEPAGE").and_then(HVal::as_str).map(str::trim).unwrap_or_default(); let known = read.is_ascii() && read.len() <= 32 && cadcraft_dxf::decode_code_page(b"", read).is_some(); if version < Version::R2007 && known { read.to_string() } else { "ANSI_1252".into() } diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 8cad3f24f..ccfba201e 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -92,10 +92,13 @@ pub fn write_with(d: &Drawing, name: impl AsRef, opts: &SaveOptions) -> Re match Format::from_path(name) { Some(Format::Dxf) => { let tags = dxf_write::tags(d, version); - Ok(if opts.binary { - cadcraft_dxf::write_binary(&tags, version.utf8()) - } else { - cadcraft_dxf::write_ascii_as(&tags, version.utf8()).into_bytes() + // Before 2007 the text is in the code page the header declares. + let code_page = dxf_write::code_page(d, version); + Ok(match (opts.binary, version.utf8()) { + (true, true) => cadcraft_dxf::write_binary(&tags, true), + (true, false) => cadcraft_dxf::write_binary_in(&tags, &code_page), + (false, true) => cadcraft_dxf::write_ascii_as(&tags, true).into_bytes(), + (false, false) => cadcraft_dxf::write_ascii_in(&tags, &code_page), }) } // The DWG bridge reads our R2000 DXF and writes the DWG at the version asked for. Without diff --git a/crates/io/tests/it/codepages.rs b/crates/io/tests/it/codepages.rs index 40d85b3f4..56f9af80f 100644 --- a/crates/io/tests/it/codepages.rs +++ b/crates/io/tests/it/codepages.rs @@ -116,6 +116,55 @@ fn code_page_text_survives_a_round_trip() { assert_eq!(names(&back), ("Слой".into(), "Блок".into(), "Привет".into())); } +/// Saving a pre-2007 DXF writes the text in the file's code page, as the DXF Reference +/// describes: a Portuguese drawing keeps `Elétrica`, `Ação` and `Função` as ANSI_1252 bytes +/// instead of `\U+00E9` escapes, a Cyrillic one its ANSI_1251 bytes, and characters the code +/// page lacks are still escaped. ASCII and binary, R2000 and R2004, read back the same. +#[test] +fn saving_writes_text_in_the_code_page() { + use cadcraft_io::{SaveOptions, Version}; + let pt = ascii_dxf("AC1015", "ANSI_1252", b"El\xE9trica", b"A\xE7\xE3o", b"Fun\xE7\xE3o \\U+56FE"); + let ru = ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2 \\U+00E9"); + let cases: [(&[u8], (&str, &str, &str), &[&[u8]]); 2] = [ + (&pt, ("Elétrica", "Ação", "Função 图"), &[b"El\xE9trica", b"A\xE7\xE3o", b"Fun\xE7\xE3o \\U+56FE"]), + (&ru, ("Слой", "Блок", "Привет é"), &[b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2 \\U+00E9"]), + ]; + for (file, want, raw) in cases { + let d = cadcraft_io::read_dxf(file).unwrap(); + assert_eq!(names(&d), (want.0.into(), want.1.into(), want.2.into())); + for version in [Version::R2000, Version::R2004] { + for binary in [false, true] { + let out = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary }).unwrap(); + for r in raw { + assert!(out.windows(r.len()).any(|w| w == *r), "{version} binary={binary}: {}", String::from_utf8_lossy(r)); + } + assert!(!out.windows(7).any(|w| w == b"\\U+00E7"), "{version} binary={binary}: ç is written as its byte"); + let back = cadcraft_io::read_dxf(&out).unwrap(); + assert_eq!(names(&back), (want.0.into(), want.1.into(), want.2.into()), "{version} binary={binary}"); + } + } + } +} + +/// A file whose header declares `ASCII` keeps that code page and stays ASCII when saved: its +/// non-ASCII text is written as `\U+` escapes, not as Windows-1252 bytes. +#[test] +fn saving_an_ascii_file_keeps_it_ascii() { + use cadcraft_io::{SaveOptions, Version}; + let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ASCII", b"El\\U+00E9trica", b"B", b"Eletrica \\U+00E9")).unwrap(); + assert_eq!(names(&d), ("Elétrica".into(), "B".into(), "Eletrica é".into())); + for version in [Version::R2000, Version::R2004] { + let ascii = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary: false }).unwrap(); + assert!(ascii.is_ascii(), "{version}"); + let binary = cadcraft_io::write_with(&d, "x.dxf", &SaveOptions { version: Some(version), binary: true }).unwrap(); + for out in [&ascii, &binary] { + assert!(out.windows(14).any(|w| w == b"El\\U+00E9trica"), "{version}"); + assert!(out.windows(5).any(|w| w == b"ASCII"), "{version}: the header keeps its code page"); + assert_eq!(names(&cadcraft_io::read_dxf(out).unwrap()), ("Elétrica".into(), "B".into(), "Eletrica é".into()), "{version}"); + } + } +} + /// A binary DXF: R2000 strings are in the code page, R2007+ strings UTF-8. #[test] fn binary_dxf_honours_the_code_page() { diff --git a/crates/io/tests/it/dxf_versions.rs b/crates/io/tests/it/dxf_versions.rs index 40aaa878c..390c94703 100644 --- a/crates/io/tests/it/dxf_versions.rs +++ b/crates/io/tests/it/dxf_versions.rs @@ -87,9 +87,11 @@ fn dxf_at_every_version_reads_back_the_same() { assert_eq!(acadver(&bytes), v.code(), "{what}"); assert_eq!(cadcraft_dxf::is_binary(&bytes), binary, "{what}"); let raw = bytes.windows("图号".len()).any(|w| w == "图号".as_bytes()); - assert_eq!(raw, v.utf8(), "{what}: UTF-8 text from 2007, \\U+ escapes before"); - if !v.utf8() && !binary { - assert!(bytes.is_ascii(), "{what}"); + assert_eq!(raw, v.utf8(), "{what}: UTF-8 text from 2007, the code page before"); + if !v.utf8() { + // ANSI_1252 has the Latin characters; the others are escaped. + let has = |b: &[u8]| bytes.windows(b.len()).any(|w| w == b); + assert!(has(b"\\U+56FE\\U+53F7 Caf\xe9 \xd8 \\U+D83D\\U+DE00"), "{what}"); } let back = read(&bytes, "x.dxf").unwrap(); assert_eq!(first_text(&back).as_deref(), Some(INTL), "{what}"); @@ -124,11 +126,15 @@ fn class_records_follow_the_version() { } } -/// R2000 ASCII is exactly the R2000 DXF CADCraft always wrote. +/// R2000 ASCII is the R2000 DXF CADCraft always wrote, tag for tag; only the text is in the +/// code page instead of all `\U+` escapes. #[test] fn r2000_ascii_is_unchanged() { let d = sample(); - assert_eq!(write_with(&d, "x.dxf", &opts(Version::R2000, false)).unwrap(), write_dxf(&d).into_bytes()); + let saved = write_with(&d, "x.dxf", &opts(Version::R2000, false)).unwrap(); + let escaped = write_dxf(&d); + assert_ne!(saved, escaped.as_bytes(), "Café is written in ANSI_1252"); + assert_eq!(cadcraft_dxf::parse(&saved).unwrap(), cadcraft_dxf::parse(escaped.as_bytes()).unwrap()); } /// Without a version asked for: 2018 for a new drawing, otherwise the file's own version (the diff --git a/crates/io/tests/it/followups_626.rs b/crates/io/tests/it/followups_626.rs index a44a27eb4..10e355cd3 100644 --- a/crates/io/tests/it/followups_626.rs +++ b/crates/io/tests/it/followups_626.rs @@ -177,10 +177,9 @@ fn code_page_is_kept_before_2007() { .unwrap(); for v in [Version::R2000, Version::R2004] { let bytes = save(&d, "x.dxf", v); - assert!(bytes.is_ascii(), "{v:?}: ASCII with escapes"); - let text = String::from_utf8(bytes).unwrap(); - assert_eq!(code_page(&text), "ANSI_1251", "{v:?}"); - let back = cadcraft_io::read_dxf(text.as_bytes()).unwrap(); + assert!(bytes.windows(6).any(|w| w == b"\xCF\xF0\xE8\xE2\xE5\xF2"), "{v:?}: the text in ANSI_1251"); + assert_eq!(code_page(&String::from_utf8_lossy(&bytes)), "ANSI_1251", "{v:?}"); + let back = cadcraft_io::read_dxf(&bytes).unwrap(); assert_eq!(back.header.str("DWGCODEPAGE", ""), "ANSI_1251"); assert!(back.model.iter().any(|e| matches!(&e.kind, EntityKind::Text(t) if t.value == "Привет"))); } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 747264849..d85415123 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure with that `$ACADVER`; UTF-8 text from 2007; CLASS instance counts (91) from 2004 only (#635; with them, AutoCAD LT 2026 stops responding on an R2000 file) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | writes `\U+` escapes (#169), which is correct; declares `ANSI_1252` | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169); a double-byte character whose trail byte is `\` (Shift-JIS, Big5, GBK) is escaped where `U+` or `M+` follows it | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type @@ -146,6 +146,7 @@ our output in AutoCAD black-box. | 2026-10-22 | minor | DXF/DWG save: angular, diametric and radial dimensions get standard DIMASSOC objects (one centre reference to the circle for radial/diametric, the definition points for angular), so AutoCAD follows their geometry; the standard objects of those kinds are read back too, without CADCraft's xdata (#1651) | | 2026-10-19 | minor | DXF/DWG save: angular dimension block text honors DIMDSEP, including per-dimension overrides (#1646) | | 2026-10-19 | minor | A file's own arrowhead blocks with standard names (AutoCAD's `_Dot`, `_DotSmall`), named only by a dimension style, are kept and saved as they were instead of being replaced by CADCraft's solids; PURGE keeps arrowhead blocks dimension styles name | +| 2026-10-19 | minor | Pre-2007 DXF saves write text in the file's code page (`Ação` as ANSI_1252 bytes, not `A\U+00E7\U+00E3o`); characters outside it stay `\U+` escapes | | 2026-10-20 | minor | DIMENSION: the `*D` block's MTEXT, and the MTEXT EXPLODE leaves, hold the diameter sign as `%%c` instead of the decoded U+2300, which no SHX font has, so other programs plotted diameter dimensions in an SHX style without the sign (#1645) | | 2026-10-21 | minor | Proxy objects and entities keep their class through a save: group 91 renumbered to the written CLASSES (the class it names is kept), a proxy whose number names no class dropped (#1812) | | 2026-10-17 | minor | DWG read: DGN line style components no line style definition reaches are left out before the DXF bridge, with a read note; an 83 MB site plan with 888,483 of them (1.2 GB of DXF, over the limit) now opens (#1482) | diff --git a/docs/localization-parity.md b/docs/localization-parity.md index ff44de470..a6bcb5079 100644 --- a/docs/localization-parity.md +++ b/docs/localization-parity.md @@ -31,8 +31,8 @@ shaping and right-to-left text through TrueType, vertical SHX text styles. English. For scale: about 1,160 unique capitalised string literals exist in `crates/engine` and `crates/ui-egui` (grep, 2026-10-10). - **Drawing text:** TrueType/OpenType/`.ttc` through `skrifa` with per-character font fallback - and FONTALT, so CJK characters render (#173). DXF text is written with `\U+` escapes so - non-ASCII survives in R2000 files (#169). Lines with right-to-left text get the Unicode + and FONTALT, so CJK characters render (#173). Pre-2007 DXF text is written in the file's code page, + with `\U+` escapes for characters outside it, so non-ASCII survives in R2000 files (#169). Lines with right-to-left text get the Unicode Bidirectional Algorithm and OpenType shaping with `harfrust` (Arabic, Persian and Urdu joining, lam-alef, ZWNJ, marks; Hebrew), in TEXT and MTEXT including wrapping (#287). Left-to-right lines are still placed one glyph per code point: **no shaping** for Devanagari and other Indic