diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index ec350adaa..7f40c8f32 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -547,18 +547,21 @@ pub fn is_binary(bytes: &[u8]) -> bool { /// its bytes, and every other one `\U+XXXX` (a surrogate pair of escapes above U+FFFF). Without /// a code page the result is ASCII, so it is valid in any code page. In every form a backslash /// that would start a `\U+` or `\M+` sequence is written as `\U+005C`, so the reader gives back -/// the literal text (#2050). +/// the literal text (#2050). So is a double-byte character whose trail byte is 0x5C (`\`) in that +/// place, such as Shift-JIS `表` or Big5 `許`: CADCraft decodes the code page before it looks for +/// escapes, but a reader that looks first would take that byte and the `U+` after it for one. fn push_string(s: &str, strings: Strings, out: &mut Vec) { let chars: Vec = s.chars().collect(); + let escape_at = |j: usize| matches!(chars.get(j), Some('U' | 'M')) && chars.get(j + 1) == Some(&'+'); for (i, &c) in chars.iter().enumerate() { match (c, strings) { ('\r' | '\n', _) => out.push(b' '), - ('\\', _) if matches!(chars.get(i + 1), Some('U' | 'M')) && chars.get(i + 2) == Some(&'+') => { + ('\\', _) if escape_at(i + 1) => { out.extend_from_slice(b"\\U+005C"); } (c, Strings::Utf8) => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), (c, _) if c.is_ascii() => out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()), - (c, Strings::Escaped(Some(d))) if codepage::encode_char(c, d, out) => {} + (c, Strings::Escaped(Some(d))) if code_page_bytes(c, d, escape_at(i + 1), out) => {} (c, Strings::Escaped(_)) => { let mut units = [0u16; 2]; for u in c.encode_utf16(&mut units) { @@ -569,6 +572,20 @@ fn push_string(s: &str, strings: Strings, out: &mut Vec) { } } +/// `c` in the code page `d`, if it has it, unless it is a double-byte character whose trail byte +/// is 0x5C and an escape's `U+` or `M+` follows (`before_escape`): see [`push_string`]. +fn code_page_bytes(c: char, d: codepage::Decoder, before_escape: bool, out: &mut Vec) -> bool { + let start = out.len(); + if !codepage::encode_char(c, d, out) { + return false; + } + if before_escape && out.len() - start > 1 && out.last() == Some(&b'\\') { + out.truncate(start); + return false; + } + true +} + /// A section: `0 SECTION / 2 NAME ... 0 ENDSEC`. #[derive(Clone, Debug, Default)] pub struct Section { diff --git a/crates/dxf/tests/code_page_write.rs b/crates/dxf/tests/code_page_write.rs index 466a2b2a6..75d647c08 100644 --- a/crates/dxf/tests/code_page_write.rs +++ b/crates/dxf/tests/code_page_write.rs @@ -92,3 +92,25 @@ fn literal_escapes_survive_code_page_text() { } } } + +/// Double-byte code pages whose trail byte can be 0x5C (`\\`): Shift-JIS `表` (95 5C) and `ソ` +/// (83 5C), Big5 `許` (B3 5C), GBK `乗` (81 5C). Right before `U+` or `M+`, such a character is +/// written as an escape, so even a reader that looks for escapes before it decodes the code page +/// reads it right. Elsewhere it stays as its bytes. Both forms read back as written. +#[test] +fn a_trail_byte_backslash_never_starts_an_escape() { + let cases: [(&str, &str, &[u8], &[u8]); 4] = [ + ("ANSI_932", "表U+0041 表示", b"\\U+8868U+0041 \x95\x5C\x8E\xA6", b"\x95\x5CU+"), + ("ANSI_932", "ソM+1A2B", b"\\U+30BDM+1A2B", b"\x83\x5CM+"), + ("ANSI_950", "許U+0041", b"\\U+8A31U+0041", b"\xB3\x5CU+"), + ("ANSI_936", "乗U+0041", b"\\U+4E57U+0041", b"\x81\x5CU+"), + ]; + for (cp, text, written, never) in cases { + let tags = file(cp, text); + for (bytes, what) in [(write_ascii_in(&tags, cp), "ASCII"), (write_binary_in(&tags, cp), "binary")] { + assert!(contains(&bytes, written), "{cp} {what}: {:?}", String::from_utf8_lossy(&bytes)); + assert!(!contains(&bytes, never), "{cp} {what}: a trail byte starts an escape"); + assert_eq!(parse(&bytes).unwrap()[10].str(), text, "{cp} {what}"); + } + } +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index d919cf6ba..eda4a0872 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -66,7 +66,7 @@ on read with a note, so such files no longer exceed the DXF limit (#1482). **DWG | R12 binary | read, write | **no** (binary parser expects R13+ 2-byte group codes) | no | | R13–R2018, ASCII | read, write each of 2000/2004/2007/2010/2013/2018 | yes, one reader for all versions | yes, each of 2000–2018 (#421): R2000 structure (later versions add only optional groups) with that `$ACADVER`; UTF-8 text from 2007; no CLASS instance counts (91) | | R13–R2018, binary | read, write | yes | yes, each of 2000–2018 (SAVEAS `binary`; a binary file stays binary on QSAVE) | -| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169) | +| Code pages (`$DWGCODEPAGE`, ANSI_932/936/949/950/125x) | yes | yes (#505): ASCII and binary DXF before 2007 decode from their code page (ANSI_874/932/936/949/950/1250–1258, DOS437/850/852/866, ISO 8859, Macintosh); a file that is valid UTF-8 reads as UTF-8; Johab and other DOS code pages read as ANSI_1252 (`dxf/src/codepage.rs`) | before 2007 writes text in the declared code page (the one read from the file, else `ANSI_1252`), characters outside it as `\U+` escapes; an unknown code page, or ASCII DXF whose code page bytes would also read as UTF-8, gives pure ASCII with escapes (#169); a double-byte character whose trail byte is `\` (Shift-JIS, Big5, GBK) is escaped where `U+` or `M+` follows it | | `\U+`/`\M+` escapes | yes | yes (`\M+` since #505; Johab `\M+4` stays literal) | `\U+` yes | ### DXF content: read and write by object type