DXF: read pre-2007 files in their $DWGCODEPAGE (CJK, Cyrillic, Central European…) (fixes #505) (#511)

Squash of PR #511 by @vangel76.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Valentin Klein
2026-10-11 11:47:42 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 7dead193e4
commit 43958b537e
6 changed files with 539 additions and 16 deletions
Generated
+2
View File
@@ -606,6 +606,8 @@ dependencies = [
name = "cadcraft-dxf"
version = "0.5.0"
dependencies = [
"encoding_rs",
"log",
"thiserror 2.0.21",
]
+1
View File
@@ -40,6 +40,7 @@ skrifa = "0.47"
harfrust = "0.12"
unicode-bidi = "0.3.18"
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
encoding_rs = "0.8.35"
# ui
craft-ui = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "5c874d51eeb3de70e735c8ce87a59a7dedd47675", features = ["serde"] }
+3 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "cadcraft-dxf"
description = "Standalone DXF tag reader/writer (ASCII and binary), from the published DXF reference"
description = "Standalone DXF tag reader/writer (ASCII and binary, pre-2007 code pages), from the published DXF reference"
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -10,3 +10,5 @@ workspace = true
[dependencies]
thiserror = { workspace = true }
log = { workspace = true }
encoding_rs = { workspace = true }
+294
View File
@@ -0,0 +1,294 @@
//! Text decoding for DXF files.
//!
//! R2007 (AC1021) and later DXF files are UTF-8. Older ones store text in the code page named by
//! the `$DWGCODEPAGE` header variable (`ANSI_1252`, `ANSI_932`, `DOS850`, …); characters outside
//! it are written as `\U+XXXX` (any character) or `\M+nXXXX` (a double-byte character of an East
//! Asian code page) escapes, which [`crate::parse`] decodes after the bytes.
//!
//! The Windows, ISO and East Asian code pages are decoded with `encoding_rs`; DOS 437, 850 and
//! 852 with the tables below (the upper halves of the IBM PC code pages; the lower halves are
//! ASCII). Decoding never fails: a byte sequence that is invalid in the code page (a truncated
//! double-byte character, an unassigned byte) becomes U+FFFD and the bytes after it decode
//! normally, so line ends and the group structure survive.
use std::borrow::Cow;
use encoding_rs::Encoding;
/// How the bytes of a code page become text.
#[derive(Clone, Copy, Debug)]
pub(crate) enum Decoder {
Utf8,
Encoding(&'static Encoding),
/// Single-byte DOS code page: ASCII, then these 128 characters for bytes 0x80–0xFF.
Table(&'static [char; 128]),
}
/// The decoder for a `$DWGCODEPAGE` value (case-insensitive), or `None` when CADCraft doesn't
/// know the code page.
pub(crate) fn decoder(code_page: &str) -> Option<Decoder> {
use encoding_rs as e;
let name = code_page.trim().to_ascii_uppercase().replace(['-', ' '], "_");
let enc = match name.as_str() {
"UTF8" | "UTF_8" | "UNICODE" => return Some(Decoder::Utf8),
"ASCII" | "ANSI_1252" | "ISO8859_1" | "ISO_8859_1" => e::WINDOWS_1252,
"ANSI_874" => e::WINDOWS_874,
"ANSI_932" | "DOS932" | "SHIFT_JIS" => e::SHIFT_JIS,
"ANSI_936" | "GB2312" | "GBK" => e::GBK,
"ANSI_949" | "KSC5601" | "KOREAN" => e::EUC_KR,
"ANSI_950" | "BIG5" => e::BIG5,
"ANSI_1250" => e::WINDOWS_1250,
"ANSI_1251" => e::WINDOWS_1251,
"ANSI_1253" => e::WINDOWS_1253,
"ANSI_1254" | "ISO8859_9" | "ISO_8859_9" => e::WINDOWS_1254,
"ANSI_1255" => e::WINDOWS_1255,
"ANSI_1256" => e::WINDOWS_1256,
"ANSI_1257" => e::WINDOWS_1257,
"ANSI_1258" => e::WINDOWS_1258,
"ISO8859_2" | "ISO_8859_2" => e::ISO_8859_2,
"ISO8859_3" | "ISO_8859_3" => e::ISO_8859_3,
"ISO8859_4" | "ISO_8859_4" => e::ISO_8859_4,
"ISO8859_5" | "ISO_8859_5" => e::ISO_8859_5,
"ISO8859_6" | "ISO_8859_6" => e::ISO_8859_6,
"ISO8859_7" | "ISO_8859_7" => e::ISO_8859_7,
"ISO8859_8" | "ISO_8859_8" => e::ISO_8859_8,
"DOS866" => e::IBM866,
"MACINTOSH" | "MAC_ROMAN" => e::MACINTOSH,
"DOS437" => return Some(Decoder::Table(&DOS437)),
"DOS850" => return Some(Decoder::Table(&DOS850)),
"DOS852" => return Some(Decoder::Table(&DOS852)),
_ => return None,
};
Some(Decoder::Encoding(enc))
}
/// Decode `bytes` from the code page `code_page` names (a `$DWGCODEPAGE` value such as
/// `ANSI_1251`), or `None` when CADCraft doesn't know that code page. Invalid sequences become
/// U+FFFD; escapes (`\U+XXXX`, `\M+nXXXX`) are left as they are.
pub fn decode_code_page(bytes: &[u8], code_page: &str) -> Option<String> {
decoder(code_page).map(|d| decode(bytes, d).into_owned())
}
/// Decode bytes with a decoder. UTF-8 that isn't valid falls back to Windows-1252.
pub(crate) fn decode(bytes: &[u8], d: Decoder) -> Cow<'_, str> {
match d {
Decoder::Utf8 => match std::str::from_utf8(bytes) {
Ok(s) => Cow::Borrowed(s),
Err(_) => encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes).0,
},
Decoder::Encoding(e) => e.decode_without_bom_handling(bytes).0,
Decoder::Table(t) => match std::str::from_utf8(bytes) {
Ok(s) if s.is_ascii() => Cow::Borrowed(s),
_ => Cow::Owned(
bytes
.iter()
.map(|&b| if b.is_ascii() { char::from(b) } else { t.get(usize::from(b & 0x7F)).copied().unwrap_or('\u{FFFD}') })
.collect(),
),
},
}
}
/// The `$DWGCODEPAGE` value of an ASCII DXF file: the line two lines after a `$DWGCODEPAGE`
/// line (the group code `3` between them), looked for in the HEADER section only (the scan stops
/// at the first `ENDSEC`). Works on the raw bytes, before they are decoded.
fn ascii_code_page(bytes: &[u8]) -> Option<&[u8]> {
let mut lines = bytes.split(|&b| b == b'\n').map(<[u8]>::trim_ascii);
while let Some(line) = lines.next() {
if line == b"ENDSEC" {
return None;
}
if line.eq_ignore_ascii_case(b"$DWGCODEPAGE") {
return match (lines.next(), lines.next()) {
(Some(b"3"), Some(value)) => Some(value),
_ => None,
};
}
}
None
}
/// Decode a whole ASCII DXF file. Valid UTF-8 is read as UTF-8: R2007+ files, pure-ASCII files,
/// and older files from writers that ignored the code page. Anything else is decoded from the
/// file's `$DWGCODEPAGE`, or from Windows-1252 (the usual `ANSI_1252`) when the header has none
/// or names a code page CADCraft doesn't know, which is logged as a warning.
pub(crate) fn decode_ascii_file(bytes: &[u8]) -> Cow<'_, str> {
if let Ok(s) = std::str::from_utf8(bytes) {
return Cow::Borrowed(s);
}
let d = match ascii_code_page(bytes) {
Some(name) => {
let name = String::from_utf8_lossy(name);
decoder(&name).unwrap_or_else(|| {
log::warn!("DXF: unknown code page `{name}`; reading the text as ANSI_1252");
Decoder::Utf8
})
}
None => Decoder::Utf8,
};
decode(bytes, d)
}
/// The text state of a binary DXF file, learnt from its header as the tags go by: `$ACADVER`
/// (R2007+ strings are UTF-8) and `$DWGCODEPAGE` (the code page of older files). Both come
/// first in the HEADER section and are ASCII, so every later string is decoded right.
#[derive(Default)]
pub(crate) struct BinaryText {
/// The header variable whose value comes next (after a group 9).
var: Option<HeaderVar>,
utf8: bool,
code_page: Option<Decoder>,
}
#[derive(Clone, Copy, PartialEq)]
enum HeaderVar {
AcadVer,
CodePage,
}
impl BinaryText {
/// Decode one string value of group `code` and update the state from it. Strings of R2007+
/// files, and of files without a code page, are UTF-8 when valid (else Windows-1252); older
/// files' strings are in their code page.
pub(crate) fn string(&mut self, code: i32, raw: &[u8]) -> String {
let d = if self.utf8 { Decoder::Utf8 } else { self.code_page.unwrap_or(Decoder::Utf8) };
let s = decode(raw, d).into_owned();
match (code, self.var.take()) {
(9, _) => {
self.var = match s.trim() {
"$ACADVER" => Some(HeaderVar::AcadVer),
"$DWGCODEPAGE" => Some(HeaderVar::CodePage),
_ => None,
}
}
(1, Some(HeaderVar::AcadVer)) => self.utf8 = is_utf8_version(&s),
(3, Some(HeaderVar::CodePage)) => {
self.code_page = decoder(&s);
if self.code_page.is_none() {
log::warn!("DXF: unknown code page `{}`; reading the text as ANSI_1252", s.trim());
}
}
_ => {}
}
s
}
}
/// True for an `$ACADVER` of R2007 (AC1021) or later, whose text is UTF-8.
fn is_utf8_version(acadver: &str) -> bool {
acadver.trim().strip_prefix("AC").and_then(|n| n.parse::<u32>().ok()).is_some_and(|n| n >= 1021)
}
/// The double-byte code page of a `\M+n` escape: 1 Japanese (Shift-JIS), 2 Traditional Chinese
/// (Big5), 3 Korean (Wansung), 5 Simplified Chinese (GB 2312). 4 (Korean Johab) is not
/// supported, so its escapes stay literal.
pub(crate) fn mif_encoding(n: char) -> Option<&'static Encoding> {
match n {
'1' => Some(encoding_rs::SHIFT_JIS),
'2' => Some(encoding_rs::BIG5),
'3' => Some(encoding_rs::EUC_KR),
'5' => Some(encoding_rs::GBK),
_ => None,
}
}
// Generated from the IBM PC code page definitions (bytes 0x80–0xFF in order).
const DOS437: [char; 128] = [
'\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}',
'\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}',
'\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00A2}', '\u{00A3}', '\u{00A5}', '\u{20A7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}',
'\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{2310}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}',
'\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{2561}', '\u{2562}', '\u{2556}', '\u{2555}', '\u{2563}', '\u{2551}', '\u{2557}',
'\u{255D}', '\u{255C}', '\u{255B}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{255E}', '\u{255F}',
'\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{2567}', '\u{2568}', '\u{2564}', '\u{2565}', '\u{2559}',
'\u{2558}', '\u{2552}', '\u{2553}', '\u{256B}', '\u{256A}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{258C}', '\u{2590}', '\u{2580}',
'\u{03B1}', '\u{00DF}', '\u{0393}', '\u{03C0}', '\u{03A3}', '\u{03C3}', '\u{00B5}', '\u{03C4}', '\u{03A6}', '\u{0398}', '\u{03A9}', '\u{03B4}',
'\u{221E}', '\u{03C6}', '\u{03B5}', '\u{2229}', '\u{2261}', '\u{00B1}', '\u{2265}', '\u{2264}', '\u{2320}', '\u{2321}', '\u{00F7}', '\u{2248}',
'\u{00B0}', '\u{2219}', '\u{00B7}', '\u{221A}', '\u{207F}', '\u{00B2}', '\u{25A0}', '\u{00A0}',
];
const DOS850: [char; 128] = [
'\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{00E0}', '\u{00E5}', '\u{00E7}', '\u{00EA}', '\u{00EB}', '\u{00E8}', '\u{00EF}',
'\u{00EE}', '\u{00EC}', '\u{00C4}', '\u{00C5}', '\u{00C9}', '\u{00E6}', '\u{00C6}', '\u{00F4}', '\u{00F6}', '\u{00F2}', '\u{00FB}', '\u{00F9}',
'\u{00FF}', '\u{00D6}', '\u{00DC}', '\u{00F8}', '\u{00A3}', '\u{00D8}', '\u{00D7}', '\u{0192}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}',
'\u{00F1}', '\u{00D1}', '\u{00AA}', '\u{00BA}', '\u{00BF}', '\u{00AE}', '\u{00AC}', '\u{00BD}', '\u{00BC}', '\u{00A1}', '\u{00AB}', '\u{00BB}',
'\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{00C0}', '\u{00A9}', '\u{2563}', '\u{2551}', '\u{2557}',
'\u{255D}', '\u{00A2}', '\u{00A5}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{00E3}', '\u{00C3}',
'\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{00F0}', '\u{00D0}', '\u{00CA}', '\u{00CB}',
'\u{00C8}', '\u{0131}', '\u{00CD}', '\u{00CE}', '\u{00CF}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{00A6}', '\u{00CC}', '\u{2580}',
'\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{00D2}', '\u{00F5}', '\u{00D5}', '\u{00B5}', '\u{00FE}', '\u{00DE}', '\u{00DA}', '\u{00DB}', '\u{00D9}',
'\u{00FD}', '\u{00DD}', '\u{00AF}', '\u{00B4}', '\u{00AD}', '\u{00B1}', '\u{2017}', '\u{00BE}', '\u{00B6}', '\u{00A7}', '\u{00F7}', '\u{00B8}',
'\u{00B0}', '\u{00A8}', '\u{00B7}', '\u{00B9}', '\u{00B3}', '\u{00B2}', '\u{25A0}', '\u{00A0}',
];
const DOS852: [char; 128] = [
'\u{00C7}', '\u{00FC}', '\u{00E9}', '\u{00E2}', '\u{00E4}', '\u{016F}', '\u{0107}', '\u{00E7}', '\u{0142}', '\u{00EB}', '\u{0150}', '\u{0151}',
'\u{00EE}', '\u{0179}', '\u{00C4}', '\u{0106}', '\u{00C9}', '\u{0139}', '\u{013A}', '\u{00F4}', '\u{00F6}', '\u{013D}', '\u{013E}', '\u{015A}',
'\u{015B}', '\u{00D6}', '\u{00DC}', '\u{0164}', '\u{0165}', '\u{0141}', '\u{00D7}', '\u{010D}', '\u{00E1}', '\u{00ED}', '\u{00F3}', '\u{00FA}',
'\u{0104}', '\u{0105}', '\u{017D}', '\u{017E}', '\u{0118}', '\u{0119}', '\u{00AC}', '\u{017A}', '\u{010C}', '\u{015F}', '\u{00AB}', '\u{00BB}',
'\u{2591}', '\u{2592}', '\u{2593}', '\u{2502}', '\u{2524}', '\u{00C1}', '\u{00C2}', '\u{011A}', '\u{015E}', '\u{2563}', '\u{2551}', '\u{2557}',
'\u{255D}', '\u{017B}', '\u{017C}', '\u{2510}', '\u{2514}', '\u{2534}', '\u{252C}', '\u{251C}', '\u{2500}', '\u{253C}', '\u{0102}', '\u{0103}',
'\u{255A}', '\u{2554}', '\u{2569}', '\u{2566}', '\u{2560}', '\u{2550}', '\u{256C}', '\u{00A4}', '\u{0111}', '\u{0110}', '\u{010E}', '\u{00CB}',
'\u{010F}', '\u{0147}', '\u{00CD}', '\u{00CE}', '\u{011B}', '\u{2518}', '\u{250C}', '\u{2588}', '\u{2584}', '\u{0162}', '\u{016E}', '\u{2580}',
'\u{00D3}', '\u{00DF}', '\u{00D4}', '\u{0143}', '\u{0144}', '\u{0148}', '\u{0160}', '\u{0161}', '\u{0154}', '\u{00DA}', '\u{0155}', '\u{0170}',
'\u{00FD}', '\u{00DD}', '\u{0163}', '\u{00B4}', '\u{00AD}', '\u{02DD}', '\u{02DB}', '\u{02C7}', '\u{02D8}', '\u{00A7}', '\u{00F7}', '\u{00B8}',
'\u{00B0}', '\u{00A8}', '\u{02D9}', '\u{0171}', '\u{0158}', '\u{0159}', '\u{25A0}', '\u{00A0}',
];
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn code_page_names() {
let d = |bytes: &[u8], cp: &str| decode_code_page(bytes, cp);
assert_eq!(d(b"\xCF\xF0\xE8\xE2\xE5\xF2", "ANSI_1251").as_deref(), Some("Привет"));
assert_eq!(d(b"\xCF\xF0\xE8\xE2\xE5\xF2", " ansi_1251 ").as_deref(), Some("Привет"));
assert_eq!(d(b"\x90\x7D\x96\xCA", "ANSI_932").as_deref(), Some("図面"));
assert_eq!(d(b"\xCD\xBC\xB2\xE3", "GB2312").as_deref(), Some("图层"));
assert_eq!(d(b"\xB9\xCF\xBC\x68", "BIG5").as_deref(), Some("圖層"));
assert_eq!(d(b"\xB5\xB5\xB8\xE9", "ANSI_949").as_deref(), Some("도면"));
assert_eq!(d(b"\xD3\xF7\xDD\xE4\xE9\xEF", "ANSI_1253").as_deref(), Some("Σχέδιο"));
assert_eq!(d(b"\xC0\xD2\xBE", "ANSI_874").as_deref(), Some("ภาพ"));
assert_eq!(d(b"\x80", "ANSI_1252").as_deref(), Some("€"));
// DOS 437 and 850 agree on letters but differ on 0x9B (¢ / ø).
assert_eq!(d(b"Gr\x94\xE1e \x9B", "DOS437").as_deref(), Some("Größe ¢"));
assert_eq!(d(b"Gr\x94\xE1e \x9B", "dos850").as_deref(), Some("Größe ø"));
assert_eq!(d(b"\xBD\xA2\x88w", "DOS852").as_deref(), Some("Żółw"));
assert_eq!(d(b"\x91\xAB\xAE\xA9", "DOS866").as_deref(), Some("Слой"));
assert_eq!(d("é".as_bytes(), "UTF8").as_deref(), Some("é"));
assert_eq!(d(b"x", "ANSI_1361"), None);
assert_eq!(d(b"x", "EBCDIC"), None);
assert_eq!(d(b"x", ""), None);
}
#[test]
fn invalid_sequences_become_replacement_characters() {
// A Shift-JIS lead byte cut off by the line end keeps the line end.
assert_eq!(decode_code_page(b"\x90\n1\n", "ANSI_932").as_deref(), Some("\u{FFFD}\n1\n"));
assert_eq!(decode_code_page(b"\x90", "ANSI_932").as_deref(), Some("\u{FFFD}"));
assert_eq!(decode_code_page(b"\xFF\xFF", "ANSI_936").as_deref(), Some("\u{FFFD}\u{FFFD}"));
}
#[test]
fn header_scan() {
let head = b"0\nSECTION\n2\nHEADER\n9\n$ACADVER\n1\nAC1015\n9\n$DWGCODEPAGE\n3\nANSI_1251\r\n0\nENDSEC\n";
assert_eq!(ascii_code_page(head), Some(&b"ANSI_1251"[..]));
// Only the HEADER section counts.
assert_eq!(ascii_code_page(b"0\nSECTION\n2\nHEADER\n0\nENDSEC\n9\n$DWGCODEPAGE\n3\nANSI_1251\n"), None);
// Wrong group code, or the file ends early.
assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n1\nANSI_1251\n"), None);
assert_eq!(ascii_code_page(b"9\n$DWGCODEPAGE\n3"), None);
assert!(is_utf8_version("AC1021") && is_utf8_version(" AC1032 "));
assert!(!is_utf8_version("AC1018") && !is_utf8_version("AC1009") && !is_utf8_version("garbage"));
}
#[test]
fn mif_escapes() {
let one = |s: &str| crate::parse(format!("1\n{s}\n0\nEOF\n").as_bytes()).unwrap()[0].str();
assert_eq!(one("\\M+5BCFE\\M+1907D\\M+2B9CF\\M+3B5B5!"), "件図圖도!");
assert_eq!(one("\\M+5BCFE\\U+0021"), "件!");
// Johab, unknown code pages, bad hex, short escapes and unassigned codes stay literal.
for s in ["\\M+4ABCD", "\\M+9BCFE", "\\M+5GGGG", "\\M+5BC", "\\M+", "\\M+50041", "\\M+5FFFF"] {
assert_eq!(one(s), s);
}
}
}
+35 -15
View File
@@ -166,14 +166,8 @@ const BINARY_SENTINEL: &[u8] = b"AutoCAD Binary DXF\r\n\x1a\0";
/// tag list to a few GB; real drawings stay far below (a 10 MB DWG is typically a few million).
pub const MAX_TAGS: usize = 50_000_000;
/// Decode bytes as text: UTF-8 when valid (borrowed, no copy), otherwise Latin-1 (old ANSI_1252
/// files).
fn decode_text(b: &[u8]) -> std::borrow::Cow<'_, str> {
match std::str::from_utf8(b) {
Ok(s) => std::borrow::Cow::Borrowed(s),
Err(_) => std::borrow::Cow::Owned(b.iter().map(|&c| char::from(c)).collect()),
}
}
mod codepage;
pub use codepage::decode_code_page;
/// Parse a DXF file (ASCII or binary).
pub fn parse(bytes: &[u8]) -> Result<Vec<Tag>> {
@@ -192,7 +186,7 @@ pub fn parse_with_limit(bytes: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
if tags > max_tags {
return Err(DxfError::TooLarge { tags, limit: max_tags });
}
parse_ascii_with_limit(&decode_text(bytes), max_tags)
parse_ascii_with_limit(&codepage::decode_ascii_file(bytes), max_tags)
}
/// The four hex digits of a `\U+XXXX` escape starting at `chars[i]`, if there is one.
@@ -207,17 +201,41 @@ fn escape_at(chars: &[char], i: usize) -> Option<u32> {
hex.iter().try_fold(0u32, |acc, c| Some(acc * 16 + c.to_digit(16)?))
}
/// The character of a `\M+nXXXX` escape starting at `chars[i]`, if there is one: `n` names a
/// double-byte code page and `XXXX` is the character's two bytes in hex.
fn mif_escape_at(chars: &[char], i: usize) -> Option<char> {
if chars.get(i) != Some(&'\\') || chars.get(i + 1) != Some(&'M') || chars.get(i + 2) != Some(&'+') {
return None;
}
let enc = codepage::mif_encoding(*chars.get(i + 3)?)?;
let hex = chars.get(i + 4..i + 8)?;
if !hex.iter().all(char::is_ascii_hexdigit) {
return None;
}
let code = hex.iter().try_fold(0u16, |acc, c| Some(acc * 16 + u16::try_from(c.to_digit(16)?).ok()?))?;
let bytes = code.to_be_bytes();
let text = enc.decode_without_bom_handling_and_without_replacement(&bytes)?;
let mut it = text.chars();
it.next().filter(|_| it.next().is_none())
}
/// Unescape the `\U+XXXX` sequences pre-R2007 DXF uses for characters outside the file's code
/// page (four hex digits, one UTF-16 code unit; characters above U+FFFF are a surrogate pair of
/// escapes). Malformed escapes and unpaired surrogates are kept as literal text.
/// escapes), and the `\M+nXXXX` sequences for double-byte East Asian characters. Malformed
/// escapes, unpaired surrogates and unknown `\M+` code pages are kept as literal text.
fn unescape_unicode(s: &str) -> String {
if !s.contains("\\U+") {
if !s.contains("\\U+") && !s.contains("\\M+") {
return s.to_string();
}
let mut out = String::with_capacity(s.len());
let chars: Vec<char> = s.chars().collect();
let mut i = 0;
while let Some(&c) = chars.get(i) {
if let Some(ch) = mif_escape_at(&chars, i) {
out.push(ch);
i += 8;
continue;
}
if let Some(unit) = escape_at(&chars, i) {
if let Some(ch) = char::from_u32(unit) {
out.push(ch);
@@ -279,7 +297,7 @@ struct Cur<'a> {
p: usize,
}
impl Cur<'_> {
impl<'a> Cur<'a> {
fn take(&mut self, n: usize) -> Result<&[u8]> {
let end = self.p.checked_add(n).ok_or(DxfError::Eof)?;
let s = self.b.get(self.p..end).ok_or(DxfError::Eof)?;
@@ -309,12 +327,13 @@ impl Cur<'_> {
a.copy_from_slice(s);
Ok(f64::from_le_bytes(a))
}
fn cstr(&mut self) -> Result<String> {
/// The bytes of a zero-terminated string (without the zero), decoded by the caller.
fn cstr(&mut self) -> Result<&'a [u8]> {
let start = self.p;
while self.p < self.b.len() && self.b.get(self.p) != Some(&0) {
self.p += 1;
}
let s = decode_text(self.b.get(start..self.p).unwrap_or(&[])).into_owned();
let s = self.b.get(start..self.p).unwrap_or(&[]);
if self.p >= self.b.len() {
return Err(DxfError::Eof);
}
@@ -330,6 +349,7 @@ pub fn parse_binary(b: &[u8]) -> Result<Vec<Tag>> {
fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
let mut c = Cur { b, p: 0 };
let mut text = codepage::BinaryText::default();
let mut tags = Vec::new();
while c.p < b.len() {
let mut code = i32::from(c.i16()?);
@@ -337,7 +357,7 @@ fn parse_binary_with_limit(b: &[u8], max_tags: usize) -> Result<Vec<Tag>> {
code = i32::from(c.i16()?);
}
let v = match kind_of(code) {
Kind::Str => Value::Str(unescape_unicode(&c.cstr()?)),
Kind::Str => Value::Str(unescape_unicode(&text.string(code, c.cstr()?))),
Kind::Real => Value::Real(c.f64()?),
Kind::Int16 => Value::Int(i64::from(c.i16()?)),
Kind::Int32 => Value::Int(i64::from(c.i32()?)),
+204
View File
@@ -0,0 +1,204 @@
//! Pre-2007 DXF text is in the code page named by `$DWGCODEPAGE` (#505): Cyrillic, Central
//! European, Greek, Japanese, Chinese and Korean drawings open with their layer names, block names
//! and text intact, in ASCII and binary DXF, and a hostile file never panics.
use cadcraft_doc::{Drawing, EntityKind};
/// An ASCII DXF of `version` declaring `code_page`, with a layer, a block and a TEXT whose names
/// and value are the raw bytes given.
fn ascii_dxf(version: &str, code_page: &str, layer: &[u8], block: &[u8], text: &[u8]) -> Vec<u8> {
let mut b = Vec::new();
let mut put = |code: i32, v: &[u8]| {
b.extend_from_slice(format!("{code:>3}\r\n").as_bytes());
b.extend_from_slice(v);
b.extend_from_slice(b"\r\n");
};
for (c, v) in [(0, &b"SECTION"[..]), (2, b"HEADER"), (9, b"$ACADVER"), (1, version.as_bytes()), (9, b"$DWGCODEPAGE"), (3, code_page.as_bytes())] {
put(c, v);
}
for (c, v) in [(0, &b"ENDSEC"[..]), (0, b"SECTION"), (2, b"TABLES"), (0, b"TABLE"), (2, b"LAYER"), (0, b"LAYER"), (2, layer), (70, b"0")] {
put(c, v);
}
for (c, v) in [(62, &b"7"[..]), (6, b"CONTINUOUS"), (0, b"ENDTAB"), (0, b"ENDSEC"), (0, b"SECTION"), (2, b"BLOCKS"), (0, b"BLOCK")] {
put(c, v);
}
for (c, v) in [(8, &b"0"[..]), (2, block), (70, b"0"), (10, b"0"), (20, b"0"), (30, b"0"), (0, b"LINE"), (8, b"0")] {
put(c, v);
}
for (c, v) in [(10, &b"0"[..]), (20, b"0"), (11, b"1"), (21, b"1"), (0, b"ENDBLK"), (0, b"ENDSEC"), (0, b"SECTION"), (2, b"ENTITIES")] {
put(c, v);
}
for (c, v) in [(0, &b"TEXT"[..]), (8, layer), (10, b"0"), (20, b"0"), (40, b"1"), (1, text), (0, b"INSERT"), (8, b"0")] {
put(c, v);
}
for (c, v) in [(2, block), (10, b"5"), (20, b"5"), (0, b"ENDSEC"), (0, b"EOF")] {
put(c, v);
}
b
}
fn text_of(d: &Drawing) -> String {
d.model.iter().find_map(|e| if let EntityKind::Text(t) = &e.kind { Some(t.value.clone()) } else { None }).unwrap_or_default()
}
fn names(d: &Drawing) -> (String, String, String) {
// The TEXT's layer, which the LAYER table defines.
let layer = d.model.iter().find(|e| matches!(e.kind, EntityKind::Text(_))).map(|e| e.common.layer.clone()).unwrap_or_default();
assert!(d.layers.iter().any(|l| l.name == layer), "layer `{layer}` is defined");
let insert = d.model.iter().find_map(|e| if let EntityKind::Insert(i) = &e.kind { Some(i.block.clone()) } else { None });
let block = insert.unwrap_or_default();
assert!(d.block(&block).is_some(), "block `{block}` is defined");
(layer, block, text_of(d))
}
/// Text, layer and block names decode from each code page AutoCAD writes pre-2007 files in.
#[test]
fn reads_text_in_its_code_page() {
// (code page, layer bytes, layer, text bytes, text); byte sequences from the code page tables.
let cases: [(&str, &[u8], &str, &[u8], &str); 10] = [
("ANSI_1251", b"\xD1\xEB\xEE\xE9", "Слой", b"\xCF\xF0\xE8\xE2\xE5\xF2", "Привет"),
("ANSI_1250", b"\xA3\xF3d\x9F", "Łódź", b"\x8Ail\xE1", "Šilá"),
("ANSI_1253", b"\xD3\xF7\xDD\xE4\xE9\xEF", "Σχέδιο", b"\xE1\xE2\xE3", "αβγ"),
("ANSI_1254", b"\xC7izim", "Çizim", b"\xDE\xF0", "Şğ"),
("ANSI_874", b"\xC0\xD2\xBE", "ภาพ", b"\xC0\xD2\xBE", "ภาพ"),
("ANSI_932", b"\x90\x7D\x96\xCA", "図面", b"\x90\x7D\x96\xCA", "図面"),
("ANSI_936", b"\xCD\xBC\xB2\xE3", "图层", b"\xCD\xBC\xB2\xE3", "图层"),
("ANSI_949", b"\xB5\xB5\xB8\xE9", "도면", b"\xB5\xB5\xB8\xE9", "도면"),
("ANSI_950", b"\xB9\xCF\xBC\x68", "圖層", b"\xB9\xCF\xBC\x68", "圖層"),
("DOS850", b"Gr\x94\xE1e", "Größe", b"\x9B\x9D", "øØ"),
];
for (cp, layer_bytes, layer, text_bytes, text) in cases {
let mut block_bytes = b"B_".to_vec();
block_bytes.extend_from_slice(layer_bytes);
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", cp, layer_bytes, &block_bytes, text_bytes)).unwrap();
assert_eq!(names(&d), (layer.to_string(), format!("B_{layer}"), text.to_string()), "{cp}");
}
}
/// A Shift-JIS trail byte can be 0x5C (`\`): 表 is 95 5C. Decoding the bytes before the escapes
/// keeps it a kanji instead of starting a `\U+` escape; real escapes still decode.
#[test]
fn shift_jis_backslash_trail_byte_is_not_an_escape() {
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_932", b"L", b"B", b"\x95\x5CU+0041 \\U+00E9 \\M+5BCFE")).unwrap();
assert_eq!(text_of(&d), "表U+0041 é 件");
}
/// R2007+ files and files whose bytes are valid UTF-8 are UTF-8, whatever code page they declare
/// (the DWG bridge writes UTF-8 with the DWG's code page name); invalid UTF-8 in a 2007+ file
/// falls back to the declared code page.
#[test]
fn utf8_wins_when_valid() {
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_1251", "Слой".as_bytes(), b"B", "Привет".as_bytes())).unwrap();
assert_eq!(names(&d), ("Слой".into(), "B".into(), "Привет".into()));
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1032", "ANSI_1252", "图层".as_bytes(), b"B", "Café".as_bytes())).unwrap();
assert_eq!(names(&d), ("图层".into(), "B".into(), "Café".into()));
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1021", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"B", b"x")).unwrap();
assert_eq!(names(&d).0, "Слой");
}
/// A code page CADCraft doesn't know (or none) reads as ANSI_1252, as before.
#[test]
fn unknown_code_page_falls_back_to_ansi_1252() {
for cp in ["ANSI_1361", "EBCDIC", "", "\u{1}"] {
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", cp, b"Caf\xE9", b"B", b"\x80 \xD8")).unwrap();
assert_eq!(names(&d), ("Café".into(), "B".into(), "€ Ø".into()), "{cp:?}");
}
}
/// Reading converts to Unicode; saving writes pure ASCII with `\U+` escapes, which reads back the
/// same.
#[test]
fn code_page_text_survives_a_round_trip() {
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2")).unwrap();
let out = cadcraft_io::write_dxf(&d);
assert!(out.is_ascii());
let back = cadcraft_io::read_dxf(out.as_bytes()).unwrap();
assert_eq!(names(&back), ("Слой".into(), "Блок".into(), "Привет".into()));
}
/// A binary DXF: R2000 strings are in the code page, R2007+ strings UTF-8.
#[test]
fn binary_dxf_honours_the_code_page() {
let bin = |version: &str, cp: &str, text: &[u8]| {
let mut b = b"AutoCAD Binary DXF\r\n\x1a\0".to_vec();
let mut s = |code: i16, v: &[u8]| {
b.extend_from_slice(&code.to_le_bytes());
b.extend_from_slice(v);
b.push(0);
};
s(0, b"SECTION");
s(2, b"HEADER");
s(9, b"$ACADVER");
s(1, version.as_bytes());
s(9, b"$DWGCODEPAGE");
s(3, cp.as_bytes());
s(0, b"ENDSEC");
s(0, b"SECTION");
s(2, b"ENTITIES");
s(0, b"TEXT");
s(8, b"0");
b.extend_from_slice(&10i16.to_le_bytes());
b.extend_from_slice(&0f64.to_le_bytes());
b.extend_from_slice(&20i16.to_le_bytes());
b.extend_from_slice(&0f64.to_le_bytes());
b.extend_from_slice(&40i16.to_le_bytes());
b.extend_from_slice(&1f64.to_le_bytes());
let mut s = |code: i16, v: &[u8]| {
b.extend_from_slice(&code.to_le_bytes());
b.extend_from_slice(v);
b.push(0);
};
s(1, text);
s(0, b"ENDSEC");
s(0, b"EOF");
b
};
let read = |b: &[u8]| text_of(&cadcraft_io::read_dxf(b).unwrap());
assert_eq!(read(&bin("AC1015", "ANSI_1251", b"\xCF\xF0\xE8\xE2\xE5\xF2")), "Привет");
assert_eq!(read(&bin("AC1018", "ANSI_932", b"\x95\x5C\\U+00E9")), "表é");
// A short Cyrillic string that is also valid UTF-8 (D1 B2) is still read in the code page.
assert_eq!(read(&bin("AC1015", "ANSI_1251", b"\xD1\xB2")), "СІ");
assert_eq!(read(&bin("AC1021", "ANSI_1251", "Привет".as_bytes())), "Привет");
assert_eq!(read(&bin("AC1015", "ANSI_1361", b"Caf\xE9")), "Café");
}
/// Truncated and invalid multibyte sequences become U+FFFD; nothing panics and the structure of
/// the file (line ends, group codes) survives.
#[test]
fn hostile_multibyte_never_panics() {
// A lead byte cut off by the line end, a lone trail byte, an invalid pair.
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_932", b"\x90", b"\x81", b"A\x90")).unwrap();
assert_eq!(names(&d), ("\u{FFFD}".into(), "\u{FFFD}".into(), "A\u{FFFD}".into()));
let d = cadcraft_io::read_dxf(&ascii_dxf("AC1015", "ANSI_936", b"\xFF\xFF", b"B", b"\xA1")).unwrap();
assert_eq!(text_of(&d), "\u{FFFD}");
// A file cut off inside a double-byte character, garbage after the header, every byte value.
let mut cut = ascii_dxf("AC1015", "ANSI_949", b"L", b"B", b"x");
cut.truncate(cut.len() - 10);
cut.push(0xB5);
let _ = cadcraft_io::read_dxf(&cut);
let all: Vec<u8> = (0..=255u8).collect();
for cp in ["ANSI_932", "ANSI_936", "ANSI_949", "ANSI_950", "ANSI_1251", "DOS437", "DOS852", "MACINTOSH"] {
let d = cadcraft_io::read_dxf(&ascii_dxf(
"AC1015",
cp,
b"L",
b"B",
&all.iter().copied().filter(|&b| b != b'\n' && b != b'\r').collect::<Vec<_>>(),
));
assert!(d.is_ok(), "{cp}");
let mut junk = ascii_dxf("AC1015", cp, b"L", b"B", b"x");
junk.extend_from_slice(&all);
let _ = cadcraft_io::read_dxf(&junk);
}
}
/// A pre-2007 DWG keeps its code page through the acadrust bridge: a Cyrillic R2000 DXF saved as
/// DWG (acadrust writes 2004, text in ANSI_1251) opens with its text.
#[cfg(not(target_arch = "wasm32"))]
#[test]
fn pre_2007_dwg_text_decodes_through_the_bridge() {
let dxf = ascii_dxf("AC1015", "ANSI_1251", b"\xD1\xEB\xEE\xE9", b"\xC1\xEB\xEE\xEA", b"\xCF\xF0\xE8\xE2\xE5\xF2");
let dwg = cadcraft_dwg::dxf_to_dwg(&dxf).unwrap();
let d = cadcraft_io::read(&dwg, "x.dwg").unwrap();
assert_eq!(names(&d), ("Слой".into(), "Блок".into(), "Привет".into()));
}