Use shared MTEXT lexical tokens

Cargo.lock

@@ -700,7 +700,6 @@ dependencies = [
  "cadcraft-render",
  "craft-docpath",
  "craft-dxf",
- "craft-mime",
  "libc",
  "log",
  "rstar",
@@ -1135,7 +1134,7 @@ dependencies = [
 [[package]]
 name = "craft-codepage"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=451a390542fbde0519cbb8ade3a3325cf02d97be#451a390542fbde0519cbb8ade3a3325cf02d97be"
+source = "git+https://github.com/storytold/craft-libs?rev=b1d4c67196ee7c6b668428c8d211e7d5171dff88#b1d4c67196ee7c6b668428c8d211e7d5171dff88"
 
 [[package]]
 name = "craft-docpath"
@@ -1154,9 +1153,11 @@ dependencies = [
 [[package]]
 name = "craft-dxf"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=006e4df225fca57e95ead0fb1f91920900798a99#006e4df225fca57e95ead0fb1f91920900798a99"
+source = "git+https://github.com/storytold/craft-libs?rev=b1d4c67196ee7c6b668428c8d211e7d5171dff88#b1d4c67196ee7c6b668428c8d211e7d5171dff88"
 dependencies = [
+ "craft-codepage",
  "encoding_rs",
+ "kurbo",
  "log",
  "thiserror 2.0.21",
 ]
@@ -1189,16 +1190,6 @@ dependencies = [
  "writeable",
 ]
 
-[[package]]
-name = "craft-mime"
-version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=451a390542fbde0519cbb8ade3a3325cf02d97be#451a390542fbde0519cbb8ade3a3325cf02d97be"
-dependencies = [
- "chrono",
- "craft-codepage",
- "encoding_rs",
-]
-
 [[package]]
 name = "craft-pdf"
 version = "0.1.0"

Cargo.toml

@@ -14,9 +14,8 @@ repository = "https://github.com/storytold/cadcraft"
 # internal
 cadcraft-geom = { path = "crates/geom" }
 # DXF tags: the shared reader and writer (moved from crates/dxf; craft-libs#791).
-craft-dxf = { git = "https://github.com/storytold/craft-libs", rev = "006e4df225fca57e95ead0fb1f91920900798a99" }
+craft-dxf = { git = "https://github.com/storytold/craft-libs", rev = "b1d4c67196ee7c6b668428c8d211e7d5171dff88" }
 craft-dwg = { git = "https://github.com/storytold/craft-libs", rev = "7bb6c90076a0177d01cc4c482e7fd8dc9f31a8bf" }
-craft-mime = { git = "https://github.com/storytold/craft-libs", rev = "451a390542fbde0519cbb8ade3a3325cf02d97be" }
 cadcraft-color = { path = "crates/color" }
 cadcraft-doc = { path = "crates/doc" }
 cadcraft-constraints = { path = "crates/constraints" }

crates/engine/Cargo.toml

@@ -12,7 +12,6 @@ workspace = true
 cadcraft-geom = { workspace = true }
 cadcraft-color = { workspace = true }
 cadcraft-doc = { workspace = true }
-craft-mime = { workspace = true }
 craft-dxf = { workspace = true }
 cadcraft-constraints = { workspace = true }
 cadcraft-fonts = { workspace = true }

crates/engine/src/cmd/file.rs

@@ -1020,14 +1020,46 @@ fn run_saveas_reporting(s: &mut Session, p: &Value) -> Result<Value> {
     save_reporting(s, p, run_saveas, asked)
 }
 
-/// Encode bytes with the standard padded Base64 alphabet.
+const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
 pub fn base64_encode(data: &[u8]) -> String {
-    craft_mime::base64::STANDARD.encode(data)
+    let mut out = String::with_capacity(data.len().div_ceil(3) * 4);
+    for chunk in data.chunks(3) {
+        let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
+        let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
+        for i in 0..4 {
+            if i <= chunk.len() {
+                out.push(char::from(B64[((n >> (18 - 6 * i)) & 63) as usize]));
+            } else {
+                out.push('=');
+            }
+        }
+    }
+    out
 }
 
-/// Decode the legacy file/control Base64 form, retaining its permissive alphabet and padding.
 pub fn base64_decode(s: &str) -> Option<Vec<u8>> {
-    craft_mime::base64_compat::decode(s).ok()
+    let mut out = Vec::with_capacity(s.len() * 3 / 4);
+    let mut buf = 0u32;
+    let mut bits = 0;
+    for c in s.bytes() {
+        let v = match c {
+            b'A'..=b'Z' => c - b'A',
+            b'a'..=b'z' => c - b'a' + 26,
+            b'0'..=b'9' => c - b'0' + 52,
+            b'+' | b'-' => 62,
+            b'/' | b'_' => 63,
+            b'=' | b'\n' | b'\r' | b' ' => continue,
+            _ => return None,
+        };
+        buf = (buf << 6) | u32::from(v);
+        bits += 6;
+        if bits >= 8 {
+            bits -= 8;
+            out.push(((buf >> bits) & 0xff) as u8);
+        }
+    }
+    Some(out)
 }
 
 #[cfg(test)]

crates/engine/tests/base64_compat.rs

@@ -1,14 +0,0 @@
-//! File and automation inputs retain CADCraft's accepted Base64 forms.
-use cadcraft_engine::cmd::file::{base64_decode, base64_encode};
-
-#[test]
-fn legacy_inputs_and_standard_outputs_survive_the_shared_cutover() {
-    for bytes in [&b""[..], b"f", b"fo", b"foo", b"foobar", &[0, 255, 127]] {
-        assert_eq!(base64_decode(&base64_encode(bytes)).as_deref(), Some(bytes));
-    }
-    assert_eq!(base64_decode("+_8=").as_deref(), Some(&[251, 255][..]));
-    assert_eq!(base64_decode("Z=m 9\nv\r").as_deref(), Some(&b"foo"[..]));
-    assert_eq!(base64_decode("A").as_deref(), Some(&b""[..]));
-    assert!(base64_decode("Zm9v\t").is_none());
-    assert!(base64_decode("Zm9v!").is_none());
-}

crates/fonts/src/mtext.rs

@@ -12,79 +12,22 @@ pub fn plain_mtext(s: &str) -> String {
     let s = unescape_codes(s);
     let s = s.as_ref();
     let mut out = String::with_capacity(s.len());
-    let chars: Vec<char> = s.chars().collect();
-    let mut i = 0;
-    while let Some(&c) = chars.get(i) {
-        match c {
-            '\\' => {
-                let code = chars.get(i + 1).copied().unwrap_or(' ');
-                match code {
-                    'P' => {
-                        out.push('\n');
-                        i += 2;
-                    }
-                    'N' => {
-                        out.push('\n');
-                        i += 2;
-                    }
-                    '~' => {
-                        out.push(' ');
-                        i += 2;
-                    }
-                    '\\' | '{' | '}' => {
-                        out.push(code);
-                        i += 2;
-                    }
-                    'S' => {
-                        // \Snum^den; \Snum/den; \Snum#den;
-                        let mut j = i + 2;
-                        let mut stack = String::new();
-                        while let Some(&ch) = chars.get(j) {
-                            if ch == ';' {
-                                break;
-                            }
-                            stack.push(match ch {
-                                '^' | '#' => '/',
-                                c => c,
-                            });
-                            j += 1;
-                        }
-                        // "6\S1/2;" reads as "6 1/2".
-                        if out.chars().last().is_some_and(|c| c.is_ascii_digit()) && stack.starts_with(|c: char| c.is_ascii_digit()) {
-                            out.push(' ');
-                        }
-                        out.push_str(stack.trim_end_matches('/'));
-                        i = j + 1;
-                    }
-                    'L' | 'l' | 'O' | 'o' | 'K' | 'k' => i += 2,
-                    // `\U+XXXX`: a character by its Unicode code, as the layout reads it.
-                    'U' if crate::unicode_escape(&chars, i).is_some() => {
-                        if let Some(u) = crate::unicode_escape(&chars, i) {
-                            out.push(u);
-                        }
-                        i += 7;
-                    }
-                    // Codes with an argument terminated by ';' (\f, \F, \H, \W, \Q, \T, \A, \C, \c, \p).
-                    _ if takes_arg(code) => {
-                        let mut j = i + 2;
-                        while let Some(&ch) = chars.get(j) {
-                            if ch == ';' {
-                                break;
-                            }
-                            j += 1;
-                        }
-                        i = j + 1;
-                    }
-                    // An unknown code has no argument: drop the backslash and the code only.
-                    _ => i += 2,
+    use craft_dxf::mtext_tokens::{StackMode, Token, Tokens};
+    for token in Tokens::new(s, StackMode::Plain) {
+        match token {
+            Token::Control('P' | 'N') => out.push('\n'),
+            Token::Control('~') => out.push(' '),
+            Token::Control(c @ ('\\' | '{' | '}')) => out.push(c),
+            Token::Stack(raw) => {
+                let stack: String = raw.chars().map(|c| if matches!(c, '^' | '#') { '/' } else { c }).collect();
+                if out.chars().last().is_some_and(|c| c.is_ascii_digit()) && stack.starts_with(|c: char| c.is_ascii_digit()) {
+                    out.push(' ');
                 }
+                out.push_str(stack.trim_end_matches('/'));
             }
-            '{' | '}' => i += 1,
-            '\r' => i += 1,
-            _ => {
-                out.push(c);
-                i += 1;
-            }
+            Token::Char('\r') => {}
+            Token::Char(c) => out.push(c),
+            _ => {}
         }
     }
     out
@@ -225,11 +168,8 @@ const MAX_ATOMS: usize = 200_000;
 /// Scale of stacked text relative to the surrounding text.
 const STACK_SCALE: f64 = 0.7;
 
-/// Codes that take an argument terminated by `;` (besides `\S`, which has its own scan). Any
-/// other code stands alone, so a missing `;` must not swallow the text after it.
-pub(crate) fn takes_arg(code: char) -> bool {
-    matches!(code, 'f' | 'F' | 'H' | 'W' | 'Q' | 'T' | 'A' | 'C' | 'c' | 'p')
-}
+// Shared DXF escape handling and parameterized-command classification.
+pub(crate) use craft_dxf::mtext::unescape_codes;
 
 /// The line-spacing factor to lay out with: AutoCAD's 0.25..=4 range, 1 for a missing or
 /// non-finite value.
@@ -237,90 +177,6 @@ pub(crate) fn sane_line_spacing(v: f64) -> f64 {
     if v.is_finite() && v > 0.0 { v.clamp(0.25, 4.0) } else { 1.0 }
 }
 
-/// Four hex digits at `chars[at..at + 4]`.
-fn hex4(chars: &[char], at: usize) -> Option<u32> {
-    let digits = chars.get(at..at.checked_add(4)?)?;
-    digits.iter().try_fold(0u32, |acc, c| Some(acc * 16 + c.to_digit(16)?))
-}
-
-/// Push a decoded character so that the tokenizers read it as text, never as markup.
-fn push_literal(out: &mut String, c: char) {
-    if matches!(c, '\\' | '{' | '}') {
-        out.push('\\');
-    }
-    out.push(c);
-}
-
-/// Decode `\U+hhhh` (one UTF-16 unit; a surrogate pair gives one character, a lone surrogate
-/// U+FFFD) and `\M+nhhhh` (double-byte East Asian characters) to the characters they stand
-/// for. Malformed escapes are left for the tokenizers, which drop the backslash and the code.
-/// An escaped backslash (`\\U+0041`) is not an escape.
-pub(crate) fn unescape_codes(s: &str) -> std::borrow::Cow<'_, str> {
-    if !s.contains("\\U+") && !s.contains("\\M+") {
-        return std::borrow::Cow::Borrowed(s);
-    }
-    let chars: Vec<char> = s.chars().collect();
-    let mut out = String::with_capacity(s.len());
-    let mut i = 0;
-    while let Some(&c) = chars.get(i) {
-        if c != '\\' {
-            out.push(c);
-            i += 1;
-            continue;
-        }
-        let (n, plus) = (chars.get(i + 1).copied(), chars.get(i + 2).copied());
-        if n == Some('U') && plus == Some('+') {
-            if let Some(unit) = hex4(&chars, i + 3) {
-                i += 7;
-                let low = |at: usize| {
-                    (chars.get(at) == Some(&'\\') && chars.get(at + 1) == Some(&'U') && chars.get(at + 2) == Some(&'+'))
-                        .then(|| hex4(&chars, at + 3))
-                        .flatten()
-                        .filter(|u| (0xDC00..0xE000).contains(u))
-                };
-                let ch = if (0xD800..0xDC00).contains(&unit) {
-                    match low(i) {
-                        Some(lo) => {
-                            i += 7;
-                            char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (lo - 0xDC00))
-                        }
-                        None => None,
-                    }
-                } else {
-                    char::from_u32(unit)
-                };
-                push_literal(&mut out, ch.unwrap_or('\u{fffd}'));
-                continue;
-            }
-        } else if n == Some('M') && plus == Some('+') && chars.get(i + 3).is_some_and(char::is_ascii_digit) && hex4(&chars, i + 4).is_some() {
-            push_literal(&mut out, craft_dxf::mif_escape_at(&chars, i).unwrap_or('\u{fffd}'));
-            i += 8;
-            continue;
-        }
-        // Any other backslash pair stays as it is (and an escaped backslash stays one).
-        out.push('\\');
-        i += 1;
-        if let Some(&next) = chars.get(i) {
-            out.push(next);
-            i += 1;
-        }
-    }
-    std::borrow::Cow::Owned(out)
-}
-
-fn arg(chars: &[char], from: usize) -> (String, usize) {
-    let mut j = from;
-    let mut a = String::new();
-    while let Some(&ch) = chars.get(j) {
-        if ch == ';' {
-            return (a, j + 1);
-        }
-        a.push(ch);
-        j += 1;
-    }
-    (a, j)
-}
-
 fn num(a: &str) -> Option<f64> {
     a.trim().parse::<f64>().ok().filter(|v| v.is_finite())
 }
@@ -358,7 +214,7 @@ fn font_switch(a: &str) -> Option<TextFont> {
 
 /// Parse MTEXT contents into words, spaces and paragraph breaks with their formats.
 fn parse(contents: &str, base: Fmt) -> (Vec<Atom>, Vec<Fmt>) {
-    let chars: Vec<char> = contents.chars().collect();
+    use craft_dxf::mtext_tokens::{StackMode, Token, Tokens, split_stack};
     let h_min = base.height * 1e-3;
     let h_max = if (base.height * 1e3).is_finite() { base.height * 1e3 } else { f64::MAX };
     let mut fmts = vec![base];
@@ -398,158 +254,94 @@ fn parse(contents: &str, base: Fmt) -> (Vec<Atom>, Vec<Fmt>) {
             *cur = fmts.len() - 1;
         }
     };
-    let mut i = 0;
-    while let Some(&c) = chars.get(i) {
+    for token in Tokens::new(contents, StackMode::Structured) {
         if atoms.len() > MAX_ATOMS {
             break;
         }
-        match c {
-            '\\' => {
-                let code = chars.get(i + 1).copied().unwrap_or(' ');
-                i += 2;
-                match code {
-                    'P' | 'N' => {
-                        end_word(&mut atoms, &mut word);
-                        atoms.push(Atom::Para(cur));
-                    }
-                    '~' => push_char(&mut word, '\u{a0}', cur),
-                    '\\' | '{' | '}' => push_char(&mut word, code, cur),
-                    'L' => derive(&mut fmts, &mut cur, &|f| f.under = true),
-                    'l' => derive(&mut fmts, &mut cur, &|f| f.under = false),
-                    'O' => derive(&mut fmts, &mut cur, &|f| f.over = true),
-                    'o' => derive(&mut fmts, &mut cur, &|f| f.over = false),
-                    'K' | 'k' => {}
-                    // `\U+XXXX`: a character by its Unicode code.
-                    'U' if i >= 2 && crate::unicode_escape(&chars, i - 2).is_some() => {
-                        if let Some(u) = crate::unicode_escape(&chars, i - 2) {
-                            push_char(&mut word, u, cur);
-                        }
-                        i += 5;
-                    }
-                    'S' => {
-                        let mut j = i;
-                        let (mut top, mut bottom, mut sep) = (String::new(), String::new(), None);
-                        while let Some(&ch) = chars.get(j) {
-                            if ch == ';' {
-                                break;
-                            }
-                            if ch == '\\' {
-                                if let Some(&n) = chars.get(j + 1) {
-                                    if sep.is_none() { top.push(n) } else { bottom.push(n) }
-                                }
-                                j += 2;
-                                continue;
-                            }
-                            if sep.is_none() && matches!(ch, '/' | '#' | '^') {
-                                sep = Some(ch);
-                            } else if sep.is_none() {
-                                top.push(ch);
-                            } else {
-                                bottom.push(ch);
-                            }
-                            j += 1;
-                        }
-                        i = j + 1;
-                        word.push((Seg::Stack(top, bottom, sep.unwrap_or('/')), cur));
-                    }
-                    'f' | 'F' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        if let Some(font) = font_switch(&a) {
-                            derive(&mut fmts, &mut cur, &|f| f.font = font.clone());
-                        }
-                    }
-                    'H' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        let (rel, v) = match a.trim().strip_suffix(['x', 'X']) {
-                            Some(r) => (true, num(r)),
-                            None => (false, num(&a)),
-                        };
-                        if let Some(v) = v.filter(|v| *v > 0.0 && *v < 1e6) {
-                            // Relative heights compound; keep the result within 1e-3..1e3 times the
-                            // MTEXT's own height, never infinite.
-                            derive(&mut fmts, &mut cur, &|f| f.height = if rel { (f.height * v).min(h_max).max(h_min) } else { v });
-                        }
-                    }
-                    'W' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        // `\W2x;` multiplies the current factor (like `\H2x;`); `\W2;` sets it.
-                        let (rel, v) = match a.trim().strip_suffix(['x', 'X']) {
-                            Some(r) => (true, num(r)),
-                            None => (false, num(&a)),
-                        };
-                        if let Some(v) = v.filter(|v| *v > 0.01 && *v < 100.0) {
-                            // Relative factors compound; keep the result within the absolute range.
-                            derive(&mut fmts, &mut cur, &|f| f.wf = if rel { (f.wf * v).clamp(0.01, 100.0) } else { v });
-                        }
-                    }
-                    'Q' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        if let Some(v) = num(&a).filter(|v| v.abs() < 85.0) {
-                            derive(&mut fmts, &mut cur, &|f| f.oblique = v.to_radians());
-                        }
-                    }
-                    'C' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        if let Ok(v) = a.trim().parse::<u16>() {
-                            let col = if v == 0 || v >= 256 { None } else { Some(MTextColor::Aci(v)) };
-                            derive(&mut fmts, &mut cur, &|f| f.color = col);
-                        }
-                    }
-                    'c' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        if let Ok(v) = a.trim().parse::<u32>() {
-                            derive(&mut fmts, &mut cur, &|f| f.color = Some(MTextColor::Rgb(v & 0xFF_FFFF)));
-                        }
-                    }
-                    'A' => {
-                        let (a, n) = arg(&chars, i);
-                        i = n;
-                        if let Ok(v @ 0..=2) = a.trim().parse::<u8>() {
-                            derive(&mut fmts, &mut cur, &|f| f.valign = v);
-                        }
-                    }
-                    _ if takes_arg(code) => {
-                        // \T and \p: argument up to ';'.
-                        let (_, n) = arg(&chars, i);
-                        i = n;
-                    }
-                    // An unknown code has no argument: only the backslash and the code are dropped.
-                    _ => {}
+        match token {
+            Token::Control('P' | 'N') => {
+                end_word(&mut atoms, &mut word);
+                atoms.push(Atom::Para(cur));
+            }
+            Token::Control('~') => push_char(&mut word, '\u{a0}', cur),
+            Token::Control(c @ ('\\' | '{' | '}')) => push_char(&mut word, c, cur),
+            Token::Control('L') => derive(&mut fmts, &mut cur, &|f| f.under = true),
+            Token::Control('l') => derive(&mut fmts, &mut cur, &|f| f.under = false),
+            Token::Control('O') => derive(&mut fmts, &mut cur, &|f| f.over = true),
+            Token::Control('o') => derive(&mut fmts, &mut cur, &|f| f.over = false),
+            Token::Stack(raw) => {
+                let (top, bottom, sep) = split_stack(raw);
+                word.push((Seg::Stack(top, bottom, sep.unwrap_or('/')), cur));
+            }
+            Token::Argument('f' | 'F', a) => {
+                if let Some(font) = font_switch(a) {
+                    derive(&mut fmts, &mut cur, &|f| f.font = font.clone());
                 }
             }
-            '{' => {
+            Token::Argument('H', a) => {
+                let (rel, v) = match a.trim().strip_suffix(['x', 'X']) {
+                    Some(r) => (true, num(r)),
+                    None => (false, num(a)),
+                };
+                if let Some(v) = v.filter(|v| *v > 0.0 && *v < 1e6) {
+                    // Relative heights compound; keep the result within 1e-3..1e3 times the
+                    // MTEXT's own height, never infinite.
+                    derive(&mut fmts, &mut cur, &|f| f.height = if rel { (f.height * v).min(h_max).max(h_min) } else { v });
+                }
+            }
+            Token::Argument('W', a) => {
+                // `\W2x;` multiplies the current factor (like `\H2x;`); `\W2;` sets it.
+                let (rel, v) = match a.trim().strip_suffix(['x', 'X']) {
+                    Some(r) => (true, num(r)),
+                    None => (false, num(a)),
+                };
+                if let Some(v) = v.filter(|v| *v > 0.01 && *v < 100.0) {
+                    // Relative factors compound; keep the result within the absolute range.
+                    derive(&mut fmts, &mut cur, &|f| f.wf = if rel { (f.wf * v).clamp(0.01, 100.0) } else { v });
+                }
+            }
+            Token::Argument('Q', a) => {
+                if let Some(v) = num(a).filter(|v| v.abs() < 85.0) {
+                    derive(&mut fmts, &mut cur, &|f| f.oblique = v.to_radians());
+                }
+            }
+            Token::Argument('C', a) => {
+                if let Ok(v) = a.trim().parse::<u16>() {
+                    let col = if v == 0 || v >= 256 { None } else { Some(MTextColor::Aci(v)) };
+                    derive(&mut fmts, &mut cur, &|f| f.color = col);
+                }
+            }
+            Token::Argument('c', a) => {
+                if let Ok(v) = a.trim().parse::<u32>() {
+                    derive(&mut fmts, &mut cur, &|f| f.color = Some(MTextColor::Rgb(v & 0xFF_FFFF)));
+                }
+            }
+            Token::Argument('A', a) => {
+                if let Ok(v @ 0..=2) = a.trim().parse::<u8>() {
+                    derive(&mut fmts, &mut cur, &|f| f.valign = v);
+                }
+            }
+            Token::OpenGroup => {
                 if stack.len() < 256 {
                     stack.push(cur);
                 }
-                i += 1;
             }
-            '}' => {
+            Token::CloseGroup => {
                 if let Some(f) = stack.pop() {
                     cur = f;
                 }
-                i += 1;
             }
-            '\r' => i += 1,
-            '\n' => {
+            Token::Char('\r') => {}
+            Token::Char('\n') => {
                 end_word(&mut atoms, &mut word);
                 atoms.push(Atom::Para(cur));
-                i += 1;
             }
-            ' ' => {
+            Token::Char(' ') => {
                 end_word(&mut atoms, &mut word);
                 atoms.push(Atom::Space(cur));
-                i += 1;
-            }
-            _ => {
-                push_char(&mut word, c, cur);
-                i += 1;
             }
+            Token::Char(c) => push_char(&mut word, c, cur),
+            _ => {}
         }
     }
     end_word(&mut atoms, &mut word);

crates/fonts/src/mtext/para.rs

@@ -7,7 +7,7 @@
 
 use cadcraft_geom::{Bounds2, Vec2};
 
-use super::{Atom, Fmt, MTextLayout, MTextParams, MTextPiece, Seg, arg, num, parse, seg_plain, seg_shape, seg_width, split_thai};
+use super::{Atom, Fmt, MTextLayout, MTextParams, MTextPiece, Seg, num, parse, seg_plain, seg_shape, seg_width, split_thai};
 
 /// Columns an MTEXT's text flows through, each as wide as the MTEXT's width.
 #[derive(Clone, Debug, Default, PartialEq)]
@@ -265,58 +265,28 @@ pub struct Paragraphs {
 impl Paragraphs {
     pub fn scan(contents: &str) -> Paragraphs {
         let contents = super::unescape_codes(contents);
-        let chars: Vec<char> = contents.chars().collect();
+        use craft_dxf::mtext_tokens::{StackMode, Token, Tokens};
         let mut out = Paragraphs { states: vec![ParaProps::default()], ..Default::default() };
         let mut cur = 0usize;
-        let mut i = 0;
         let brk = |out: &mut Paragraphs, cur: usize, column: bool| {
             out.para.push(cur);
             out.column_break.push(column);
         };
-        while let Some(&c) = chars.get(i) {
-            match c {
-                '\\' => {
-                    let code = chars.get(i + 1).copied().unwrap_or(' ');
-                    i += 2;
-                    match code {
-                        'P' => brk(&mut out, cur, false),
-                        'N' => brk(&mut out, cur, true),
-                        '~' | '\\' | '{' | '}' | 'L' | 'l' | 'O' | 'o' | 'K' | 'k' => {}
-                        'S' => {
-                            let mut j = i;
-                            while let Some(&ch) = chars.get(j) {
-                                if ch == ';' {
-                                    break;
-                                }
-                                j += if ch == '\\' { 2 } else { 1 };
-                            }
-                            i = j + 1;
-                        }
-                        'p' => {
-                            let (a, n) = arg(&chars, i);
-                            i = n;
-                            if out.states.len() < MAX_STATES
-                                && let Some(st) = out.states.get(cur)
-                            {
-                                let mut st = st.clone();
-                                st.apply(&a);
-                                out.states.push(st);
-                                cur = out.states.len() - 1;
-                            }
-                        }
-                        _ if super::takes_arg(code) => {
-                            let (_, n) = arg(&chars, i);
-                            i = n;
-                        }
-                        // An unknown code has no argument: only the backslash and the code go.
-                        _ => {}
+        for token in Tokens::new(&contents, StackMode::Structured) {
+            match token {
+                Token::Control('P') | Token::Char('\n') => brk(&mut out, cur, false),
+                Token::Control('N') => brk(&mut out, cur, true),
+                Token::Argument('p', a) => {
+                    if out.states.len() < MAX_STATES
+                        && let Some(st) = out.states.get(cur)
+                    {
+                        let mut st = st.clone();
+                        st.apply(a);
+                        out.states.push(st);
+                        cur = out.states.len() - 1;
                     }
                 }
-                '\n' => {
-                    brk(&mut out, cur, false);
-                    i += 1;
-                }
-                _ => i += 1,
+                _ => {}
             }
         }
         // A `\p` anywhere in a paragraph applies to all of it: the break pushed the state at