Type: Thai, Lao, Khmer and Myanmar lines break between words

These scripts don't separate words with spaces, and lines only broke at
spaces, hyphens and CJK characters, so a Thai paragraph in area type ran
on to the next space (or was cut wherever it overflowed). Each
paragraph's glyphs now carry the word ends ICU4X's line segmenter finds
in text of those scripts (its dictionaries on the desktop, its far
smaller LSTM models on the web), and a line may break after them as
after a space. Other text keeps its own breaks, and clusters (a
consonant with its vowel and tone marks) still never split.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bflatastic
2026-10-11 19:31:37 +07:00
co-authored by Claude Opus 5.5
parent 36494d0385
commit 5a855971aa
7 changed files with 138 additions and 4 deletions
Generated
+52
View File
@@ -1860,11 +1860,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
dependencies = [
"displaydoc",
"litemap",
"serde",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_locale_fallback"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9"
dependencies = [
"icu_locale_core",
"icu_locale_fallback_data",
"icu_provider",
"potential_utf",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locale_fallback_data"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8"
[[package]]
name = "icu_normalizer"
version = "2.3.0"
@@ -1914,6 +1935,8 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
dependencies = [
"displaydoc",
"icu_locale_core",
"serde",
"stable_deref_trait",
"writeable",
"yoke",
"zerofrom",
@@ -1921,6 +1944,29 @@ dependencies = [
"zerovec",
]
[[package]]
name = "icu_segmenter"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db"
dependencies = [
"core_maths",
"icu_collections",
"icu_locale_fallback",
"icu_provider",
"icu_segmenter_data",
"potential_utf",
"smallvec",
"utf8_iter",
"zerovec",
]
[[package]]
name = "icu_segmenter_data"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad"
[[package]]
name = "idna"
version = "1.1.0"
@@ -3152,6 +3198,8 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
dependencies = [
"serde_core",
"writeable",
"zerovec",
]
@@ -4133,6 +4181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
dependencies = [
"displaydoc",
"serde_core",
"zerovec",
]
@@ -4806,6 +4855,7 @@ name = "vectorcraft-text"
version = "0.9.0"
dependencies = [
"harfrust 0.13.3",
"icu_segmenter",
"kurbo 0.13.1",
"log",
"skrifa 0.47.0",
@@ -6184,6 +6234,7 @@ dependencies = [
"displaydoc",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
@@ -6192,6 +6243,7 @@ version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
dependencies = [
"serde",
"yoke",
"zerofrom",
"zerovec-derive",
+1 -1
View File
File diff suppressed because one or more lines are too long
+4
View File
@@ -21,6 +21,10 @@ subsetter = { version = "0.2", default-features = false, features = ["variable-f
unicode-bidi = "0.3.18"
unicode-general-category = "1"
unicode-script = "0.5"
# Word boundaries in scripts written without spaces between words (Thai, Lao, Khmer, Myanmar), where
# lines may break: ICU4X's line segmenter (Unicode-3.0, as icu_properties already in the build),
# with its dictionaries on the desktop and its far smaller LSTM models on the web.
icu_segmenter = "2.3"
# Grapheme clusters (a Thai consonant with its vowel and tone marks, an accented letter, an emoji
# sequence) for caret movement; MIT OR Apache-2.0, already in the build.
unicode-segmentation = "1.13"
+5 -1
View File
@@ -13,7 +13,8 @@ use crate::composer::{Breakpoint, Params, compose};
use crate::fontdb::FontDb;
use crate::hyphen::hyphen_points;
use crate::shape::{
Punct, SGlyph, Tcy, cap_x_heights, hyphen_glyph, is_cjk, no_line_end, no_line_start, punct, shape_range, soft_line_start, style_metrics,
Punct, SGlyph, Tcy, cap_x_heights, hyphen_glyph, is_cjk, mark_word_ends, no_line_end, no_line_start, punct, shape_range, soft_line_start,
style_metrics,
};
use crate::{Composer, FirstBaseline, InlineGlyph, LayoutOptions, LineInfo, OtFeatures, PositionedGlyph, TextLayout, VerticalAlign};
@@ -1184,6 +1185,7 @@ mod wrapping_tests {
ch,
tcy: None,
inline: None,
word_end: false,
lead: 0.0,
proportional: false,
vpal: None,
@@ -1442,6 +1444,8 @@ fn flow(cx: &mut Ctx<'_>, paras: &[Range<usize>], t: &TextObject, regions: Optio
let bidi = para_bidi(text.get(pr.clone()).unwrap_or_default(), para.direction);
let rtl = is_rtl(bidi.as_ref());
let mut sg = cx.shape_para(pr.clone(), bidi.as_ref());
// Thai, Lao, Khmer and Myanmar break lines between words, not at spaces.
mark_word_ends(text.get(pr.clone()).unwrap_or_default(), pr.start, &mut sg);
// Japanese composition: consecutive punctuation shares one half-em space.
let compressed = if para.mojikumi == Mojikumi::LineEndHalf { compress_punctuation(&mut sg) } else { vec![] };
// …and a quarter em between Japanese and Latin text.
+34 -1
View File
@@ -59,6 +59,9 @@ pub(crate) struct SGlyph {
pub level: Level,
/// An inline graphic ([`vectorcraft_doc::TextRun::inline`]) standing in for a glyph.
pub inline: Option<InlineBox>,
/// A word of a script written without spaces (Thai, Lao, Khmer, Myanmar) ends after the
/// glyph's cluster, so a line may break there ([`mark_word_ends`]).
pub word_end: bool,
}
/// How `vpal` moves an upright glyph in vertical type and changes its length down the column, in
@@ -102,7 +105,7 @@ impl SGlyph {
}
/// A line may break after this glyph.
pub fn break_after(&self) -> bool {
self.is_space() || matches!(self.ch, '-' | '\u{2010}' | '\u{2013}' | '\u{2014}' | '/' | SOFT_HYPHEN) || is_cjk(self.ch)
self.is_space() || self.word_end || matches!(self.ch, '-' | '\u{2010}' | '\u{2013}' | '\u{2014}' | '/' | SOFT_HYPHEN) || is_cjk(self.ch)
}
/// A soft (discretionary) hyphen: invisible unless a line breaks after it.
pub fn is_soft_hyphen(&self) -> bool {
@@ -232,6 +235,34 @@ pub(crate) fn punct(c: char) -> Option<Punct> {
}
}
/// Is `c` of a script written without spaces between words, whose line breaks need a model of its
/// words: Thai, Lao, Khmer or Myanmar (UAX #14's class SA)?
pub(crate) fn is_unspaced_script(c: char) -> bool {
matches!(c.script(), Script::Thai | Script::Lao | Script::Khmer | Script::Myanmar)
}
/// Mark the glyphs of `sg` (shaped from `text`, which starts at byte `start` of the text the
/// glyphs index) after which a word of Thai, Lao, Khmer or Myanmar ends: where a line may break in
/// text without spaces between words. The words come from ICU4X's line segmenter (its LSTM models
/// for those scripts); other text keeps its own breaks.
pub(crate) fn mark_word_ends(text: &str, start: usize, sg: &mut [SGlyph]) {
// The dictionaries on the desktop (the most accurate, a few megabytes); the far smaller LSTM
// models on the web.
#[cfg(not(target_arch = "wasm32"))]
static SEGMENTER: std::sync::LazyLock<icu_segmenter::LineSegmenterBorrowed<'static>> =
std::sync::LazyLock::new(|| icu_segmenter::LineSegmenter::new_dictionary(Default::default()));
#[cfg(target_arch = "wasm32")]
static SEGMENTER: std::sync::LazyLock<icu_segmenter::LineSegmenterBorrowed<'static>> =
std::sync::LazyLock::new(|| icu_segmenter::LineSegmenter::new_auto(Default::default()));
if !text.chars().any(is_unspaced_script) {
return;
}
let breaks: Vec<usize> = SEGMENTER.segment_str(text).map(|b| b + start).collect();
for g in sg.iter_mut().filter(|g| g.inline.is_none() && is_unspaced_script(g.ch)) {
g.word_end = breaks.binary_search(&(g.byte + g.len)).is_ok();
}
}
pub(crate) fn is_cjk(c: char) -> bool {
matches!(c as u32, 0x2E80..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF | 0xFF00..=0xFFEF | 0x20000..=0x2FFFF)
}
@@ -446,6 +477,7 @@ fn inline_glyph(face: &Arc<FontFace>, st: &CharStyle, art: &InlineArt, run: usiz
// RTL text like any other glyph once the line is reordered.
level,
inline: Some(InlineBox { art: drawn, xf }),
word_end: false,
}
}
@@ -614,6 +646,7 @@ fn shape_segment(text: &str, context: &Range<usize>, seg: &Segment, feats: &OtFe
vpal: vpal.get(gi).copied().flatten(),
level: *level,
inline: None,
word_end: false,
});
}
}
+39
View File
@@ -728,3 +728,42 @@ fn burasagari_leaves_latin_commas_and_full_stops_inside_the_line() {
}
}
}
/// Thai is written without spaces between words: area type breaks its lines between words (as a
/// model of Thai words finds them), never inside a word nor before a vowel or tone mark.
#[test]
fn thai_lines_break_between_words() {
let text = "สวัสดีครับ ภาษาไทยง่ายนิดเดียว";
// Narrower than the shortest words: each line holds one word.
let l = layout(db(), &area(text, style(12.0), Rect::new(0.0, 0.0, 70.0, 2000.0), Justify::Left));
let lines: Vec<&str> = l.lines.iter().map(|ln| &text[ln.start..ln.end]).collect();
assert!(lines.len() >= 3, "{lines:?}");
// Its words: สวัสดี ครับ, ภาษา ไทย ง่าย นิดเดียว ("hello", "the Thai language is very easy").
let words = ["สวัสดี", "ครับ ", "ภาษา", "ไทย", "ง่าย", "นิดเดียว"];
let ends: Vec<usize> = words
.iter()
.scan(0, |at, w| {
*at += w.len();
Some(*at)
})
.collect();
assert_eq!(ends.last(), Some(&text.len()));
for ln in &l.lines {
assert!(ends.contains(&ln.end), "a line ends inside a word: {lines:?}");
}
for w in l.lines.windows(2) {
assert_eq!(w[0].end, w[1].start, "lines tile the text");
}
let mark = |c: char| matches!(c, '\u{0E31}' | '\u{0E34}'..='\u{0E3A}' | '\u{0E47}'..='\u{0E4E}');
assert!(lines.iter().all(|s| s.chars().next().is_some_and(|c| !mark(c))), "a line starts with a mark: {lines:?}");
}
/// The desktop's dictionaries keep loanwords whole that a smaller model splits: โปรแกรม
/// ("program").
#[test]
fn thai_loanwords_stay_whole() {
let text = "ยินดีต้อนรับสู่โปรแกรมวาดภาพเวกเตอร์";
let l = layout(db(), &area(text, style(12.0), Rect::new(0.0, 0.0, 70.0, 2000.0), Justify::Left));
let lines: Vec<&str> = l.lines.iter().map(|ln| &text[ln.start..ln.end]).collect();
assert!(lines.contains(&"โปรแกรม"), "{lines:?}");
}
+3 -1
View File
@@ -1,6 +1,6 @@
# Type parity
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (the caret steps over whole clusters such as Thai consonants with their marks; Thai typed in a font without Thai falls back to a font made for Thai; earlier: Proportional Metrics in vertical type, `vpal`, #966) · **Target:** Adobe Illustrator 2026 (30.x)
> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-11 · **Change:** minor (Thai, Lao, Khmer and Myanmar lines break between words; the caret steps over whole clusters such as Thai consonants with their marks; Thai typed in a font without Thai falls back to a font made for Thai; earlier: Proportional Metrics in vertical type, `vpal`, #966) · **Target:** Adobe Illustrator 2026 (30.x)
Type is Illustrator's largest single area by use (weight 13 of 105 across "Type core" and "Type advanced" in
[target-app-parity.md](target-app-parity.md)). This checklist lists the features one by one. Status is from the
@@ -41,6 +41,7 @@ against Illustrator's public documentation; **estimated** unless noted.
| Touch Type tool | missing | listed in the toolbar, not implemented |
| Retype (identify fonts in outlines/images) | missing | Illustrator's uses a cloud model |
| Snap to Glyph | missing | menu stub |
| Thai, Lao, Khmer, Myanmar: shaping (marks, reordering), lines broken between words | done | words found by ICU4X's line segmenter: dictionaries on the desktop, LSTM models on the web |
| Middle Eastern: bidirectional layout, Paragraph Direction | done | |
| Middle Eastern: Character Direction, digit types, kashidas, Middle Eastern composers, split caret | missing | |
| CJK: input methods (marked text, candidate window) | done | |
@@ -53,6 +54,7 @@ against Illustrator's public documentation; **estimated** unless noted.
| Date | Change | Summary |
|---|---|---|
| 2026-10-11 | minor | Thai, Lao, Khmer and Myanmar text breaks its lines between words (ICU4X line segmenter) instead of only at spaces; score unchanged (estimated) |
| 2026-10-11 | minor | The Type tool's arrows and Delete move by grapheme cluster, never between a Thai consonant and its vowel or tone marks; score unchanged (estimated) |
| 2026-10-11 | minor | Thai (Lao, Khmer, Myanmar) missing from the font in use falls back to an installed font made for the script instead of whichever face covering it loaded first; score unchanged (estimated) |
| 2026-10-11 | minor | Proportional Metrics done: `palt` in horizontal type, `vpal` in vertical type (#966, #1082); ruby and manual tate-chu-yoko still missing |